622 lines
18 KiB
JavaScript
622 lines
18 KiB
JavaScript
const logger = require('../utils/logger');
|
|
const database = require('../config/database');
|
|
const redis = require('../config/redis');
|
|
const notificationService = require('./notificationService');
|
|
|
|
class HealingService {
|
|
constructor() {
|
|
this.healingRules = new Map();
|
|
this.activeHealingActions = new Map();
|
|
this.isInitialized = false;
|
|
}
|
|
|
|
async initialize() {
|
|
try {
|
|
await this.loadHealingRules();
|
|
await this.setupHealingMonitoring();
|
|
this.isInitialized = true;
|
|
logger.info('Healing service initialized successfully');
|
|
} catch (error) {
|
|
logger.error('Failed to initialize Healing service:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async loadHealingRules() {
|
|
try {
|
|
const rules = await database.query('SELECT * FROM healing_rules WHERE status = "active"');
|
|
|
|
for (const rule of rules) {
|
|
this.healingRules.set(rule.id, {
|
|
...rule,
|
|
conditions: JSON.parse(rule.conditions),
|
|
actions: JSON.parse(rule.actions)
|
|
});
|
|
}
|
|
|
|
logger.info(`Loaded ${rules.length} healing rules`);
|
|
} catch (error) {
|
|
logger.error('Failed to load healing rules:', error);
|
|
}
|
|
}
|
|
|
|
async setupHealingMonitoring() {
|
|
// Monitor active healing actions every 30 seconds
|
|
setInterval(async () => {
|
|
await this.monitorActiveHealingActions();
|
|
}, 30 * 1000);
|
|
}
|
|
|
|
async triggerHealing(alert) {
|
|
try {
|
|
const startTime = Date.now();
|
|
|
|
// Find applicable healing rules
|
|
const applicableRules = await this.findApplicableHealingRules(alert);
|
|
|
|
if (applicableRules.length === 0) {
|
|
logger.info(`No healing rules found for alert: ${alert.type}`);
|
|
return null;
|
|
}
|
|
|
|
// Execute healing actions
|
|
const healingActions = [];
|
|
for (const rule of applicableRules) {
|
|
const action = await this.executeHealingRule(rule, alert);
|
|
if (action) {
|
|
healingActions.push(action);
|
|
}
|
|
}
|
|
|
|
const processingTime = Date.now() - startTime;
|
|
logger.logHealingAction('trigger_healing', alert.deviceId, {
|
|
alertType: alert.type,
|
|
rulesApplied: applicableRules.length,
|
|
actionsExecuted: healingActions.length
|
|
}, processingTime);
|
|
|
|
return healingActions;
|
|
} catch (error) {
|
|
logger.error('Error triggering healing:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async findApplicableHealingRules(alert) {
|
|
try {
|
|
const applicableRules = [];
|
|
|
|
for (const [ruleId, rule] of this.healingRules) {
|
|
if (this.isRuleApplicable(rule, alert)) {
|
|
applicableRules.push(rule);
|
|
}
|
|
}
|
|
|
|
return applicableRules;
|
|
} catch (error) {
|
|
logger.error('Error finding applicable healing rules:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
isRuleApplicable(rule, alert) {
|
|
try {
|
|
// Check if rule applies to this device
|
|
if (rule.device_id && rule.device_id !== alert.deviceId) {
|
|
return false;
|
|
}
|
|
|
|
// Check if rule applies to this alert type
|
|
if (rule.alert_types && !rule.alert_types.includes(alert.type)) {
|
|
return false;
|
|
}
|
|
|
|
// Check if rule applies to this severity
|
|
if (rule.severity_levels && !rule.severity_levels.includes(alert.severity)) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
} catch (error) {
|
|
logger.error('Error checking rule applicability:', error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async executeHealingRule(rule, alert) {
|
|
try {
|
|
const healingAction = {
|
|
ruleId: rule.id,
|
|
deviceId: alert.deviceId,
|
|
alertId: alert.id,
|
|
type: rule.healing_type,
|
|
description: rule.description,
|
|
actions: rule.actions,
|
|
priority: rule.priority,
|
|
status: 'pending',
|
|
created_at: new Date()
|
|
};
|
|
|
|
// Store healing action
|
|
const actionId = await this.storeHealingAction(healingAction);
|
|
healingAction.id = actionId;
|
|
|
|
// Execute healing actions
|
|
const results = await this.executeHealingActions(healingAction);
|
|
|
|
// Update action status
|
|
const success = results.every(result => result.success);
|
|
await this.updateHealingActionStatus(actionId, success ? 'completed' : 'failed', results);
|
|
|
|
// Log healing action
|
|
logger.logHealingAction(healingAction.type, alert.deviceId, {
|
|
ruleId: rule.id,
|
|
actionsCount: rule.actions.length,
|
|
success: success
|
|
}, 0);
|
|
|
|
return healingAction;
|
|
} catch (error) {
|
|
logger.error('Error executing healing rule:', error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async executeHealingActions(healingAction) {
|
|
try {
|
|
const results = [];
|
|
|
|
for (const action of healingAction.actions) {
|
|
const result = await this.executeAction(action, healingAction);
|
|
results.push(result);
|
|
}
|
|
|
|
return results;
|
|
} catch (error) {
|
|
logger.error('Error executing healing actions:', error);
|
|
return [{ success: false, error: error.message }];
|
|
}
|
|
}
|
|
|
|
async executeAction(action, healingAction) {
|
|
try {
|
|
const startTime = Date.now();
|
|
|
|
switch (action.type) {
|
|
case 'device_restart':
|
|
return await this.executeDeviceRestart(action, healingAction);
|
|
|
|
case 'parameter_adjustment':
|
|
return await this.executeParameterAdjustment(action, healingAction);
|
|
|
|
case 'configuration_update':
|
|
return await this.executeConfigurationUpdate(action, healingAction);
|
|
|
|
case 'maintenance_schedule':
|
|
return await this.executeMaintenanceSchedule(action, healingAction);
|
|
|
|
case 'backup_restore':
|
|
return await this.executeBackupRestore(action, healingAction);
|
|
|
|
case 'load_balancing':
|
|
return await this.executeLoadBalancing(action, healingAction);
|
|
|
|
case 'circuit_breaker':
|
|
return await this.executeCircuitBreaker(action, healingAction);
|
|
|
|
default:
|
|
return { success: false, error: `Unknown action type: ${action.type}` };
|
|
}
|
|
} catch (error) {
|
|
logger.error('Error executing action:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeDeviceRestart(action, healingAction) {
|
|
try {
|
|
// Store restart command
|
|
await database.query(
|
|
`INSERT INTO device_controls
|
|
(device_id, action, parameters, triggered_by_healing, status)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.deviceId,
|
|
'restart',
|
|
JSON.stringify(action.parameters || {}),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
// Send notification
|
|
await notificationService.sendNotification('admin', {
|
|
type: 'healing_action',
|
|
title: 'Device Restart Initiated',
|
|
message: `Device ${healingAction.deviceId} restart initiated by healing system`,
|
|
severity: 'info',
|
|
deviceId: healingAction.deviceId
|
|
});
|
|
|
|
return { success: true, action: 'device_restart' };
|
|
} catch (error) {
|
|
logger.error('Error executing device restart:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeParameterAdjustment(action, healingAction) {
|
|
try {
|
|
const { parameter, value, adjustment_type } = action;
|
|
|
|
// Store parameter adjustment
|
|
await database.query(
|
|
`INSERT INTO device_parameter_adjustments
|
|
(device_id, parameter, old_value, new_value, adjustment_type, healing_action_id, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
parameter,
|
|
action.current_value || 'unknown',
|
|
value,
|
|
adjustment_type || 'manual',
|
|
healingAction.id
|
|
]
|
|
);
|
|
|
|
// Store control command
|
|
await database.query(
|
|
`INSERT INTO device_controls
|
|
(device_id, action, parameters, triggered_by_healing, status)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.deviceId,
|
|
'parameter_adjustment',
|
|
JSON.stringify({ parameter, value, adjustment_type }),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
return { success: true, action: 'parameter_adjustment', parameter, value };
|
|
} catch (error) {
|
|
logger.error('Error executing parameter adjustment:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeConfigurationUpdate(action, healingAction) {
|
|
try {
|
|
const { configuration, backup_existing } = action;
|
|
|
|
if (backup_existing) {
|
|
// Create backup of current configuration
|
|
await this.createConfigurationBackup(healingAction.deviceId);
|
|
}
|
|
|
|
// Store configuration update
|
|
await database.query(
|
|
`INSERT INTO device_configuration_updates
|
|
(device_id, configuration, healing_action_id, created_at)
|
|
VALUES (?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
JSON.stringify(configuration),
|
|
healingAction.id
|
|
]
|
|
);
|
|
|
|
// Store control command
|
|
await database.query(
|
|
`INSERT INTO device_controls
|
|
(device_id, action, parameters, triggered_by_healing, status)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.deviceId,
|
|
'configuration_update',
|
|
JSON.stringify({ configuration, backup_existing }),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
return { success: true, action: 'configuration_update' };
|
|
} catch (error) {
|
|
logger.error('Error executing configuration update:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeMaintenanceSchedule(action, healingAction) {
|
|
try {
|
|
const { maintenance_type, priority, estimated_duration } = action;
|
|
|
|
// Schedule maintenance
|
|
await database.query(
|
|
`INSERT INTO maintenance_schedules
|
|
(device_id, maintenance_type, priority, estimated_duration, healing_action_id, status, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
maintenance_type,
|
|
priority || 'medium',
|
|
estimated_duration || 60,
|
|
healingAction.id,
|
|
'scheduled'
|
|
]
|
|
);
|
|
|
|
// Send notification
|
|
await notificationService.sendNotification('admin', {
|
|
type: 'maintenance_scheduled',
|
|
title: 'Maintenance Scheduled',
|
|
message: `${maintenance_type} maintenance scheduled for device ${healingAction.deviceId}`,
|
|
severity: 'info',
|
|
deviceId: healingAction.deviceId
|
|
});
|
|
|
|
return { success: true, action: 'maintenance_schedule', maintenance_type };
|
|
} catch (error) {
|
|
logger.error('Error executing maintenance schedule:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeBackupRestore(action, healingAction) {
|
|
try {
|
|
const { backup_id, restore_point } = action;
|
|
|
|
// Store backup restore action
|
|
await database.query(
|
|
`INSERT INTO backup_restore_actions
|
|
(device_id, backup_id, restore_point, healing_action_id, status, created_at)
|
|
VALUES (?, ?, ?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
backup_id,
|
|
restore_point || 'latest',
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
// Store control command
|
|
await database.query(
|
|
`INSERT INTO device_controls
|
|
(device_id, action, parameters, triggered_by_healing, status)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.deviceId,
|
|
'backup_restore',
|
|
JSON.stringify({ backup_id, restore_point }),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
return { success: true, action: 'backup_restore', backup_id };
|
|
} catch (error) {
|
|
logger.error('Error executing backup restore:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeLoadBalancing(action, healingAction) {
|
|
try {
|
|
const { target_devices, load_distribution } = action;
|
|
|
|
// Store load balancing action
|
|
await database.query(
|
|
`INSERT INTO load_balancing_actions
|
|
(source_device_id, target_devices, load_distribution, healing_action_id, status, created_at)
|
|
VALUES (?, ?, ?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
JSON.stringify(target_devices),
|
|
JSON.stringify(load_distribution),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
return { success: true, action: 'load_balancing', target_devices };
|
|
} catch (error) {
|
|
logger.error('Error executing load balancing:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async executeCircuitBreaker(action, healingAction) {
|
|
try {
|
|
const { circuit_state, timeout } = action;
|
|
|
|
// Store circuit breaker action
|
|
await database.query(
|
|
`INSERT INTO circuit_breaker_actions
|
|
(device_id, circuit_state, timeout, healing_action_id, status, created_at)
|
|
VALUES (?, ?, ?, ?, ?, NOW())`,
|
|
[
|
|
healingAction.deviceId,
|
|
circuit_state || 'open',
|
|
timeout || 300,
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
// Store control command
|
|
await database.query(
|
|
`INSERT INTO device_controls
|
|
(device_id, action, parameters, triggered_by_healing, status)
|
|
VALUES (?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.deviceId,
|
|
'circuit_breaker',
|
|
JSON.stringify({ circuit_state, timeout }),
|
|
healingAction.id,
|
|
'pending'
|
|
]
|
|
);
|
|
|
|
return { success: true, action: 'circuit_breaker', circuit_state };
|
|
} catch (error) {
|
|
logger.error('Error executing circuit breaker:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async storeHealingAction(healingAction) {
|
|
try {
|
|
const result = await database.query(
|
|
`INSERT INTO healing_actions
|
|
(rule_id, device_id, alert_id, type, description, actions, priority, status, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
[
|
|
healingAction.ruleId,
|
|
healingAction.deviceId,
|
|
healingAction.alertId,
|
|
healingAction.type,
|
|
healingAction.description,
|
|
JSON.stringify(healingAction.actions),
|
|
healingAction.priority,
|
|
healingAction.status,
|
|
healingAction.created_at
|
|
]
|
|
);
|
|
|
|
return result.insertId;
|
|
} catch (error) {
|
|
logger.error('Failed to store healing action:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async updateHealingActionStatus(actionId, status, results = []) {
|
|
try {
|
|
await database.query(
|
|
`UPDATE healing_actions
|
|
SET status = ?, results = ?, updated_at = NOW()
|
|
WHERE id = ?`,
|
|
[status, JSON.stringify(results), actionId]
|
|
);
|
|
} catch (error) {
|
|
logger.error('Failed to update healing action status:', error);
|
|
}
|
|
}
|
|
|
|
async monitorActiveHealingActions() {
|
|
try {
|
|
const activeActions = await database.query(
|
|
'SELECT * FROM healing_actions WHERE status IN ("pending", "in_progress")'
|
|
);
|
|
|
|
for (const action of activeActions) {
|
|
await this.checkHealingActionProgress(action);
|
|
}
|
|
} catch (error) {
|
|
logger.error('Error monitoring active healing actions:', error);
|
|
}
|
|
}
|
|
|
|
async checkHealingActionProgress(action) {
|
|
try {
|
|
// Check if healing action has been completed
|
|
const controls = await database.query(
|
|
'SELECT status FROM device_controls WHERE triggered_by_healing = ?',
|
|
[action.id]
|
|
);
|
|
|
|
if (controls.length > 0) {
|
|
const allCompleted = controls.every(control => control.status === 'completed');
|
|
const anyFailed = controls.some(control => control.status === 'failed');
|
|
|
|
if (allCompleted) {
|
|
await this.updateHealingActionStatus(action.id, 'completed');
|
|
} else if (anyFailed) {
|
|
await this.updateHealingActionStatus(action.id, 'failed');
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.error('Error checking healing action progress:', error);
|
|
}
|
|
}
|
|
|
|
async createConfigurationBackup(deviceId) {
|
|
try {
|
|
// Get current device configuration
|
|
const [device] = await database.query(
|
|
'SELECT configuration FROM devices WHERE id = ?',
|
|
[deviceId]
|
|
);
|
|
|
|
if (device && device.configuration) {
|
|
await database.query(
|
|
`INSERT INTO device_configuration_backups
|
|
(device_id, configuration, backup_type, created_at)
|
|
VALUES (?, ?, ?, NOW())`,
|
|
[deviceId, device.configuration, 'healing_backup']
|
|
);
|
|
}
|
|
} catch (error) {
|
|
logger.error('Error creating configuration backup:', error);
|
|
}
|
|
}
|
|
|
|
async getHealingHistory(deviceId = null, limit = 100) {
|
|
try {
|
|
let query = 'SELECT * FROM healing_actions WHERE 1=1';
|
|
const params = [];
|
|
|
|
if (deviceId) {
|
|
query += ' AND device_id = ?';
|
|
params.push(deviceId);
|
|
}
|
|
|
|
query += ' ORDER BY created_at DESC LIMIT ?';
|
|
params.push(limit);
|
|
|
|
return await database.query(query, params);
|
|
} catch (error) {
|
|
logger.error('Error getting healing history:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async getHealingStatistics() {
|
|
try {
|
|
const stats = await database.query(`
|
|
SELECT
|
|
status,
|
|
COUNT(*) as count,
|
|
AVG(TIMESTAMPDIFF(SECOND, created_at, updated_at)) as avg_duration
|
|
FROM healing_actions
|
|
WHERE created_at >= DATE_SUB(NOW(), INTERVAL 24 HOUR)
|
|
GROUP BY status
|
|
`);
|
|
|
|
return stats;
|
|
} catch (error) {
|
|
logger.error('Error getting healing statistics:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async healthCheck() {
|
|
try {
|
|
return {
|
|
status: this.isInitialized ? 'healthy' : 'not_initialized',
|
|
message: this.isInitialized ? 'Healing service is healthy' : 'Service not initialized',
|
|
activeRules: this.healingRules.size,
|
|
activeActions: this.activeHealingActions.size
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
status: 'unhealthy',
|
|
message: 'Healing service health check failed',
|
|
error: error.message
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = new HealingService();
|