/// <summary>
/// Close SCOM Alert associated with the Alarm
/// </summary>
/// <param name="eventObject"></param>
/// <param name="monitoringObject"></param>
private void CloseSCOMAlert(EventData eventObject, MonitoringDeviceObject monitoringObject)
{
var alarm = eventObject.AlarmData;
logger.Polling.Info($"[{alarm.Sn}] alarm is cleared, close SCOM alert now.");
// We will identify the alert using suppression rule.
// Already closed alerts should be ignored.
var criteria = $"ResolutionState != '255' and CustomField1 = '{this.FusionDirector.UniqueId}' " +
$"and CustomField3 = '{eventObject.AlarmSn}'";
ReadOnlyCollection <MonitoringAlert> alerts = monitoringObject.Device.GetMonitoringAlerts(new MonitoringAlertCriteria(criteria));
if (alerts.Count == 0)
{
logger.Polling.Warn($"[{alarm.Sn}] No un-closed SCOM alert is associated with current alarm.");
}
else
{
// It should be 1 in normal sutiation.
logger.Polling.Info($"[{alarm.Sn}] Associated SCOM alerts count is: {alerts.Count}.");
}
foreach (MonitoringAlert alert in alerts)
{
alert.ResolutionState = EnclosureConnector.Instance.CloseState.ResolutionState;
var reason = !string.IsNullOrEmpty(eventObject.AlarmData.ClearType) ? eventObject.AlarmData.ClearType : "Receive alarm cleared notification from subscription.";
alert.Update(reason);
logger.Polling.Info($"[{alarm.Sn}] Close SCOM alert successfully.");
}
}
/// <summary>
/// Start Device event processor
/// </summary>
private void StartNeDeviceEventProcessor()
{
if (deviceEventProcessor == null)
{
deviceEventProcessor = new Thread(delegate()
{
while (this.IsRunning)
{
logger.Polling.Info($"Current Device Event Processing Queue amount: {deviceEventQueue.Count}.");
if (deviceEventQueue.Count > 0 || receiveDeviceEvent.WaitOne())
{
NedeviceData eventObject = null;
lock (deviceEventQueuelocker)
{
if (deviceEventQueue.Count > 0)
{
eventObject = deviceEventQueue.Dequeue();
}
}
if (eventObject != null)
{
logger.Polling.Info($"[{eventObject.DeviceId}] Start processing new device event.");
// should we catagory alarm with server type ?
var deviceId = eventObject.DeviceId;
MonitoringDeviceObject monitoringObject = GetDeviceByDN(deviceId);
if (monitoringObject == null)
{
logger.Polling.Error($"[{eventObject.DeviceId}] No MonitoringObject({deviceId}) exists, event will be ignore");
continue;
}
// waiting for monitoring-object ready.
WaitForDeviceMonitored(monitoringObject);
// Create New EventLog for new events, and generate SCOM event through associated rule
CreateNewEventLogForDeviceEvent(eventObject);
}
}
}
});
}
this.deviceEventProcessor.Start();
logger.Polling.Info("Device event processor starts successfully.");
}
/// <summary>
/// Waiting for device object monitored.
/// </summary>
/// <param name="obj"></param>
public bool WaitForDeviceMonitored(MonitoringDeviceObject obj)
{
// When an object is first created, it's status is "not monitoring".
// The status will changed when Monitor run with a configed interval.
TimeSpan expectTimeLong = TimeSpan.FromMinutes(10);
TimeSpan maxWaitTimeLong = TimeSpan.FromMinutes(15);
if (obj.Device.StateLastModified == null)
{
while (obj.Device.HealthState == HealthState.Uninitialized)
{
this.logger.Polling.Info($"MonitoringObject({obj.DeviceId}) is not monitoring.");
this.logger.Polling.Info($" Device added time is: {obj.Device.TimeAdded}.");
this.logger.Polling.Info($" Device last modified time is: {obj.Device.LastModified}.");
this.logger.Polling.Info($" Device state last modified time is: {obj.Device.StateLastModified}.");
logger.Polling.Info($"Current health state for device `{obj.DeviceId}` is {obj.Device.HealthState}");
// Do not know why RecalculateMonitoringState will stop Service, So, we just wait the monitor run automate.
var stateLastModified = obj.Device.StateLastModified ?? obj.Device.LastModified;
TimeSpan stateNotChangedTimeLong = DateTime.UtcNow - stateLastModified;
// the interval of monitor for our object is 5 minutes. So we will wait 5m.
if (stateNotChangedTimeLong <= expectTimeLong)
{
Thread.Sleep(expectTimeLong - stateNotChangedTimeLong);
}
else if (stateNotChangedTimeLong <= maxWaitTimeLong)
{
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
logger.Polling.Info($"Max waiting time for device's monitoring status excess, will skip current device for now.");
return(false);
}
obj.Reload();
logger.Polling.Info($"Current health state for device `{obj.DeviceId}` is {obj.Device.HealthState}");
}
}
return(true);
}
/// <summary>
/// Close SCOM Alert associated with the Alarm
/// </summary>
/// <param name="eventObject"></param>
/// <param name="monitoringObject"></param>
private void CloseSCOMAlert(EventData eventObject, MonitoringDeviceObject monitoringObject)
{
var alarm = eventObject.AlarmData;
logger.Polling.Info($"[{alarm.AlarmSN}] ESight alarm is cleared, close SCOM alert now.");
/**
* <Suppression>
* <SuppressionValue>$Data/Params/Param[3]$</SuppressionValue>
* <SuppressionValue>$Data/Params/Param[6]$</SuppressionValue>
* <SuppressionValue>$Data/Params/Param[11]$</SuppressionValue>
* </Suppression>
*/
// We will identify the alert using suppression rule.
// Already closed alerts should be ignored.
var criteria = $"ResolutionState != '255' and CustomField5 = '{eventObject.AlarmData.AlarmId}' " +
$"and CustomField8 = '{eventObject.AlarmData.ObjectInstance}'";
ReadOnlyCollection <MonitoringAlert> alerts = monitoringObject.Device.GetMonitoringAlerts(new MonitoringAlertCriteria(criteria));
if (alerts.Count == 0)
{
logger.Polling.Warn($"[{alarm.AlarmSN}] No un-closed SCOM alert is associated with current alarm.");
}
else
{
// It should be 1 in normal sutiation.
logger.Polling.Info($"[{alarm.AlarmSN}] Associated SCOM alerts count is {alerts.Count}.");
}
foreach (MonitoringAlert alert in alerts)
{
alert.ResolutionState = BladeConnector.Instance.CloseState.ResolutionState;
var reason = AlarmClearType.GetClearReason(eventObject.AlarmData.ClearedType);
alert.Update($"Close by ESight: {reason}.");
logger.Polling.Info($"[{alarm.AlarmSN}] Close SCOM alert successfully.");
}
}
/// <summary>
///启用插入事件的任务
/// </summary>
/// <returns>Task.</returns>
private void StartAlarmEventProcessor()
{
if (alarmProcessor == null)
{
alarmProcessor = new Thread(delegate()
{
while (this.IsRunning) // TODO(turnbig) 假如Queue里面还有未处理的数据，直接丢弃？
{
logger.Polling.Info($"Current Alarm Processing Queue amount: {alarmQueue.Count}.");
if (alarmQueue.Count > 0 || receiveAlarmEvent.WaitOne())
{
AlarmData alarm = null;
lock (this.locker)
{
if (alarmQueue.Count > 0)
{
alarm = alarmQueue.Dequeue();
}
}
if (alarm != null)
{
EventData eventObject = new EventData(alarm, this.ESightIp);
logger.Polling.Info($"[{alarm.AlarmSN}] Start processing new alarm: Cleared: {alarm.Cleared}, OptType: {alarm.OptType}, " +
$"EventType: {alarm.EventType}, EventLevel: {eventObject.LevelId.ToString()}");
// should we catagory alarm with server type ?
var deviceId = eventObject.DeviceId;
MonitoringDeviceObject monitoringObject = this.GetDeviceByDeveceId(deviceId);
if (monitoringObject == null)
{
logger.Polling.Error($"[{alarm.AlarmSN}] No MonitoringObject({deviceId}) exists, alarm will be ignore");
continue;
}
// waiting for monitoring-object ready.
WaitForDeviceMonitored(monitoringObject);
var now = DateTime.Now;
ProcessedOn previewNProcessedOn = null;
if (eventObject.Cleared)
{
// Close SCOM alert
CloseSCOMAlert(eventObject, monitoringObject);
}
else if (ShouldProcessedAlarmLevels.Contains(eventObject.LevelId))
{
// Create New EventLog for new alarms, and generate SCOM alert through associated rule
CreateNewEventLogForAlarm(eventObject);
previewNProcessedOn = processedOnQueue.Enqueue(new ProcessedOn(now));
}
else if (eventObject.OptType == 6)
{
// Create New EventLog for new events, and generate SCOM event through associated rule
CreateNewEventLogForEvent(eventObject);
previewNProcessedOn = processedOnQueue.Enqueue(new ProcessedOn(now));
}
if (previewNProcessedOn != null)
{
TimeSpan timeSpan = now - previewNProcessedOn.Timestamp;
// do not know why system time was changed to yestoday.
if (now >= previewNProcessedOn.Timestamp && timeSpan < RateLimitTimeSpan)
{
TimeSpan timeout = RateLimitTimeSpan - timeSpan;
logger.Polling.Info($"Alarm processing reach rate limit, {processedOnQueue.Size} alarms have been processed during time span {timeSpan}, will sleep {timeout} now.");
Thread.Sleep(timeout);
}
}
}
}
}
});
}
this.alarmProcessor.Start();
logger.Polling.Info("Alarm processor starts successfully.");
}
/// <summary>
/// 处理告警队列
/// </summary>
private void StartAlarmEventProcessor()
{
Version = MGroup.Instance.Version;
logger.Polling.Info("Current Management Server version is: " + Version.ToString());
RateLimitQueueSize = this.pluginConfig.RateLimitQueueSize;
RateLimitTimeSpan = TimeSpan.FromSeconds(this.pluginConfig.RateLimitTimeSpan);
if (AlarmProcessor == null)
{
AlarmProcessor = new Thread(delegate()
{
while (this.IsRunning) // TODO(turnbig) 假如Queue里面还有未处理的数据，直接丢弃？
{
logger.Polling.Info($"Current Alarm Processing Queue amount: {AlarmQueue.Count}.");
if (AlarmQueue.Count > 0 || ReceiveAlarmEvent.WaitOne())
{
AlarmData alarm = null;
lock (this.locker)
{
if (AlarmQueue.Count > 0)
{
alarm = AlarmQueue.Dequeue();
}
}
if (alarm != null)
{
EventData eventObject = new EventData(alarm, this.FusionDirectorIp);
logger.Polling.Info($"[{alarm.Sn}] Start processing alarm:: Source:{alarm.DeviceId}, Category:{alarm.EventCategory}, Status: {alarm.Status}.");
var objectId = eventObject.UnionId;
string mpClazzName = EventCategory.BMC.Equals(alarm.EventCategory) ? EntityTypeConst.Server.MainName : EntityTypeConst.Enclosure.MainName;
ManagementPackClass mpClazz = MGroup.Instance.GetManagementPackClass(mpClazzName);
MonitoringDeviceObject monitoringObject = BaseConnector.GetDeviceByObjectId(mpClazz, objectId);
if (monitoringObject == null)
{
// TODO(turnbig.net) should we trigger an update server task, and retry later?
logger.Polling.Warn($"[{alarm.Sn}] No MonitoringObject({objectId}) exists, alarm will be ignored.");
continue;
}
// waiting for monitoring-object ready.
WaitForDeviceMonitored(monitoringObject);
var now = DateTime.Now;
ProcessedOn previewNProcessedOn = null;
FixedSizedQueue <ProcessedOn> processedOnQueue = GetProcessedOnQueue(eventObject);
if (EventStatus.Cleared.Equals(alarm.Status))
{
// Close SCOM alert
CloseSCOMAlert(eventObject, monitoringObject);
}
else if (ShouldProcessedAlarmLevels.Contains(eventObject.LevelId))
{
// Create New EventLog for new alarms, and generate SCOM alert through associated rule
CreateNewEventLogForAlarm(eventObject);
// use a seperated process on tracker
previewNProcessedOn = processedOnQueue.Enqueue(new ProcessedOn(now));
}
if (previewNProcessedOn != null)
{
TimeSpan timeSpan = now - previewNProcessedOn.Timestamp;
// do not know why system time was changed to yestoday.
if (now >= previewNProcessedOn.Timestamp && timeSpan < RateLimitTimeSpan)
{
TimeSpan timeout = RateLimitTimeSpan - timeSpan;
logger.Polling.Info($"Alarm processing reach rate limit, {processedOnQueue.Size} alarms have been processed during time span {timeSpan}, will sleep {timeout} now.");
Thread.Sleep(timeout);
}
}
}
}
}
});
}
this.AlarmProcessor.Start();
logger.Polling.Info("Alarm processor starts successfully.");
}