private FixedSizedQueue <ProcessedOn> GetProcessedOnQueue(EventData eventObject)
{
// for SCOM below 2019(10.x.x.x), Rule Rate limit has bug, we use single ProcessOnQueue for all rules
var key = "Common";
if (Version.Major >= 10)
{
key = $"{eventObject.AlarmData.EventCategory}_{eventObject.EventId}"; // group by rule
}
bool exists = ProcessedOnQueueMap.ContainsKey(key);
if (!exists)
{
FixedSizedQueue <ProcessedOn> ProcessedOnQueue = new FixedSizedQueue <ProcessedOn>(RateLimitQueueSize);
ProcessedOnQueueMap.Add(key, ProcessedOnQueue);
}
return(ProcessedOnQueueMap[key]);
}
/// <summary>
/// 处理告警队列
/// </summary>
private void StartAlarmEventProcessor()
{
Version = MGroup.Instance.Version;
logger.Polling.Info("Current Management Server version is: " + Version.ToString());
RateLimitQueueSize = this.pluginConfig.RateLimitQueueSize;
RateLimitTimeSpan = TimeSpan.FromSeconds(this.pluginConfig.RateLimitTimeSpan);
if (AlarmProcessor == null)
{
AlarmProcessor = new Thread(delegate()
{
while (this.IsRunning) // TODO(turnbig) 假如Queue里面还有未处理的数据，直接丢弃？
{
logger.Polling.Info($"Current Alarm Processing Queue amount: {AlarmQueue.Count}.");
if (AlarmQueue.Count > 0 || ReceiveAlarmEvent.WaitOne())
{
AlarmData alarm = null;
lock (this.locker)
{
if (AlarmQueue.Count > 0)
{
alarm = AlarmQueue.Dequeue();
}
}
if (alarm != null)
{
EventData eventObject = new EventData(alarm, this.FusionDirectorIp);
logger.Polling.Info($"[{alarm.Sn}] Start processing alarm:: Source:{alarm.DeviceId}, Category:{alarm.EventCategory}, Status: {alarm.Status}.");
var objectId = eventObject.UnionId;
string mpClazzName = EventCategory.BMC.Equals(alarm.EventCategory) ? EntityTypeConst.Server.MainName : EntityTypeConst.Enclosure.MainName;
ManagementPackClass mpClazz = MGroup.Instance.GetManagementPackClass(mpClazzName);
MonitoringDeviceObject monitoringObject = BaseConnector.GetDeviceByObjectId(mpClazz, objectId);
if (monitoringObject == null)
{
// TODO(turnbig.net) should we trigger an update server task, and retry later?
logger.Polling.Warn($"[{alarm.Sn}] No MonitoringObject({objectId}) exists, alarm will be ignored.");
continue;
}
// waiting for monitoring-object ready.
WaitForDeviceMonitored(monitoringObject);
var now = DateTime.Now;
ProcessedOn previewNProcessedOn = null;
FixedSizedQueue <ProcessedOn> processedOnQueue = GetProcessedOnQueue(eventObject);
if (EventStatus.Cleared.Equals(alarm.Status))
{
// Close SCOM alert
CloseSCOMAlert(eventObject, monitoringObject);
}
else if (ShouldProcessedAlarmLevels.Contains(eventObject.LevelId))
{
// Create New EventLog for new alarms, and generate SCOM alert through associated rule
CreateNewEventLogForAlarm(eventObject);
// use a seperated process on tracker
previewNProcessedOn = processedOnQueue.Enqueue(new ProcessedOn(now));
}
if (previewNProcessedOn != null)
{
TimeSpan timeSpan = now - previewNProcessedOn.Timestamp;
// do not know why system time was changed to yestoday.
if (now >= previewNProcessedOn.Timestamp && timeSpan < RateLimitTimeSpan)
{
TimeSpan timeout = RateLimitTimeSpan - timeSpan;
logger.Polling.Info($"Alarm processing reach rate limit, {processedOnQueue.Size} alarms have been processed during time span {timeSpan}, will sleep {timeout} now.");
Thread.Sleep(timeout);
}
}
}
}
}
});
}
this.AlarmProcessor.Start();
logger.Polling.Info("Alarm processor starts successfully.");
}