public ElasticQueueDetails(ElasticQueueDetails details)
{
Comment = $"{details.QueueName}";
QueueName = details.QueueName;
Environment = System.Environment.GetEnvironmentVariable("Environment");
}
public static async Task Run([TimerTrigger("0 */5 * * * *")] TimerInfo timer, ILogger log, [Inject] IElasticService elasticService, [Inject] IAlertsService alertsService)
{
log.LogInformation($"{nameof(ServiceBusMonitorTimerFunction)} function executed at: {DateTime.UtcNow:yyyy-MM-dd HH::mm:ss}");
try
{
alertsService.Logger = log;
var elasticIndex = Environment.GetEnvironmentVariable("ElasticIndex");
var managementClient = new ManagementClient(Environment.GetEnvironmentVariable("ServiceBusConnection"));
var queues = await managementClient.GetQueuesAsync();
var alertList = new List <Alert>();
foreach (var queueDescription in queues)
{
log.LogInformation($"Investigating queue '{queueDescription.Path}'");
var queue = await managementClient.GetQueueRuntimeInfoAsync(queueDescription.Path);
var activeMessageCount = queue.MessageCountDetails.ActiveMessageCount;
var deadLetterMessageCount = queue.MessageCountDetails.DeadLetterMessageCount;
log.LogInformation($"Active messages: {activeMessageCount}");
log.LogInformation($"Dead letter messages: {deadLetterMessageCount}");
var list = await elasticService.ElasticQueueDetailsRepository.GetListAsync(queueDescription.Path);
var single = list.FirstOrDefault();
if (single == null)
{
var queueDetails = new ElasticQueueDetails(queueDescription.Path)
{
ActiveMessagesCount = activeMessageCount,
DeadletterMessagesCount = deadLetterMessageCount,
TransferMessagesCount = queue.MessageCountDetails.TransferMessageCount
};
await elasticService.PostAsync(elasticIndex, queueDetails);
}
else
{
var alert = await alertsService.AuditAsync(single, new ElasticQueueDetails(queueDescription.Path)
{
ActiveMessagesCount = activeMessageCount,
DeadletterMessagesCount = deadLetterMessageCount,
TransferMessagesCount = queue.MessageCountDetails.TransferMessageCount
});
if (alert != null)
{
log.LogInformation($"Found alert and added for dispatch," +
$"{Environment.NewLine}" +
$"{Enum.GetName(typeof(Severity), alert.Severity)}" +
$"{Environment.NewLine}" +
$"Details: '{alert.Message}', {queueDescription.Path}" +
$"{Environment.NewLine}");
alertList.Add(alert);
}
single.ActiveMessagesCount = activeMessageCount;
single.DeadletterMessagesCount = deadLetterMessageCount;
single.TransferMessagesCount = queue.MessageCountDetails.TransferMessageCount;
single.Updated = DateTime.UtcNow;
await elasticService.PutAsync(elasticIndex, single);
}
}
if (alertList.Count > 0)
{
var correlations = await alertsService.ProcessAlertsAsync(alertList);
foreach (var correlationId in correlations)
{
log.LogInformation($"Mail message '{correlationId}' was added to queue");
}
}
}
catch (Exception exception)
{
log.LogError($"Error in {nameof(ServiceBusMonitorTimerFunction)}, details: {exception.Message}");
throw;
}
}
public async Task <Alert> AuditAsync(ElasticQueueDetails existingDetails, ElasticQueueDetails createdDetails)
{
if (createdDetails.DeadletterMessagesCount < 1 && existingDetails.DeadletterMessagesCount < 1)
{
return(null);
}
var lowThresholdStr = _configuration["AlertRules:LowThreshold"];
var mediumThresholdStr = _configuration["AlertRules:MediumThreshold"];
var highThresholdStr = _configuration["AlertRules:HighThreshold"];
var differentiatorThresholdStr = _configuration["AlertRules:FiveMinutePercentageThreshold"];
if (!long.TryParse(lowThresholdStr, out var lowThreshold))
{
Logger.LogWarning($"Could not parse lowThreshold '{lowThresholdStr}', setting to default 5");
lowThreshold = 5;
}
if (!long.TryParse(mediumThresholdStr, out var mediumThreshold))
{
Logger.LogWarning($"Could not parse mediumThreshold '{mediumThresholdStr}', setting to default 15");
mediumThreshold = 15;
}
if (!long.TryParse(highThresholdStr, out var highThreshold))
{
Logger.LogWarning($"Could not parse highThreshold '{highThresholdStr}', setting to default 100");
highThreshold = 100;
}
if (!long.TryParse(differentiatorThresholdStr, out var differentiatorThreshold))
{
Logger.LogWarning($"Could not parse differentiatorThreshold '{differentiatorThresholdStr}', setting to default 50");
differentiatorThreshold = 50;
}
var difference = createdDetails.DeadletterMessagesCount - existingDetails.DeadletterMessagesCount;
var increase = difference / existingDetails.DeadletterMessagesCount * 100;
if (increase < 0)
{
Logger.LogInformation($"Since the deadletter count is decreasing, no more alerts will be generated.");
return(null);
}
if (increase > differentiatorThreshold)
{
var message = $"Deadletter message count on '{createdDetails.QueueName}' is {createdDetails.DeadletterMessagesCount} and has increased with {increase}% since last run.";
return(new Alert(Severity.High, "High increase on deadletter", message));
}
if (createdDetails.DeadletterMessagesCount > highThreshold)
{
return(new Alert(Severity.High, "High deadletter count", $"Deadletter message count on '{createdDetails.QueueName}' is {createdDetails.DeadletterMessagesCount}"));
}
if (createdDetails.DeadletterMessagesCount > mediumThreshold)
{
return(new Alert(Severity.Medium, "Medium deadletter count", $"Deadletter message count on '{createdDetails.QueueName}' is {createdDetails.DeadletterMessagesCount}"));
}
if (createdDetails.DeadletterMessagesCount > lowThreshold)
{
return(new Alert(Severity.Low, "Notable deadletter count", $"Deadletter message count on '{createdDetails.QueueName}' is {createdDetails.DeadletterMessagesCount}"));
}
return(null);
}
