private QueueMetricViewModel CreateMetricModel(List <IncomingMetrics> incomingMetrics)
{
List <QueueMetricModel> metricModels = new List <QueueMetricModel>();
QueueMetricViewModel model = new QueueMetricViewModel();
foreach (var incomingMetric in incomingMetrics)
{
foreach (var value in incomingMetric.Value)
{
foreach (var time in value.Timeseries)
{
foreach (var metadata in time.Metadatavalues)
{
QueueMetricModel metricModel = new QueueMetricModel();
List <QueueMetric> metrics = new List <QueueMetric>();
foreach (var data in time.Data)
{
QueueMetric metric = new QueueMetric(data.TimeStamp, data.Total);
metrics.Add(metric);
}
metricModel.MetricName = value.Name.Value;
metricModel.QueueMetrics = metrics;
metricModel.QueueName = metadata.Value;
metricModels.Add(metricModel);
}
}
}
}
model.QueueMetrics = metricModels;
return(model);
}
internal virtual async Task <bool> UpdateQueueMetrics()
{
CloudQueue workItemQueue = AzureStorageOrchestrationService.GetWorkItemQueue(this.storageAccount, this.taskHub);
CloudQueue[] controlQueues = await AzureStorageOrchestrationService.GetControlQueuesAsync(
this.storageAccount,
this.taskHub,
defaultPartitionCount : AzureStorageOrchestrationServiceSettings.DefaultPartitionCount);
Task <QueueMetric> workItemMetricTask = GetQueueMetricsAsync(workItemQueue);
List <Task <QueueMetric> > controlQueueMetricTasks = controlQueues.Select(GetQueueMetricsAsync).ToList();
var tasks = new List <Task>(controlQueueMetricTasks.Count + 1);
tasks.Add(workItemMetricTask);
tasks.AddRange(controlQueueMetricTasks);
try
{
await Task.WhenAll(tasks);
}
catch (StorageException e) when(e.RequestInformation?.HttpStatusCode == 404)
{
// The queues are not yet provisioned.
AnalyticsEventSource.Log.MonitorWarning(
this.storageAccount.Credentials.AccountName,
this.taskHub,
$"Task hub has not been provisioned: {e.RequestInformation.ExtendedErrorInformation?.ErrorMessage}");
return(false);
}
QueueMetric workItemQueueMetric = workItemMetricTask.Result;
this.WorkItemQueueLatencies.Add((int)workItemQueueMetric.Latency.TotalMilliseconds);
int i;
for (i = 0; i < controlQueueMetricTasks.Count; i++)
{
QueueMetric controlQueueMetric = controlQueueMetricTasks[i].Result;
if (i >= this.ControlQueueLatencies.Count)
{
this.ControlQueueLatencies.Add(new QueueMetricHistory(QueueLengthSampleSize));
}
this.ControlQueueLatencies[i].Add((int)controlQueueMetric.Latency.TotalMilliseconds);
}
// Handle the case where the number of control queues has been reduced since we last checked.
while (i < this.ControlQueueLatencies.Count && this.ControlQueueLatencies.Count > 0)
{
this.ControlQueueLatencies.RemoveAt(this.ControlQueueLatencies.Count - 1);
}
this.currentPartitionCount = controlQueues.Length;
this.currentWorkItemQueueLength = workItemQueueMetric.Length;
this.currentControlQueueLengths = controlQueueMetricTasks.Select(t => t.Result.Length).ToArray();
return(true);
}
/// <summary>
/// Read a message from the queue.
/// </summary>
/// <param name="onDequeue">The action to take when recieving a message</param>
/// <param name="onError">If an error occurs, provide an action to take.</param>
/// <param name="exchangeName">Name of the exchange.</param>
/// <param name="queueName">Name of the queue.</param>
/// <param name="routingKeyName">Name of the routing key.</param>
public void ReadFromQueue(Action <string, QueueConsumerService, ulong, QueueMetric> onDequeue, Action <Exception, QueueConsumerService, ulong, QueueMetric> onError,
string exchangeName, string queueName, string routingKeyName)
{
BindToQueue(exchangeName, queueName, routingKeyName);
if (this.EventingBasicConsumer == null)
{
EventingBasicConsumer = new EventingBasicConsumer(Model);
}
// Receive the message from the queue and act on that message
EventingBasicConsumer.Received += (o, e) =>
{
_logger.LogInformation("Messge Received handler invoked.");
var queueMetric = new QueueMetric
{
QueueName = this.QueueName,
ExchangeName = this.ExchangeName,
RoutingKeyName = this.RoutingKeyName,
ConsumedDateTime = DateTime.UtcNow,
InstanceId = GetInstanceId(),
};
try
{
var queuedMessage = Encoding.ASCII.GetString(e.Body);
queueMetric.MessageLength = queuedMessage.Length;
onDequeue.Invoke(queuedMessage, this, e.DeliveryTag, queueMetric);
_metricsRepository.SaveMetric(queueMetric).GetAwaiter();
}
catch (Exception ex)
{
onError.Invoke(ex, this, e.DeliveryTag, queueMetric);
_metricsRepository.SaveMetric(queueMetric).GetAwaiter();
}
};
// If the consumer shutdowns reconnect to rabbit and begin reading from the queue again.
EventingBasicConsumer.Shutdown += (o, e) =>
{
Connect();
ReadFromQueue(onDequeue, onError, exchangeName, queueName, routingKeyName);
};
Model.BasicConsume(queueName, false, EventingBasicConsumer);
}
public MessageQueue(IMessageFabricObserver observer, string name)
{
_observer = observer;
_name = name;
_receivers = new MessageReceiverCollection(receivers => new RoundRobinReceiverLoadBalancer(receivers));
_metrics = new QueueMetric(name);
_channel = Channel.CreateUnbounded <DeliveryContext <GrpcTransportMessage> >(new UnboundedChannelOptions
{
SingleWriter = false,
SingleReader = true,
AllowSynchronousContinuations = false
});
_dispatcher = Task.Run(() => StartDispatcher());
}
private void RaiseException(Exception ex, IQueueConsumerService queueConsumerService, ulong deliveryTag, QueueMetric queueMetric)
{
queueConsumerService.Model.BasicNack(deliveryTag, false, false);
queueMetric.RoutingAction = RoutingAction.Failed;
_logger.LogError($"Error raised from QueueService: {ex.Message}");
}
public void ProcessMessage(string message, IQueueConsumerService queueConsumerService, ulong deliveryTag, QueueMetric queueMetric)
{
var handlerFunc = ResolveHandler();
if (handlerFunc.Invoke(message))
{
queueConsumerService.Model.BasicAck(deliveryTag, false);
queueMetric.RoutingAction = RoutingAction.Processed;
return;
}
this.RaiseException(new Exception("Message not processed."), queueConsumerService, deliveryTag, queueMetric);
}
public async Task SaveMetric(QueueMetric metric)
{
var db = GetDatabase();
await db.HashSetAsync("QueueMetric-" + metric.Id, metric.ToHashEntries());
}
