public async Task MonitorIdleTaskHubDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIdleTaskHubDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
await service.DeleteAsync();
// A null heartbeat is expected when the task hub does not exist.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNull(heartbeat);
await service.CreateAsync();
ScaleRecommendation recommendation;
for (int i = 0; i < 10; i++)
{
heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNotNull(heartbeat);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLatencies.Count);
Assert.AreEqual(0, heartbeat.ControlQueueLengths.Count(l => l != 0));
Assert.AreEqual(0, heartbeat.ControlQueueLatencies.Count(l => l != TimeSpan.Zero));
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
// If any workers are assigned, the recommendation should be to have them removed.
heartbeat = await monitor.PulseAsync(currentWorkerCount : 1);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
public async Task <DurableTaskTriggerMetrics> GetMetricsAsync()
{
DurableTaskTriggerMetrics metrics = new DurableTaskTriggerMetrics();
// Durable stores its own metrics, so we just collect them here
PerformanceHeartbeat heartbeat = null;
try
{
DisconnectedPerformanceMonitor performanceMonitor = this.GetPerformanceMonitor();
heartbeat = await performanceMonitor.PulseAsync();
}
catch (StorageException e)
{
this.traceHelper.ExtensionWarningEvent(this.hubName, this.functionName.Name, string.Empty, e.ToString());
}
if (heartbeat != null)
{
metrics.PartitionCount = heartbeat.PartitionCount;
metrics.ControlQueueLengths = JsonConvert.SerializeObject(heartbeat.ControlQueueLengths);
metrics.ControlQueueLatencies = JsonConvert.SerializeObject(heartbeat.ControlQueueLatencies);
metrics.WorkItemQueueLength = heartbeat.WorkItemQueueLength;
if (heartbeat.WorkItemQueueLatency != null)
{
metrics.WorkItemQueueLatency = heartbeat.WorkItemQueueLatency.ToString();
}
}
return(metrics);
}
public async Task MonitorIncreasingControlQueueLoadDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings()
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
int simulatedWorkerCount = 0;
await service.CreateAsync();
// A heartbeat should come back with no recommendation since there is no data.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
Assert.IsNotNull(heartbeat.ScaleRecommendation);
Assert.AreEqual(ScaleAction.None, heartbeat.ScaleRecommendation.Action);
Assert.IsFalse(heartbeat.ScaleRecommendation.KeepWorkersAlive);
var client = new TaskHubClient(service);
var previousTotalLatency = TimeSpan.Zero;
for (int i = 1; i < settings.PartitionCount + 10; i++)
{
await client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input : null);
heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.IsTrue(recommendation.KeepWorkersAlive);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(i, heartbeat.ControlQueueLengths.Sum());
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
TimeSpan currentTotalLatency = TimeSpan.FromTicks(heartbeat.ControlQueueLatencies.Sum(ts => ts.Ticks));
Assert.IsTrue(currentTotalLatency > previousTotalLatency);
if (i + 1 < DisconnectedPerformanceMonitor.QueueLengthSampleSize)
{
int queuesWithNonZeroLatencies = heartbeat.ControlQueueLatencies.Count(t => t > TimeSpan.Zero);
Assert.IsTrue(queuesWithNonZeroLatencies > 0 && queuesWithNonZeroLatencies <= i);
int queuesWithAtLeastOneMessage = heartbeat.ControlQueueLengths.Count(l => l > 0);
Assert.IsTrue(queuesWithAtLeastOneMessage > 0 && queuesWithAtLeastOneMessage <= i);
ScaleAction expectedScaleAction = simulatedWorkerCount == 0 ? ScaleAction.AddWorker : ScaleAction.None;
Assert.AreEqual(expectedScaleAction, recommendation.Action);
}
else
{
// Validate that control queue latencies are going up with each iteration.
Assert.IsTrue(currentTotalLatency.Ticks > previousTotalLatency.Ticks);
previousTotalLatency = currentTotalLatency;
}
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
if (recommendation.Action == ScaleAction.AddWorker)
{
simulatedWorkerCount++;
}
// The high-latency threshold is 1 second
Thread.Sleep(TimeSpan.FromSeconds(1.1));
}
}
public Task <PerformanceHeartbeat> PulseAsync(int currentWorkerCount)
{
return(_monitor.PulseAsync(currentWorkerCount));
}
