public async Task ScaleDecision_ControlQueueLatency_Idle2()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 1, 1, 1, 1 });
mock.AddLatencies(0, new[] { 0, 0, 1, 1 });
mock.AddLatencies(0, new[] { 0, 0, 1, 1 });
mock.AddLatencies(0, new[] { 0, 0, 1, 1 });
mock.AddLatencies(0, new[] { 0, 0, 1, 1 });
mock.AddLatencies(0, new[] { 0, 0, 1, 1 });
for (int simulatedWorkerCount = 1; simulatedWorkerCount < 10; simulatedWorkerCount++)
{
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsTrue(recommendation.KeepWorkersAlive);
if (simulatedWorkerCount > 2)
{
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
}
else
{
Assert.AreEqual(ScaleAction.None, recommendation.Action);
}
}
}
public async Task ScaleDecision_ControlQueueLatency_High4()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 600, 600, 600, 600 });
mock.AddLatencies(0, new[] { 700, 700, 700, 700 });
mock.AddLatencies(0, new[] { 800, 800, 800, 800 });
mock.AddLatencies(0, new[] { 900, 900, 900, 900 });
mock.AddLatencies(0, new[] { 1000, 1000, 1000, 1000 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 3);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.AddWorker, recommendation.Action, "Four hot partitions");
Assert.IsTrue(recommendation.KeepWorkersAlive);
heartbeat = await mock.PulseAsync(simulatedWorkerCount : 4);
recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action, "Only four hot partitions");
Assert.IsTrue(recommendation.KeepWorkersAlive);
heartbeat = await mock.PulseAsync(simulatedWorkerCount : 5);
recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action, "No work items and only four hot partitions");
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task <DurableTaskTriggerMetrics> GetMetricsAsync()
{
DurableTaskTriggerMetrics metrics = new DurableTaskTriggerMetrics();
// Durable stores its own metrics, so we just collect them here
PerformanceHeartbeat heartbeat = null;
try
{
DisconnectedPerformanceMonitor performanceMonitor = this.GetPerformanceMonitor();
heartbeat = await performanceMonitor.PulseAsync();
}
catch (StorageException e)
{
this.traceHelper.ExtensionWarningEvent(this.hubName, this.functionName.Name, string.Empty, e.ToString());
}
if (heartbeat != null)
{
metrics.PartitionCount = heartbeat.PartitionCount;
metrics.ControlQueueLengths = JsonConvert.SerializeObject(heartbeat.ControlQueueLengths);
metrics.ControlQueueLatencies = JsonConvert.SerializeObject(heartbeat.ControlQueueLatencies);
metrics.WorkItemQueueLength = heartbeat.WorkItemQueueLength;
if (heartbeat.WorkItemQueueLatency != null)
{
metrics.WorkItemQueueLatency = heartbeat.WorkItemQueueLatency.ToString();
}
}
return(metrics);
}
public async Task ScaleDecision_ControlQueueLatency_NotMaxPollingDelay()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 10, 10, 10 });
mock.AddLatencies(0, new[] { 9999, 9999, 9999, 9999 });
// Queue was not idle, so we consider high threshold but not max polling latency
for (int simulatedWorkerCount = 1; simulatedWorkerCount < 10; simulatedWorkerCount++)
{
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsTrue(recommendation.KeepWorkersAlive);
if (simulatedWorkerCount < 3)
{
Assert.AreEqual(ScaleAction.AddWorker, recommendation.Action);
}
else if (simulatedWorkerCount <= 4)
{
Assert.AreEqual(ScaleAction.None, recommendation.Action);
}
else
{
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
}
}
}
public async Task ScaleDecision_ControlQueueLatency_QuickDrain()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 30000, 30000, 30000, 30000 });
mock.AddLatencies(0, new[] { 30000, 30000, 30000, 30000 });
mock.AddLatencies(0, new[] { 30000, 30000, 30000, 30000 });
mock.AddLatencies(0, new[] { 30000, 30000, 30000, 30000 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
// Something happened and we immediately drained the work-item queue
for (int simulatedWorkerCount = 1; simulatedWorkerCount < 10; simulatedWorkerCount++)
{
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsTrue(recommendation.KeepWorkersAlive);
if (simulatedWorkerCount > 4)
{
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
}
else
{
Assert.AreEqual(ScaleAction.None, recommendation.Action);
}
}
}
private static void SetScaleRecommendation(PerformanceHeartbeat performanceHeartbeat,
ScaleRecommendation scaleRecommendation)
{
var t = typeof(PerformanceHeartbeat);
var prop = t.GetProperty("ScaleRecommendation");
prop.SetValue(performanceHeartbeat, scaleRecommendation);
}
public void GetScaleStatus_HandlesMalformedMetrics()
{
// Null metrics
var context = new ScaleStatusContext <DurableTaskTriggerMetrics>
{
WorkerCount = 1,
Metrics = null,
};
var recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
// Empty metrics
var heartbeats = new PerformanceHeartbeat[0];
context.Metrics = new DurableTaskTriggerMetrics[0];
this.performanceMonitor
.Setup(m => m.MakeScaleRecommendation(1, heartbeats))
.Returns <ScaleRecommendation>(null);
recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
// Metrics with null properties
var metrics = new DurableTaskTriggerMetrics[5];
for (int i = 0; i < metrics.Length; ++i)
{
metrics[i] = new DurableTaskTriggerMetrics();
}
context.Metrics = metrics;
heartbeats = new PerformanceHeartbeat[5];
for (int i = 0; i < heartbeats.Length; ++i)
{
heartbeats[i] = new PerformanceHeartbeat
{
ControlQueueLengths = new List <int>(),
ControlQueueLatencies = new List <TimeSpan>(),
};
}
this.performanceMonitor
.Setup(m => m.MakeScaleRecommendation(1, this.MatchEquivalentHeartbeats(heartbeats)))
.Returns <ScaleRecommendation>(null);
recommendation = this.scaleMonitor.GetScaleStatus(context);
Assert.Equal(ScaleVote.None, recommendation.Vote);
}
public async Task MonitorIdleTaskHubDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIdleTaskHubDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
await service.DeleteAsync();
// A null heartbeat is expected when the task hub does not exist.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNull(heartbeat);
await service.CreateAsync();
ScaleRecommendation recommendation;
for (int i = 0; i < 10; i++)
{
heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNotNull(heartbeat);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLatencies.Count);
Assert.AreEqual(0, heartbeat.ControlQueueLengths.Count(l => l != 0));
Assert.AreEqual(0, heartbeat.ControlQueueLatencies.Count(l => l != TimeSpan.Zero));
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
// If any workers are assigned, the recommendation should be to have them removed.
heartbeat = await monitor.PulseAsync(currentWorkerCount : 1);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
public ServiceFixture(ScaleAction action, int currentWorkerCount, bool inputKeepWorkersAlive = true)
{
_kubernetesRepositoryMock = new Mock <IKubernetesRepository>();
_performanceMonitorRepositoryMock = new Mock <IPerformanceMonitorRepository>();
var performanceHeartbeat = new PerformanceHeartbeat();
var scaleRecommendation = CreateScaleRecommendation(action, inputKeepWorkersAlive, "baz");
SetScaleRecommendation(performanceHeartbeat, scaleRecommendation);
_performanceMonitorRepositoryMock.Setup(p => p.PulseAsync(currentWorkerCount)).ReturnsAsync(performanceHeartbeat);
_kubernetesRepositoryMock.Setup(p => p.GetNumberOfPodAsync(ExpectedName, ExpectedNamespace)).ReturnsAsync(currentWorkerCount);
_loggerMock = new Mock <ILogger <ExternalScalerService> >();
}
public override async Task <PerformanceHeartbeat> PulseAsync(int simulatedWorkerCount)
{
Trace.TraceInformation(
"PULSE INPUT: Worker count: {0}; work items: {1}; control items: {2} {3}.",
simulatedWorkerCount,
this.WorkItemQueueLatencies,
Environment.NewLine,
string.Join(Environment.NewLine + " ", this.ControlQueueLatencies));
PerformanceHeartbeat heartbeat = await base.PulseAsync(simulatedWorkerCount);
Trace.TraceInformation($"PULSE OUTPUT: {heartbeat}");
return(heartbeat);
}
private void GetCorrespondingHeartbeatsAndMetrics(out PerformanceHeartbeat[] heartbeats, out DurableTaskTriggerMetrics[] metrics)
{
heartbeats = new PerformanceHeartbeat[]
{
new PerformanceHeartbeat
{
PartitionCount = 4,
ControlQueueLengths = new List <int> {
1, 2, 3, 4
},
ControlQueueLatencies = new List <TimeSpan> {
TimeSpan.FromMilliseconds(1), TimeSpan.FromMilliseconds(2), TimeSpan.FromMilliseconds(3), TimeSpan.FromMilliseconds(4),
},
WorkItemQueueLength = 5,
WorkItemQueueLatency = TimeSpan.FromMilliseconds(6),
},
new PerformanceHeartbeat
{
PartitionCount = 7,
ControlQueueLengths = new List <int> {
8, 9, 10, 11
},
ControlQueueLatencies = new List <TimeSpan> {
TimeSpan.FromMilliseconds(12), TimeSpan.FromMilliseconds(13), TimeSpan.FromMilliseconds(14), TimeSpan.FromMilliseconds(15),
},
WorkItemQueueLength = 16,
WorkItemQueueLatency = TimeSpan.FromMilliseconds(17),
},
};
metrics = new DurableTaskTriggerMetrics[]
{
new DurableTaskTriggerMetrics
{
PartitionCount = 4,
ControlQueueLengths = "[1,2,3,4]",
ControlQueueLatencies = "[\"00:00:00.0010000\",\"00:00:00.0020000\",\"00:00:00.0030000\",\"00:00:00.0040000\"]",
WorkItemQueueLength = 5,
WorkItemQueueLatency = "00:00:00.0060000",
},
new DurableTaskTriggerMetrics
{
PartitionCount = 7,
ControlQueueLengths = "[8,9,10,11]",
ControlQueueLatencies = "[\"00:00:00.0120000\",\"00:00:00.0130000\",\"00:00:00.0140000\",\"00:00:00.0150000\"]",
WorkItemQueueLength = 16,
WorkItemQueueLatency = "00:00:00.0170000",
},
};
}
public async Task ScaleDecision_WorkItemLatency_Low()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
var random = new Random();
// Scale down for low latency is semi-random, so need to take a lot of samples
var recommendations = new ScaleRecommendation[500];
for (int i = 0; i < recommendations.Length; i++)
{
mock.AddLatencies(random.Next(50), new[] { 0, 0, 0, 0 });
heartbeat = await mock.PulseAsync(simulatedWorkerCount : 2);
recommendations[i] = heartbeat.ScaleRecommendation;
}
int scaleOutCount = recommendations.Count(r => r.Action == ScaleAction.AddWorker);
int scaleInCount = recommendations.Count(r => r.Action == ScaleAction.RemoveWorker);
int noScaleCount = recommendations.Count(r => r.Action == ScaleAction.None);
int keepAliveCount = recommendations.Count(r => r.KeepWorkersAlive);
Trace.TraceInformation($"Scale-out count : {scaleOutCount}.");
Trace.TraceInformation($"Scale-in count : {scaleInCount}.");
Trace.TraceInformation($"No-scale count : {noScaleCount}.");
Trace.TraceInformation($"Keep-alive count : {keepAliveCount}.");
// It is expected that we scale-in only a small percentage of the time and never scale-out.
Assert.AreEqual(0, scaleOutCount);
Assert.AreNotEqual(0, scaleInCount);
Assert.IsTrue(noScaleCount > scaleInCount, "Should have more no-scale decisions");
Assert.IsTrue(keepAliveCount > recommendations.Length * 0.9, "Almost all should be keep-alive");
}
public async Task ScaleDecision_ControlQueueLatency_MaxPollingDelay2()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 10000, 10000, 10000, 10000 });
mock.AddLatencies(0, new[] { 100, 100, 100, 100 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_WorkItemLatency_Moderate()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(500, new[] { 0, 0, 0, 0 });
mock.AddLatencies(600, new[] { 0, 0, 0, 0 });
mock.AddLatencies(700, new[] { 0, 0, 0, 0 });
mock.AddLatencies(800, new[] { 0, 0, 0, 0 });
mock.AddLatencies(900, new[] { 0, 0, 0, 0 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_ControlQueueLatency_High1()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 600 });
mock.AddLatencies(0, new[] { 0, 0, 0, 700 });
mock.AddLatencies(0, new[] { 0, 0, 0, 800 });
mock.AddLatencies(0, new[] { 0, 0, 0, 900 });
mock.AddLatencies(0, new[] { 0, 0, 0, 1000 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action, "Only one hot partition");
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_ControlQueueLatency_MaxPollingDelay1()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 9999, 9999, 9999, 9999 });
// When queue is idle, first non-zero latency must be > max polling interval
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_ControlQueueLatency_Idle4()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 1 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
Assert.IsFalse(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_WorkItemLatency_NotMaxPollingDelay()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(10, new[] { 0, 0, 0, 0 });
mock.AddLatencies(9999, new[] { 0, 0, 0, 0 });
// Queue was not idle, so we consider high threshold but not max polling latency
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.AddWorker, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_WorkItemLatency_QuickDrain()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(30000, new[] { 0, 0, 0, 0 });
mock.AddLatencies(30000, new[] { 0, 0, 0, 0 });
mock.AddLatencies(30000, new[] { 0, 0, 0, 0 });
mock.AddLatencies(30000, new[] { 0, 0, 0, 0 });
mock.AddLatencies(3, new[] { 0, 0, 0, 0 });
// Something happened and we immediately drained the work-item queue
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
public async Task ScaleDecision_ControlQueueLatency_NotIdle()
{
var mock = GetFakePerformanceMonitor();
mock.AddLatencies(0, new[] { 0, 0, 0, 1 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
for (int i = 0; i < 100; i++)
{
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
// We should never scale to zero unless all control queues are idle.
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
}
public async Task ScaleDecision_WorkItemLatency_NotIdle()
{
var mock = GetFakePerformanceMonitor();
for (int i = 0; i < 100; i++)
{
mock.AddLatencies(1, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
mock.AddLatencies(0, new[] { 0, 0, 0, 0 });
PerformanceHeartbeat heartbeat = await mock.PulseAsync(simulatedWorkerCount : 1);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
// Should never scale to zero when there was a message in a queue
// within the last 5 samples.
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.IsTrue(recommendation.KeepWorkersAlive);
}
}
private ScaleStatus GetScaleStatusCore(int workerCount, DurableTaskTriggerMetrics[] metrics)
{
var scaleStatus = new ScaleStatus()
{
Vote = ScaleVote.None
};
if (metrics == null)
{
return(scaleStatus);
}
var heartbeats = new PerformanceHeartbeat[metrics.Length];
for (int i = 0; i < metrics.Length; ++i)
{
TimeSpan workItemQueueLatency;
bool parseResult = TimeSpan.TryParse(metrics[i].WorkItemQueueLatency, out workItemQueueLatency);
heartbeats[i] = new PerformanceHeartbeat()
{
PartitionCount = metrics[i].PartitionCount,
WorkItemQueueLatency = parseResult ? workItemQueueLatency : TimeSpan.FromMilliseconds(0),
WorkItemQueueLength = metrics[i].WorkItemQueueLength,
};
if (metrics[i].ControlQueueLengths == null)
{
heartbeats[i].ControlQueueLengths = new List <int>();
}
else
{
heartbeats[i].ControlQueueLengths = JsonConvert.DeserializeObject <IReadOnlyList <int> >(metrics[i].ControlQueueLengths);
}
if (metrics[i].ControlQueueLatencies == null)
{
heartbeats[i].ControlQueueLatencies = new List <TimeSpan>();
}
else
{
heartbeats[i].ControlQueueLatencies = JsonConvert.DeserializeObject <IReadOnlyList <TimeSpan> >(metrics[i].ControlQueueLatencies);
}
}
DisconnectedPerformanceMonitor performanceMonitor = this.GetPerformanceMonitor();
var scaleRecommendation = performanceMonitor.MakeScaleRecommendation(workerCount, heartbeats.ToArray());
bool writeToUserLogs = false;
switch (scaleRecommendation?.Action)
{
case ScaleAction.AddWorker:
scaleStatus.Vote = ScaleVote.ScaleOut;
writeToUserLogs = true;
break;
case ScaleAction.RemoveWorker:
scaleStatus.Vote = ScaleVote.ScaleIn;
writeToUserLogs = true;
break;
default:
scaleStatus.Vote = ScaleVote.None;
break;
}
this.traceHelper.ExtensionInformationalEvent(
this.hubName,
string.Empty,
this.functionName.Name,
$"Durable Functions Trigger Scale Decision: {scaleStatus.Vote.ToString()}, Reason: {scaleRecommendation?.Reason}",
writeToUserLogs: writeToUserLogs);
return(scaleStatus);
}
public async Task MonitorIncreasingControlQueueLoadDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings()
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
int simulatedWorkerCount = 0;
await service.CreateAsync();
// A heartbeat should come back with no recommendation since there is no data.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
Assert.IsNotNull(heartbeat.ScaleRecommendation);
Assert.AreEqual(ScaleAction.None, heartbeat.ScaleRecommendation.Action);
Assert.IsFalse(heartbeat.ScaleRecommendation.KeepWorkersAlive);
var client = new TaskHubClient(service);
var previousTotalLatency = TimeSpan.Zero;
for (int i = 1; i < settings.PartitionCount + 10; i++)
{
await client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input : null);
heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.IsTrue(recommendation.KeepWorkersAlive);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(i, heartbeat.ControlQueueLengths.Sum());
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
TimeSpan currentTotalLatency = TimeSpan.FromTicks(heartbeat.ControlQueueLatencies.Sum(ts => ts.Ticks));
Assert.IsTrue(currentTotalLatency > previousTotalLatency);
if (i + 1 < DisconnectedPerformanceMonitor.QueueLengthSampleSize)
{
int queuesWithNonZeroLatencies = heartbeat.ControlQueueLatencies.Count(t => t > TimeSpan.Zero);
Assert.IsTrue(queuesWithNonZeroLatencies > 0 && queuesWithNonZeroLatencies <= i);
int queuesWithAtLeastOneMessage = heartbeat.ControlQueueLengths.Count(l => l > 0);
Assert.IsTrue(queuesWithAtLeastOneMessage > 0 && queuesWithAtLeastOneMessage <= i);
ScaleAction expectedScaleAction = simulatedWorkerCount == 0 ? ScaleAction.AddWorker : ScaleAction.None;
Assert.AreEqual(expectedScaleAction, recommendation.Action);
}
else
{
// Validate that control queue latencies are going up with each iteration.
Assert.IsTrue(currentTotalLatency.Ticks > previousTotalLatency.Ticks);
previousTotalLatency = currentTotalLatency;
}
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
if (recommendation.Action == ScaleAction.AddWorker)
{
simulatedWorkerCount++;
}
// The high-latency threshold is 1 second
Thread.Sleep(TimeSpan.FromSeconds(1.1));
}
}
