public async Task MonitorIdleTaskHubDisconnected() { var settings = new AzureStorageOrchestrationServiceSettings { StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), TaskHubName = nameof(MonitorIdleTaskHubDisconnected), PartitionCount = 4, }; var service = new AzureStorageOrchestrationService(settings); var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); await service.DeleteAsync(); // A null heartbeat is expected when the task hub does not exist. PerformanceHeartbeat heartbeat = await monitor.PulseAsync(currentWorkerCount : 0); Assert.IsNull(heartbeat); await service.CreateAsync(); ScaleRecommendation recommendation; for (int i = 0; i < 10; i++) { heartbeat = await monitor.PulseAsync(currentWorkerCount : 0); Assert.IsNotNull(heartbeat); Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLatencies.Count); Assert.AreEqual(0, heartbeat.ControlQueueLengths.Count(l => l != 0)); Assert.AreEqual(0, heartbeat.ControlQueueLatencies.Count(l => l != TimeSpan.Zero)); Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend); Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency); recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.AreEqual(ScaleAction.None, recommendation.Action); Assert.AreEqual(false, recommendation.KeepWorkersAlive); Assert.IsNotNull(recommendation.Reason); } // If any workers are assigned, the recommendation should be to have them removed. heartbeat = await monitor.PulseAsync(currentWorkerCount : 1); recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action); Assert.AreEqual(false, recommendation.KeepWorkersAlive); Assert.IsNotNull(recommendation.Reason); }
public async Task <DurableTaskTriggerMetrics> GetMetricsAsync() { DurableTaskTriggerMetrics metrics = new DurableTaskTriggerMetrics(); // Durable stores its own metrics, so we just collect them here PerformanceHeartbeat heartbeat = null; try { DisconnectedPerformanceMonitor performanceMonitor = this.GetPerformanceMonitor(); heartbeat = await performanceMonitor.PulseAsync(); } catch (StorageException e) { this.traceHelper.ExtensionWarningEvent(this.hubName, this.functionName.Name, string.Empty, e.ToString()); } if (heartbeat != null) { metrics.PartitionCount = heartbeat.PartitionCount; metrics.ControlQueueLengths = JsonConvert.SerializeObject(heartbeat.ControlQueueLengths); metrics.ControlQueueLatencies = JsonConvert.SerializeObject(heartbeat.ControlQueueLatencies); metrics.WorkItemQueueLength = heartbeat.WorkItemQueueLength; if (heartbeat.WorkItemQueueLatency != null) { metrics.WorkItemQueueLatency = heartbeat.WorkItemQueueLatency.ToString(); } } return(metrics); }
public async Task MonitorIncreasingControlQueueLoadDisconnected() { var settings = new AzureStorageOrchestrationServiceSettings() { StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected), PartitionCount = 4, }; var service = new AzureStorageOrchestrationService(settings); var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); int simulatedWorkerCount = 0; await service.CreateAsync(); // A heartbeat should come back with no recommendation since there is no data. PerformanceHeartbeat heartbeat = await monitor.PulseAsync(simulatedWorkerCount); Assert.IsNotNull(heartbeat); Assert.IsNotNull(heartbeat.ScaleRecommendation); Assert.AreEqual(ScaleAction.None, heartbeat.ScaleRecommendation.Action); Assert.IsFalse(heartbeat.ScaleRecommendation.KeepWorkersAlive); var client = new TaskHubClient(service); var previousTotalLatency = TimeSpan.Zero; for (int i = 1; i < settings.PartitionCount + 10; i++) { await client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input : null); heartbeat = await monitor.PulseAsync(simulatedWorkerCount); Assert.IsNotNull(heartbeat); ScaleRecommendation recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.IsTrue(recommendation.KeepWorkersAlive); Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count); Assert.AreEqual(i, heartbeat.ControlQueueLengths.Sum()); Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency); TimeSpan currentTotalLatency = TimeSpan.FromTicks(heartbeat.ControlQueueLatencies.Sum(ts => ts.Ticks)); Assert.IsTrue(currentTotalLatency > previousTotalLatency); if (i + 1 < DisconnectedPerformanceMonitor.QueueLengthSampleSize) { int queuesWithNonZeroLatencies = heartbeat.ControlQueueLatencies.Count(t => t > TimeSpan.Zero); Assert.IsTrue(queuesWithNonZeroLatencies > 0 && queuesWithNonZeroLatencies <= i); int queuesWithAtLeastOneMessage = heartbeat.ControlQueueLengths.Count(l => l > 0); Assert.IsTrue(queuesWithAtLeastOneMessage > 0 && queuesWithAtLeastOneMessage <= i); ScaleAction expectedScaleAction = simulatedWorkerCount == 0 ? ScaleAction.AddWorker : ScaleAction.None; Assert.AreEqual(expectedScaleAction, recommendation.Action); } else { // Validate that control queue latencies are going up with each iteration. Assert.IsTrue(currentTotalLatency.Ticks > previousTotalLatency.Ticks); previousTotalLatency = currentTotalLatency; } Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend); if (recommendation.Action == ScaleAction.AddWorker) { simulatedWorkerCount++; } // The high-latency threshold is 1 second Thread.Sleep(TimeSpan.FromSeconds(1.1)); } }
public Task <PerformanceHeartbeat> PulseAsync(int currentWorkerCount) { return(_monitor.PulseAsync(currentWorkerCount)); }