public async Task MonitorIdleTaskHubDisconnected() { var settings = new AzureStorageOrchestrationServiceSettings { StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), TaskHubName = nameof(MonitorIdleTaskHubDisconnected), PartitionCount = 4, }; var service = new AzureStorageOrchestrationService(settings); var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); await service.DeleteAsync(); // A null heartbeat is expected when the task hub does not exist. PerformanceHeartbeat heartbeat = await monitor.PulseAsync(currentWorkerCount : 0); Assert.IsNull(heartbeat); await service.CreateAsync(); ScaleRecommendation recommendation; for (int i = 0; i < 10; i++) { heartbeat = await monitor.PulseAsync(currentWorkerCount : 0); Assert.IsNotNull(heartbeat); Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLatencies.Count); Assert.AreEqual(0, heartbeat.ControlQueueLengths.Count(l => l != 0)); Assert.AreEqual(0, heartbeat.ControlQueueLatencies.Count(l => l != TimeSpan.Zero)); Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend); Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency); recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.AreEqual(ScaleAction.None, recommendation.Action); Assert.AreEqual(false, recommendation.KeepWorkersAlive); Assert.IsNotNull(recommendation.Reason); } // If any workers are assigned, the recommendation should be to have them removed. heartbeat = await monitor.PulseAsync(currentWorkerCount : 1); recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action); Assert.AreEqual(false, recommendation.KeepWorkersAlive); Assert.IsNotNull(recommendation.Reason); }
public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) { var service = new AzureStorageOrchestrationService(settings); service.CreateAsync().GetAwaiter().GetResult(); this.settings = settings; this.worker = new TaskHubWorker(service); this.client = new TaskHubClient(service); this.addedOrchestrationTypes = new HashSet <Type>(); this.addedActivityTypes = new HashSet <Type>(); }
public static TestOrchestrationHost GetTestOrchestrationHost( bool enableExtendedSessions, int extendedSessionTimeoutInSeconds = 30) { string storageConnectionString = GetTestStorageAccountConnectionString(); var service = new AzureStorageOrchestrationService( new AzureStorageOrchestrationServiceSettings { StorageConnectionString = storageConnectionString, TaskHubName = ConfigurationManager.AppSettings.Get("TaskHubName"), ExtendedSessionsEnabled = enableExtendedSessions, ExtendedSessionIdleTimeout = TimeSpan.FromSeconds(extendedSessionTimeoutInSeconds), }); service.CreateAsync().GetAwaiter().GetResult(); return(new TestOrchestrationHost(service)); }
public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings) { try { var service = new AzureStorageOrchestrationService(settings); service.CreateAsync().GetAwaiter().GetResult(); // I change Create to CreateIfNotExistsAsync for enabling execute without fail once per twice. this.settings = settings; this.worker = new TaskHubWorker(service); this.client = new TaskHubClient(service); this.addedOrchestrationTypes = new HashSet <Type>(); this.addedActivityTypes = new HashSet <Type>(); } catch (Exception e) { throw e; } }
async Task <AzureStorageOrchestrationService> EnsureTaskHubAsync( string testName, bool testDeletion, bool deleteBeforeCreate = true, string workerId = "test") { string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(); var storageAccount = CloudStorageAccount.Parse(storageConnectionString); string taskHubName = testName; var settings = new AzureStorageOrchestrationServiceSettings { TaskHubName = taskHubName, StorageConnectionString = storageConnectionString, WorkerId = workerId, }; Trace.TraceInformation($"Task Hub name: {taskHubName}"); var service = new AzureStorageOrchestrationService(settings); if (deleteBeforeCreate) { await service.CreateAsync(); } else { await service.CreateIfNotExistsAsync(); } // Control queues Assert.IsNotNull(service.AllControlQueues, "Control queue collection was not initialized."); ControlQueue[] controlQueues = service.AllControlQueues.ToArray(); Assert.AreEqual(4, controlQueues.Length, "Expected to see the default four control queues created."); foreach (ControlQueue queue in controlQueues) { Assert.IsTrue(await queue.InnerQueue.ExistsAsync(), $"Queue {queue.Name} was not created."); } // Work-item queue WorkItemQueue workItemQueue = service.WorkItemQueue; Assert.IsNotNull(workItemQueue, "Work-item queue client was not initialized."); Assert.IsTrue(await workItemQueue.InnerQueue.ExistsAsync(), $"Queue {workItemQueue.Name} was not created."); // TrackingStore ITrackingStore trackingStore = service.TrackingStore; Assert.IsNotNull(trackingStore, "Tracking Store was not initialized."); try { Assert.IsTrue(trackingStore.ExistsAsync().Result, $"Tracking Store was not created."); } catch (NotSupportedException) { } string expectedContainerName = taskHubName.ToLowerInvariant() + "-leases"; CloudBlobContainer taskHubContainer = storageAccount.CreateCloudBlobClient().GetContainerReference(expectedContainerName); Assert.IsTrue(await taskHubContainer.ExistsAsync(), $"Task hub blob container {expectedContainerName} was not created."); // Task Hub config blob CloudBlob infoBlob = taskHubContainer.GetBlobReference("taskhub.json"); Assert.IsTrue(await infoBlob.ExistsAsync(), $"The blob {infoBlob.Name} was not created."); // Task Hub lease container CloudBlobDirectory leaseDirectory = taskHubContainer.GetDirectoryReference("default"); IListBlobItem[] leaseBlobs = (await this.ListBlobsAsync(leaseDirectory)).ToArray(); Assert.AreEqual(controlQueues.Length, leaseBlobs.Length, "Expected to see the same number of control queues and lease blobs."); foreach (IListBlobItem blobItem in leaseBlobs) { string path = blobItem.Uri.AbsolutePath; Assert.IsTrue( controlQueues.Where(q => path.Contains(q.Name)).Any(), $"Could not find any known control queue name in the lease name {path}"); } if (testDeletion) { await service.DeleteAsync(); foreach (ControlQueue queue in controlQueues) { Assert.IsFalse(await queue.InnerQueue.ExistsAsync(), $"Queue {queue.Name} was not deleted."); } Assert.IsFalse(await workItemQueue.InnerQueue.ExistsAsync(), $"Queue {workItemQueue.Name} was not deleted."); try { Assert.IsFalse(trackingStore.ExistsAsync().Result, $"Tracking Store was not deleted."); } catch (NotSupportedException) { } Assert.IsFalse(await taskHubContainer.ExistsAsync(), $"Task hub blob container {taskHubContainer.Name} was not deleted."); } return(service); }
public async Task MonitorIncreasingControlQueueLoadDisconnected() { var settings = new AzureStorageOrchestrationServiceSettings() { StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected), PartitionCount = 4, }; var service = new AzureStorageOrchestrationService(settings); var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName); int simulatedWorkerCount = 0; await service.CreateAsync(); // A heartbeat should come back with no recommendation since there is no data. PerformanceHeartbeat heartbeat = await monitor.PulseAsync(simulatedWorkerCount); Assert.IsNotNull(heartbeat); Assert.IsNotNull(heartbeat.ScaleRecommendation); Assert.AreEqual(ScaleAction.None, heartbeat.ScaleRecommendation.Action); Assert.IsFalse(heartbeat.ScaleRecommendation.KeepWorkersAlive); var client = new TaskHubClient(service); var previousTotalLatency = TimeSpan.Zero; for (int i = 1; i < settings.PartitionCount + 10; i++) { await client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input : null); heartbeat = await monitor.PulseAsync(simulatedWorkerCount); Assert.IsNotNull(heartbeat); ScaleRecommendation recommendation = heartbeat.ScaleRecommendation; Assert.IsNotNull(recommendation); Assert.IsTrue(recommendation.KeepWorkersAlive); Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount); Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count); Assert.AreEqual(i, heartbeat.ControlQueueLengths.Sum()); Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency); TimeSpan currentTotalLatency = TimeSpan.FromTicks(heartbeat.ControlQueueLatencies.Sum(ts => ts.Ticks)); Assert.IsTrue(currentTotalLatency > previousTotalLatency); if (i + 1 < DisconnectedPerformanceMonitor.QueueLengthSampleSize) { int queuesWithNonZeroLatencies = heartbeat.ControlQueueLatencies.Count(t => t > TimeSpan.Zero); Assert.IsTrue(queuesWithNonZeroLatencies > 0 && queuesWithNonZeroLatencies <= i); int queuesWithAtLeastOneMessage = heartbeat.ControlQueueLengths.Count(l => l > 0); Assert.IsTrue(queuesWithAtLeastOneMessage > 0 && queuesWithAtLeastOneMessage <= i); ScaleAction expectedScaleAction = simulatedWorkerCount == 0 ? ScaleAction.AddWorker : ScaleAction.None; Assert.AreEqual(expectedScaleAction, recommendation.Action); } else { // Validate that control queue latencies are going up with each iteration. Assert.IsTrue(currentTotalLatency.Ticks > previousTotalLatency.Ticks); previousTotalLatency = currentTotalLatency; } Assert.AreEqual(0, heartbeat.WorkItemQueueLength); Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend); if (recommendation.Action == ScaleAction.AddWorker) { simulatedWorkerCount++; } // The high-latency threshold is 1 second Thread.Sleep(TimeSpan.FromSeconds(1.1)); } }
public async Task TestInstanceAndMessageDistribution() { const int InstanceCount = 50; // Create a service and enqueue N messages. // Make sure each partition has messages in it. var settings = new AzureStorageOrchestrationServiceSettings() { StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(), TaskHubName = nameof(TestInstanceAndMessageDistribution), PartitionCount = 4, }; var service = new AzureStorageOrchestrationService(settings); await service.CreateAsync(); var client = new TaskHubClient(service); Trace.TraceInformation($"Starting {InstanceCount} orchestrations..."); var createTasks = new Task <OrchestrationInstance> [InstanceCount]; for (int i = 0; i < InstanceCount; i++) { createTasks[i] = client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input: null); } OrchestrationInstance[] instances = await Task.WhenAll(createTasks); ControlQueue[] controlQueues = service.AllControlQueues.ToArray(); Assert.AreEqual(settings.PartitionCount, controlQueues.Length, "Unexpected number of control queues"); foreach (ControlQueue cloudQueue in controlQueues) { await cloudQueue.InnerQueue.FetchAttributesAsync(); int messageCount = cloudQueue.InnerQueue.ApproximateMessageCount.GetValueOrDefault(-1); Trace.TraceInformation($"Queue {cloudQueue.Name} has {messageCount} message(s)."); Assert.IsTrue(messageCount > 0, $"Queue {cloudQueue.Name} didn't receive any messages"); } Trace.TraceInformation("Success. All queue partitions have orchestration start messages."); // Start the service and let it process the previously enqueued messages. // Check that there are exactly N unique partition keys in the table Trace.TraceInformation("Starting the worker to consume the messages and run the orchestrations..."); var worker = new TaskHubWorker(service); worker.AddTaskOrchestrations(typeof(NoOpOrchestration)); await worker.StartAsync(); try { // Wait for the instances to run and complete OrchestrationState[] states = await Task.WhenAll( instances.Select(i => client.WaitForOrchestrationAsync(i, TimeSpan.FromSeconds(30)))); Assert.IsTrue( Array.TrueForAll(states, s => s?.OrchestrationStatus == OrchestrationStatus.Completed), "Not all orchestrations completed successfully!"); var tableTrackingStore = service.TrackingStore as AzureTableTrackingStore; if (tableTrackingStore != null) { DynamicTableEntity[] entities = (await tableTrackingStore.HistoryTable.ExecuteQuerySegmentedAsync(new TableQuery(), new TableContinuationToken())).ToArray(); int uniquePartitions = entities.GroupBy(e => e.PartitionKey).Count(); Trace.TraceInformation($"Found {uniquePartitions} unique partition(s) in table storage."); Assert.AreEqual(InstanceCount, uniquePartitions, "Unexpected number of table partitions."); } } finally { await worker.StopAsync(isForced : true); } }