public async Task MonitorIdleTaskHubDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIdleTaskHubDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
await service.DeleteAsync();
// A null heartbeat is expected when the task hub does not exist.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNull(heartbeat);
await service.CreateAsync();
ScaleRecommendation recommendation;
for (int i = 0; i < 10; i++)
{
heartbeat = await monitor.PulseAsync(currentWorkerCount : 0);
Assert.IsNotNull(heartbeat);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLatencies.Count);
Assert.AreEqual(0, heartbeat.ControlQueueLengths.Count(l => l != 0));
Assert.AreEqual(0, heartbeat.ControlQueueLatencies.Count(l => l != TimeSpan.Zero));
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.None, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
// If any workers are assigned, the recommendation should be to have them removed.
heartbeat = await monitor.PulseAsync(currentWorkerCount : 1);
recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.AreEqual(ScaleAction.RemoveWorker, recommendation.Action);
Assert.AreEqual(false, recommendation.KeepWorkersAlive);
Assert.IsNotNull(recommendation.Reason);
}
public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings)
{
var service = new AzureStorageOrchestrationService(settings);
service.CreateAsync().GetAwaiter().GetResult();
this.settings = settings;
this.worker = new TaskHubWorker(service);
this.client = new TaskHubClient(service);
this.addedOrchestrationTypes = new HashSet <Type>();
this.addedActivityTypes = new HashSet <Type>();
}
public static TestOrchestrationHost GetTestOrchestrationHost(
bool enableExtendedSessions,
int extendedSessionTimeoutInSeconds = 30)
{
string storageConnectionString = GetTestStorageAccountConnectionString();
var service = new AzureStorageOrchestrationService(
new AzureStorageOrchestrationServiceSettings
{
StorageConnectionString = storageConnectionString,
TaskHubName = ConfigurationManager.AppSettings.Get("TaskHubName"),
ExtendedSessionsEnabled = enableExtendedSessions,
ExtendedSessionIdleTimeout = TimeSpan.FromSeconds(extendedSessionTimeoutInSeconds),
});
service.CreateAsync().GetAwaiter().GetResult();
return(new TestOrchestrationHost(service));
}
public TestOrchestrationHost(AzureStorageOrchestrationServiceSettings settings)
{
try
{
var service = new AzureStorageOrchestrationService(settings);
service.CreateAsync().GetAwaiter().GetResult(); // I change Create to CreateIfNotExistsAsync for enabling execute without fail once per twice.
this.settings = settings;
this.worker = new TaskHubWorker(service);
this.client = new TaskHubClient(service);
this.addedOrchestrationTypes = new HashSet <Type>();
this.addedActivityTypes = new HashSet <Type>();
}
catch (Exception e)
{
throw e;
}
}
async Task <AzureStorageOrchestrationService> EnsureTaskHubAsync(
string testName,
bool testDeletion,
bool deleteBeforeCreate = true,
string workerId = "test")
{
string storageConnectionString = TestHelpers.GetTestStorageAccountConnectionString();
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
string taskHubName = testName;
var settings = new AzureStorageOrchestrationServiceSettings
{
TaskHubName = taskHubName,
StorageConnectionString = storageConnectionString,
WorkerId = workerId,
};
Trace.TraceInformation($"Task Hub name: {taskHubName}");
var service = new AzureStorageOrchestrationService(settings);
if (deleteBeforeCreate)
{
await service.CreateAsync();
}
else
{
await service.CreateIfNotExistsAsync();
}
// Control queues
Assert.IsNotNull(service.AllControlQueues, "Control queue collection was not initialized.");
ControlQueue[] controlQueues = service.AllControlQueues.ToArray();
Assert.AreEqual(4, controlQueues.Length, "Expected to see the default four control queues created.");
foreach (ControlQueue queue in controlQueues)
{
Assert.IsTrue(await queue.InnerQueue.ExistsAsync(), $"Queue {queue.Name} was not created.");
}
// Work-item queue
WorkItemQueue workItemQueue = service.WorkItemQueue;
Assert.IsNotNull(workItemQueue, "Work-item queue client was not initialized.");
Assert.IsTrue(await workItemQueue.InnerQueue.ExistsAsync(), $"Queue {workItemQueue.Name} was not created.");
// TrackingStore
ITrackingStore trackingStore = service.TrackingStore;
Assert.IsNotNull(trackingStore, "Tracking Store was not initialized.");
try
{
Assert.IsTrue(trackingStore.ExistsAsync().Result, $"Tracking Store was not created.");
}
catch (NotSupportedException)
{ }
string expectedContainerName = taskHubName.ToLowerInvariant() + "-leases";
CloudBlobContainer taskHubContainer = storageAccount.CreateCloudBlobClient().GetContainerReference(expectedContainerName);
Assert.IsTrue(await taskHubContainer.ExistsAsync(), $"Task hub blob container {expectedContainerName} was not created.");
// Task Hub config blob
CloudBlob infoBlob = taskHubContainer.GetBlobReference("taskhub.json");
Assert.IsTrue(await infoBlob.ExistsAsync(), $"The blob {infoBlob.Name} was not created.");
// Task Hub lease container
CloudBlobDirectory leaseDirectory = taskHubContainer.GetDirectoryReference("default");
IListBlobItem[] leaseBlobs = (await this.ListBlobsAsync(leaseDirectory)).ToArray();
Assert.AreEqual(controlQueues.Length, leaseBlobs.Length, "Expected to see the same number of control queues and lease blobs.");
foreach (IListBlobItem blobItem in leaseBlobs)
{
string path = blobItem.Uri.AbsolutePath;
Assert.IsTrue(
controlQueues.Where(q => path.Contains(q.Name)).Any(),
$"Could not find any known control queue name in the lease name {path}");
}
if (testDeletion)
{
await service.DeleteAsync();
foreach (ControlQueue queue in controlQueues)
{
Assert.IsFalse(await queue.InnerQueue.ExistsAsync(), $"Queue {queue.Name} was not deleted.");
}
Assert.IsFalse(await workItemQueue.InnerQueue.ExistsAsync(), $"Queue {workItemQueue.Name} was not deleted.");
try
{
Assert.IsFalse(trackingStore.ExistsAsync().Result, $"Tracking Store was not deleted.");
}
catch (NotSupportedException)
{ }
Assert.IsFalse(await taskHubContainer.ExistsAsync(), $"Task hub blob container {taskHubContainer.Name} was not deleted.");
}
return(service);
}
public async Task MonitorIncreasingControlQueueLoadDisconnected()
{
var settings = new AzureStorageOrchestrationServiceSettings()
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(MonitorIncreasingControlQueueLoadDisconnected),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
var monitor = new DisconnectedPerformanceMonitor(settings.StorageConnectionString, settings.TaskHubName);
int simulatedWorkerCount = 0;
await service.CreateAsync();
// A heartbeat should come back with no recommendation since there is no data.
PerformanceHeartbeat heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
Assert.IsNotNull(heartbeat.ScaleRecommendation);
Assert.AreEqual(ScaleAction.None, heartbeat.ScaleRecommendation.Action);
Assert.IsFalse(heartbeat.ScaleRecommendation.KeepWorkersAlive);
var client = new TaskHubClient(service);
var previousTotalLatency = TimeSpan.Zero;
for (int i = 1; i < settings.PartitionCount + 10; i++)
{
await client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input : null);
heartbeat = await monitor.PulseAsync(simulatedWorkerCount);
Assert.IsNotNull(heartbeat);
ScaleRecommendation recommendation = heartbeat.ScaleRecommendation;
Assert.IsNotNull(recommendation);
Assert.IsTrue(recommendation.KeepWorkersAlive);
Assert.AreEqual(settings.PartitionCount, heartbeat.PartitionCount);
Assert.AreEqual(settings.PartitionCount, heartbeat.ControlQueueLengths.Count);
Assert.AreEqual(i, heartbeat.ControlQueueLengths.Sum());
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(TimeSpan.Zero, heartbeat.WorkItemQueueLatency);
TimeSpan currentTotalLatency = TimeSpan.FromTicks(heartbeat.ControlQueueLatencies.Sum(ts => ts.Ticks));
Assert.IsTrue(currentTotalLatency > previousTotalLatency);
if (i + 1 < DisconnectedPerformanceMonitor.QueueLengthSampleSize)
{
int queuesWithNonZeroLatencies = heartbeat.ControlQueueLatencies.Count(t => t > TimeSpan.Zero);
Assert.IsTrue(queuesWithNonZeroLatencies > 0 && queuesWithNonZeroLatencies <= i);
int queuesWithAtLeastOneMessage = heartbeat.ControlQueueLengths.Count(l => l > 0);
Assert.IsTrue(queuesWithAtLeastOneMessage > 0 && queuesWithAtLeastOneMessage <= i);
ScaleAction expectedScaleAction = simulatedWorkerCount == 0 ? ScaleAction.AddWorker : ScaleAction.None;
Assert.AreEqual(expectedScaleAction, recommendation.Action);
}
else
{
// Validate that control queue latencies are going up with each iteration.
Assert.IsTrue(currentTotalLatency.Ticks > previousTotalLatency.Ticks);
previousTotalLatency = currentTotalLatency;
}
Assert.AreEqual(0, heartbeat.WorkItemQueueLength);
Assert.AreEqual(0.0, heartbeat.WorkItemQueueLatencyTrend);
if (recommendation.Action == ScaleAction.AddWorker)
{
simulatedWorkerCount++;
}
// The high-latency threshold is 1 second
Thread.Sleep(TimeSpan.FromSeconds(1.1));
}
}
public async Task TestInstanceAndMessageDistribution()
{
const int InstanceCount = 50;
// Create a service and enqueue N messages.
// Make sure each partition has messages in it.
var settings = new AzureStorageOrchestrationServiceSettings()
{
StorageConnectionString = TestHelpers.GetTestStorageAccountConnectionString(),
TaskHubName = nameof(TestInstanceAndMessageDistribution),
PartitionCount = 4,
};
var service = new AzureStorageOrchestrationService(settings);
await service.CreateAsync();
var client = new TaskHubClient(service);
Trace.TraceInformation($"Starting {InstanceCount} orchestrations...");
var createTasks = new Task <OrchestrationInstance> [InstanceCount];
for (int i = 0; i < InstanceCount; i++)
{
createTasks[i] = client.CreateOrchestrationInstanceAsync(typeof(NoOpOrchestration), input: null);
}
OrchestrationInstance[] instances = await Task.WhenAll(createTasks);
ControlQueue[] controlQueues = service.AllControlQueues.ToArray();
Assert.AreEqual(settings.PartitionCount, controlQueues.Length, "Unexpected number of control queues");
foreach (ControlQueue cloudQueue in controlQueues)
{
await cloudQueue.InnerQueue.FetchAttributesAsync();
int messageCount = cloudQueue.InnerQueue.ApproximateMessageCount.GetValueOrDefault(-1);
Trace.TraceInformation($"Queue {cloudQueue.Name} has {messageCount} message(s).");
Assert.IsTrue(messageCount > 0, $"Queue {cloudQueue.Name} didn't receive any messages");
}
Trace.TraceInformation("Success. All queue partitions have orchestration start messages.");
// Start the service and let it process the previously enqueued messages.
// Check that there are exactly N unique partition keys in the table
Trace.TraceInformation("Starting the worker to consume the messages and run the orchestrations...");
var worker = new TaskHubWorker(service);
worker.AddTaskOrchestrations(typeof(NoOpOrchestration));
await worker.StartAsync();
try
{
// Wait for the instances to run and complete
OrchestrationState[] states = await Task.WhenAll(
instances.Select(i => client.WaitForOrchestrationAsync(i, TimeSpan.FromSeconds(30))));
Assert.IsTrue(
Array.TrueForAll(states, s => s?.OrchestrationStatus == OrchestrationStatus.Completed),
"Not all orchestrations completed successfully!");
var tableTrackingStore = service.TrackingStore as AzureTableTrackingStore;
if (tableTrackingStore != null)
{
DynamicTableEntity[] entities = (await tableTrackingStore.HistoryTable.ExecuteQuerySegmentedAsync(new TableQuery(), new TableContinuationToken())).ToArray();
int uniquePartitions = entities.GroupBy(e => e.PartitionKey).Count();
Trace.TraceInformation($"Found {uniquePartitions} unique partition(s) in table storage.");
Assert.AreEqual(InstanceCount, uniquePartitions, "Unexpected number of table partitions.");
}
}
finally
{
await worker.StopAsync(isForced : true);
}
}