public async Task ProcessJobsForMyDataCenter_TwoInQueue_QueryThenAcknowledgeApproach()
{
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var spinWidgetJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var fooBarJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "FooBar" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished1 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished2 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished3 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
Assert.That(finished1, Is.Not.Null);
Assert.That(finished2, Is.Not.Null);
Assert.That(finished3, Is.Null);
}).ConfigureAwait(false);
}
protected override async Task Because()
{
await base.Because().ConfigureAwait(false);
StartTime = DateTime.Now;
var creationService = new MongoJobCreationService(Database);
await creationService.Create(GeneratedQueueId.First(), new JobAttributes()).ConfigureAwait(false);
EndTime = DateTime.Now;
}
public async Task Create_ValidInitialData_Persists()
{
var exampleQueueId = QueueId.Parse(Guid.NewGuid().ToString("N"));
var exampleAttributes = new JobAttributes {
{ "name", "Thing ToDo" }
};
await RunInMongoLand(async database =>
{
var sut = new MongoJobCreationService(database);
var newId = await sut.Create(exampleQueueId, exampleAttributes).ConfigureAwait(false);
Assert.That(newId, Is.Not.Null);
var collection = database.GetCollection <Job>(CollectionNames.Job);
var newlyCreatedJob = await collection.Find(Builders <Job> .Filter.Empty).SingleAsync().ConfigureAwait(false);
Assert.That(newlyCreatedJob.Id, Is.Not.Null);
Assert.That(newlyCreatedJob.Id, Is.Not.EqualTo(JobId.Empty()));
Assert.That(newlyCreatedJob.QueueId, Is.EqualTo(exampleQueueId));
Assert.That(newlyCreatedJob.Attributes, Is.EqualTo(exampleAttributes));
Assert.That(newlyCreatedJob.Attributes["name"], Is.EqualTo("Thing ToDo"));
}).ConfigureAwait(false);
}
public async Task ProcessJobsForMyDataCenter_XInQueueWithYConsumers_TakeNextApproach(int numberOfJobsToQueue, int numberOfConsumers)
{
var testStartTime = DateTime.Now;
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var jobsInQueue = new List <JobId>();
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < numberOfJobsToQueue; i++)
{
var newJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" },
}).ConfigureAwait(false);
jobsInQueue.Add(newJobId);
}
stopwatch.Stop();
Console.WriteLine($"Creating {numberOfJobsToQueue} jobs took: {stopwatch.ElapsedMilliseconds} ms");
var finishedJobs = new ConcurrentBag <Job>();
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
stopwatch.Reset();
stopwatch.Start();
for (int i = 0; i < numberOfConsumers; i++)
{
var myDataCenter = new JobAttributes {
{ "DataCenter", "CAL01" }
};
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, myDataCenter, cancellationToken).ConfigureAwait(false);
}
do
{
Thread.Sleep(100);
} while (finishedJobs.Count < jobsInQueue.Count && DateTime.Now < testStartTime.Add(MaxTestTime));
stopwatch.Stop();
Console.WriteLine($"{numberOfConsumers} consumers processed {numberOfJobsToQueue} jobs in: {stopwatch.ElapsedMilliseconds} ms");
cancellationSource.Cancel();
var finishedJobIds = finishedJobs.Select(x => x.Id).ToList();
foreach (var jobId in jobsInQueue)
{
Assert.That(finishedJobIds, Contains.Item(jobId));
}
}).ConfigureAwait(false);
}
public async Task RollingDeploy_ModelSubJobsOrchestratedByParent()
{
var exampleQueueName = QueueId.Parse("Wind");
await RunInMongoLand(async database =>
{
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
var finishedJobs = new ConcurrentBag <Job>();
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, new JobAttributes {
{ "Name", "DeployServer" },
}, cancellationToken).ConfigureAwait(false);
// Create a rolling deploy relevant for my datacenter
var creationService = new MongoJobCreationService(database);
await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "RollingDeploy" },
{ "Environment", "Prod" }
}).ConfigureAwait(false);
// Take Next available job
var takeNextSubscriber = new TaskBasedTakeNextSubscriber(new MongoJobTakeNextService(database));
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var peekQuery = new TakeNextOptions
{
QueueId = exampleQueueName,
HasAttributes = new JobAttributes {
{ "Environment", "Prod" }
},
Acknowledgment = standardAck
};
await takeNextSubscriber.Subscribe(async rollingDeployJob =>
{
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", "Starting Rolling Deploy" }
}).ConfigureAwait(false);
var queryService = new MongoJobQueryService(database);
IEnumerable servers = new[] { "PROD2", "PROD1" };
foreach (var server in servers)
{
var deployServer = new JobAttributes
{
{ "Name", "DeployServer" },
{ "Server", server }
};
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Requesting Deploy on server {server}" }
}).ConfigureAwait(false);
var deployServerJobId = await creationService.Create(exampleQueueName, deployServer).ConfigureAwait(false);
Job hasResult;
do
{
// replace with detail service
hasResult = (await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { deployServerJobId }, HasResult = true
}).ConfigureAwait(false)).FirstOrDefault();
Thread.Sleep(500);
} while (hasResult == null);
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Deploy on server {server} Completed, {JsonConvert.SerializeObject(hasResult.Result)}" }
}).ConfigureAwait(false);
// inspect result
}
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(exampleQueueName, rollingDeployJob.Id, new JobResult {
{ "Result", "Success" }
}).ConfigureAwait(false);
var finalizedRollingDeployJob = await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { rollingDeployJob.Id }
}).ConfigureAwait(false);
cancellationSource.Cancel();
Console.WriteLine($"Finalized Rolling Deploy Job: {JsonConvert.SerializeObject(finalizedRollingDeployJob)}");
Assert.That(finalizedRollingDeployJob.First().Result["Result"], Is.EqualTo("Success"));
}, new TakeNextSubscriptionOptions
{
TakeNextOptions = peekQuery,
Token = cancellationToken,
}).ConfigureAwait(false);
}).ConfigureAwait(false);
}