public async Task ProcessJobsForMyDataCenter_TwoInQueue_TakeNextApproach()
{
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var spinWidgetJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var fooBarJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "FooBar" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var myDataCenter = new JobAttributes {
{ "DataCenter", "CAL01" }
};
var finished1 = await TryProcessOneJobUsingTakeNext(database, exampleQueueName, myDataCenter).ConfigureAwait(false);
var finished2 = await TryProcessOneJobUsingTakeNext(database, exampleQueueName, myDataCenter).ConfigureAwait(false);
var finished3 = await TryProcessOneJobUsingTakeNext(database, exampleQueueName, myDataCenter).ConfigureAwait(false);
Assert.That(finished1, Is.Not.Null);
Assert.That(finished2, Is.Not.Null);
Assert.That(finished3, Is.Null);
}).ConfigureAwait(false);
}
/// <summary>
/// Sets all of the attributes of this <code>JobAttributes</code> to
/// the same values as the attributes of obj.
/// </summary>
/// <param name="obj"> the <code>JobAttributes</code> to copy </param>
public void Set(JobAttributes obj)
{
Copies_Renamed = obj.Copies_Renamed;
DefaultSelection_Renamed = obj.DefaultSelection_Renamed;
Destination_Renamed = obj.Destination_Renamed;
Dialog_Renamed = obj.Dialog_Renamed;
FileName_Renamed = obj.FileName_Renamed;
FromPage_Renamed = obj.FromPage_Renamed;
MaxPage_Renamed = obj.MaxPage_Renamed;
MinPage_Renamed = obj.MinPage_Renamed;
MultipleDocumentHandling_Renamed = obj.MultipleDocumentHandling_Renamed;
// okay because we never modify the contents of pageRanges
PageRanges_Renamed = obj.PageRanges_Renamed;
PrFirst = obj.PrFirst;
PrLast = obj.PrLast;
Printer_Renamed = obj.Printer_Renamed;
Sides_Renamed = obj.Sides_Renamed;
ToPage_Renamed = obj.ToPage_Renamed;
}
public async Task Create_ValidInitialData_Persists()
{
var exampleQueueId = QueueId.Parse(Guid.NewGuid().ToString("N"));
var exampleAttributes = new JobAttributes {
{ "name", "Thing ToDo" }
};
await RunInMongoLand(async database =>
{
var sut = new MongoJobCreationService(database);
var newId = await sut.Create(exampleQueueId, exampleAttributes).ConfigureAwait(false);
Assert.That(newId, Is.Not.Null);
var collection = database.GetCollection <Job>(CollectionNames.Job);
var newlyCreatedJob = await collection.Find(Builders <Job> .Filter.Empty).SingleAsync().ConfigureAwait(false);
Assert.That(newlyCreatedJob.Id, Is.Not.Null);
Assert.That(newlyCreatedJob.Id, Is.Not.EqualTo(JobId.Empty()));
Assert.That(newlyCreatedJob.QueueId, Is.EqualTo(exampleQueueId));
Assert.That(newlyCreatedJob.Attributes, Is.EqualTo(exampleAttributes));
Assert.That(newlyCreatedJob.Attributes["name"], Is.EqualTo("Thing ToDo"));
}).ConfigureAwait(false);
}
/// <summary>
/// Determines whether two JobAttributes are equal to each other.
/// <para>
/// Two JobAttributes are equal if and only if each of their attributes are
/// equal. Attributes of enumeration type are equal if and only if the
/// fields refer to the same unique enumeration object. A set of page
/// ranges is equal if and only if the sets are of equal length, each range
/// enumerates the same pages, and the ranges are in the same order.
///
/// </para>
/// </summary>
/// <param name="obj"> the object whose equality will be checked. </param>
/// <returns> whether obj is equal to this JobAttribute according to the
/// above criteria. </returns>
public override bool Equals(Object obj)
{
if (!(obj is JobAttributes))
{
return(false);
}
JobAttributes rhs = (JobAttributes)obj;
if (FileName_Renamed == null)
{
if (rhs.FileName_Renamed != null)
{
return(false);
}
}
else
{
if (!FileName_Renamed.Equals(rhs.FileName_Renamed))
{
return(false);
}
}
if (PageRanges_Renamed == null)
{
if (rhs.PageRanges_Renamed != null)
{
return(false);
}
}
else
{
if (rhs.PageRanges_Renamed == null || PageRanges_Renamed.Length != rhs.PageRanges_Renamed.Length)
{
return(false);
}
for (int i = 0; i < PageRanges_Renamed.Length; i++)
{
if (PageRanges_Renamed[i][0] != rhs.PageRanges_Renamed[i][0] || PageRanges_Renamed[i][1] != rhs.PageRanges_Renamed[i][1])
{
return(false);
}
}
}
if (Printer_Renamed == null)
{
if (rhs.Printer_Renamed != null)
{
return(false);
}
}
else
{
if (!Printer_Renamed.Equals(rhs.Printer_Renamed))
{
return(false);
}
}
return(Copies_Renamed == rhs.Copies_Renamed && DefaultSelection_Renamed == rhs.DefaultSelection_Renamed && Destination_Renamed == rhs.Destination_Renamed && Dialog_Renamed == rhs.Dialog_Renamed && FromPage_Renamed == rhs.FromPage_Renamed && MaxPage_Renamed == rhs.MaxPage_Renamed && MinPage_Renamed == rhs.MinPage_Renamed && MultipleDocumentHandling_Renamed == rhs.MultipleDocumentHandling_Renamed && PrFirst == rhs.PrFirst && PrLast == rhs.PrLast && Sides_Renamed == rhs.Sides_Renamed && ToPage_Renamed == rhs.ToPage_Renamed);
}
/// <summary>
/// Constructs a <code>JobAttributes</code> instance which is a copy
/// of the supplied <code>JobAttributes</code>.
/// </summary>
/// <param name="obj"> the <code>JobAttributes</code> to copy </param>
public JobAttributes(JobAttributes obj)
{
Set(obj);
}
protected static async Task <Job> TryProcessOneJobUsingTakeNext(IMongoDatabase database, QueueId exampleQueueName, JobAttributes attributesThatShouldWork)
{
// Take Next available job
var takeNextService = new MongoJobTakeNextService(database);
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var peekQuery = new TakeNextOptions
{
QueueId = exampleQueueName,
HasAttributes = attributesThatShouldWork,
Acknowledgment = standardAck
};
var nextJob = await takeNextService.TakeFor(peekQuery).ConfigureAwait(false);
if (nextJob == null)
{
return(null);
}
var exampleReportMessage1 = "FooBar";
var exampleReportMessage2 = "WizBang";
var exampleReportMessage3 = "PowPop";
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(exampleQueueName, nextJob.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage1 } }).ConfigureAwait(false);
await
reportService.AddReport(exampleQueueName, nextJob.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage2 } }).ConfigureAwait(false);
await
reportService.AddReport(exampleQueueName, nextJob.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage3 } }).ConfigureAwait(false);
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(exampleQueueName, nextJob.Id, new JobResult { { "Result", "Success" } }).ConfigureAwait(false);
// Finish Job
var finishedJobFromDb =
await database.GetJobCollection().Find(Builders <Job> .Filter.Eq(x => x.Id, nextJob.Id)).FirstAsync().ConfigureAwait(false);
Assert.That(finishedJobFromDb, Is.Not.Null);
Assert.That(finishedJobFromDb.Acknowledgment, Is.Not.Null);
Assert.That(finishedJobFromDb.Reports, Has.Length.EqualTo(3));
var valuesOfReports = finishedJobFromDb.Reports.SelectMany(x => x.Values).ToList();
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage1));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage2));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage3));
Assert.That(finishedJobFromDb.Result, Is.Not.Null);
return(finishedJobFromDb);
}
protected static async Task <Job> TryProcessOneJobUsingPeekThanAck(IMongoDatabase database, QueueId exampleQueueName, JobAttributes attributesThatShouldWork)
{
// Query for next available a job for my datacenter
var peekNextService = new MongoJobPeekNextService(new MongoJobQueryService(database));
var peekQuery = new PeekNextOptions
{
QueueId = exampleQueueName,
HasAttributes = attributesThatShouldWork,
Limit = 5
};
var jobs = (await peekNextService.PeekFor(peekQuery).ConfigureAwait(false)).ToList();
if (!jobs.Any())
{
return(null);
}
foreach (var job in jobs)
{
// Acknowledge the job
var acknowledgmentService = new MongoJobAcknowledgmentService(database);
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var ackResult = await acknowledgmentService.Ack(exampleQueueName, job.Id, standardAck).ConfigureAwait(false);
if (!ackResult.Success)
{
continue;
}
var exampleReportMessage1 = "FooBar";
var exampleReportMessage2 = "WizBang";
var exampleReportMessage3 = "PowPop";
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(exampleQueueName, job.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage1 } }).ConfigureAwait(false);
await
reportService.AddReport(exampleQueueName, job.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage2 } }).ConfigureAwait(false);
await
reportService.AddReport(exampleQueueName, job.Id,
new JobReport { { "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage3 } }).ConfigureAwait(false);
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(exampleQueueName, job.Id, new JobResult { { "Result", "Success" } }).ConfigureAwait(false);
// Finish Job
var finishedJobFromDb =
await database.GetJobCollection().Find(Builders <Job> .Filter.Eq(x => x.Id, job.Id)).FirstAsync().ConfigureAwait(false);
Assert.That(finishedJobFromDb, Is.Not.Null);
Assert.That(finishedJobFromDb.Acknowledgment, Is.Not.Null);
Assert.That(finishedJobFromDb.Reports, Has.Length.EqualTo(3));
var valuesOfReports = finishedJobFromDb.Reports.SelectMany(x => x.Values).ToList();
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage1));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage2));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage3));
Assert.That(finishedJobFromDb.Result, Is.Not.Null);
return(finishedJobFromDb);
}
return(null);
}
public async Task ProcessJobsForMyDataCenter_XInQueueWithYConsumers_TakeNextApproach(int numberOfJobsToQueue, int numberOfConsumers)
{
var testStartTime = DateTime.Now;
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var jobsInQueue = new List <JobId>();
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < numberOfJobsToQueue; i++)
{
var newJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" },
}).ConfigureAwait(false);
jobsInQueue.Add(newJobId);
}
stopwatch.Stop();
Console.WriteLine($"Creating {numberOfJobsToQueue} jobs took: {stopwatch.ElapsedMilliseconds} ms");
var finishedJobs = new ConcurrentBag <Job>();
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
stopwatch.Reset();
stopwatch.Start();
for (int i = 0; i < numberOfConsumers; i++)
{
var myDataCenter = new JobAttributes {
{ "DataCenter", "CAL01" }
};
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, myDataCenter, cancellationToken).ConfigureAwait(false);
}
do
{
Thread.Sleep(100);
} while (finishedJobs.Count < jobsInQueue.Count && DateTime.Now < testStartTime.Add(MaxTestTime));
stopwatch.Stop();
Console.WriteLine($"{numberOfConsumers} consumers processed {numberOfJobsToQueue} jobs in: {stopwatch.ElapsedMilliseconds} ms");
cancellationSource.Cancel();
var finishedJobIds = finishedJobs.Select(x => x.Id).ToList();
foreach (var jobId in jobsInQueue)
{
Assert.That(finishedJobIds, Contains.Item(jobId));
}
}).ConfigureAwait(false);
}
private static Task SetupPeekThenAckWorkerWithSubscription(IMongoDatabase database, QueueId queueName, ConcurrentBag <Job> finishedJobs, JobAttributes attributesThatShouldWork, CancellationToken cancellationToken)
{
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var options = new PeekNextSubscriptionOptions()
{
Token = cancellationToken,
PeekNextOptions = new PeekNextOptions
{
QueueId = queueName,
HasAttributes = attributesThatShouldWork
}
};
var subscriber = new TaskBasedPeekNextSubscriber(new MongoJobPeekNextService(new MongoJobQueryService(database)));
return(subscriber.Subscribe(async nextJobs =>
{
var nextJob = nextJobs.First();
// Acknowledge the job
var acknowledgmentService = new MongoJobAcknowledgmentService(database);
var ackResult = await acknowledgmentService.Ack(queueName, nextJob.Id, standardAck).ConfigureAwait(false);
if (!ackResult.Success)
{
return;
}
var exampleReportMessage1 = "FooBar";
var exampleReportMessage2 = "WizBang";
var exampleReportMessage3 = "PowPop";
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(queueName, nextJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage1 }
}).ConfigureAwait(false);
await
reportService.AddReport(queueName, nextJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage2 }
}).ConfigureAwait(false);
await
reportService.AddReport(queueName, nextJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", exampleReportMessage3 }
}).ConfigureAwait(false);
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(queueName, nextJob.Id, new JobResult {
{ "Result", "Success" }
}).ConfigureAwait(false);
// Finish Job
var finishedJobFromDb =
await database.GetJobCollection().Find(Builders <Job> .Filter.Eq(x => x.Id, nextJob.Id)).FirstAsync(cancellationToken).ConfigureAwait(false);
Assert.That(finishedJobFromDb, Is.Not.Null);
Assert.That(finishedJobFromDb.Acknowledgment, Is.Not.Null);
Assert.That(finishedJobFromDb.Reports, Has.Length.EqualTo(3));
var valuesOfReports = finishedJobFromDb.Reports.SelectMany(x => x.Values).ToList();
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage1));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage2));
Assert.That(valuesOfReports, Contains.Item(exampleReportMessage3));
Assert.That(finishedJobFromDb.Result, Is.Not.Null);
finishedJobs.Add(finishedJobFromDb);
}, options));
}
public async Task RollingDeploy_ModelSubJobsOrchestratedByParent()
{
var exampleQueueName = QueueId.Parse("Wind");
await RunInMongoLand(async database =>
{
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
var finishedJobs = new ConcurrentBag <Job>();
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, new JobAttributes {
{ "Name", "DeployServer" },
}, cancellationToken).ConfigureAwait(false);
// Create a rolling deploy relevant for my datacenter
var creationService = new MongoJobCreationService(database);
await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "RollingDeploy" },
{ "Environment", "Prod" }
}).ConfigureAwait(false);
// Take Next available job
var takeNextSubscriber = new TaskBasedTakeNextSubscriber(new MongoJobTakeNextService(database));
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var peekQuery = new TakeNextOptions
{
QueueId = exampleQueueName,
HasAttributes = new JobAttributes {
{ "Environment", "Prod" }
},
Acknowledgment = standardAck
};
await takeNextSubscriber.Subscribe(async rollingDeployJob =>
{
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", "Starting Rolling Deploy" }
}).ConfigureAwait(false);
var queryService = new MongoJobQueryService(database);
IEnumerable servers = new[] { "PROD2", "PROD1" };
foreach (var server in servers)
{
var deployServer = new JobAttributes
{
{ "Name", "DeployServer" },
{ "Server", server }
};
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Requesting Deploy on server {server}" }
}).ConfigureAwait(false);
var deployServerJobId = await creationService.Create(exampleQueueName, deployServer).ConfigureAwait(false);
Job hasResult;
do
{
// replace with detail service
hasResult = (await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { deployServerJobId }, HasResult = true
}).ConfigureAwait(false)).FirstOrDefault();
Thread.Sleep(500);
} while (hasResult == null);
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Deploy on server {server} Completed, {JsonConvert.SerializeObject(hasResult.Result)}" }
}).ConfigureAwait(false);
// inspect result
}
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(exampleQueueName, rollingDeployJob.Id, new JobResult {
{ "Result", "Success" }
}).ConfigureAwait(false);
var finalizedRollingDeployJob = await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { rollingDeployJob.Id }
}).ConfigureAwait(false);
cancellationSource.Cancel();
Console.WriteLine($"Finalized Rolling Deploy Job: {JsonConvert.SerializeObject(finalizedRollingDeployJob)}");
Assert.That(finalizedRollingDeployJob.First().Result["Result"], Is.EqualTo("Success"));
}, new TakeNextSubscriptionOptions
{
TakeNextOptions = peekQuery,
Token = cancellationToken,
}).ConfigureAwait(false);
}).ConfigureAwait(false);
}