public async Task Ack_ValidInformation_StampsAcknowledgmentReturnsPositiveResult()
{
var exampleQueueName = QueueId.Parse("ExampleQueue");
var readyJobId = JobId.Generate();
var existingJobs = new[]
{
new Job
{
Id = readyJobId,
QueueId = exampleQueueName
}
};
var exampleAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobAcknowledgmentService(database);
var result = await sut.Ack(exampleQueueName, readyJobId, exampleAck).ConfigureAwait(false);
var jobWeExpectToHaveBeenAcknowledged = await jobs.Find(Builders <Job> .Filter.Eq(x => x.Id, readyJobId)).FirstAsync().ConfigureAwait(false);
Assert.That(result?.Success, Is.True);
Assert.That(jobWeExpectToHaveBeenAcknowledged.Acknowledgment, Is.EqualTo(exampleAck));
Assert.That(jobWeExpectToHaveBeenAcknowledged.Acknowledgment["RunnerId"], Is.EqualTo(exampleAck["RunnerId"]));
}).ConfigureAwait(false);
}
public async Task QueryCount_QueryByMultiAttributes_YieldMatchingResults()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasAttributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
{ "Geo", "USA" }
}
};
var matchingJob1 = JobId.Generate();
var unmatchedJob1 = JobId.Generate();
var unmatchedJob2 = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = unmatchedJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
}
}, new Job
{
Id = matchingJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
{ "Geo", "USA" }
}
}, new Job
{
Id = unmatchedJob2,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "Geo", "USA" },
}
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryCountService(database);
var results = (await sut.QueryCount(exampleQuery).ConfigureAwait(false));
long expected = 1;
Assert.That(results, Is.Not.Null);
Assert.AreEqual(results, expected);
}).ConfigureAwait(false);
}
public async Task Complete_ValidResult_Persists()
{
var exampleQueueId = QueueId.Parse(Guid.NewGuid().ToString("N"));
var exampleJobId = JobId.Generate();
var exampleResult = new JobResult {
{ "Datacenter", "PEN" }
};
Assert.That(exampleResult, Is.Not.Null);
await RunInMongoLand(async database =>
{
var collection = database.GetJobCollection();
collection.InsertOne(new Job {
QueueId = exampleQueueId, Id = exampleJobId
});
var sut = new MongoJobCompletionService(database);
await sut.Complete(exampleQueueId, exampleJobId, exampleResult).ConfigureAwait(false);
var result = await collection.Find(Builders <Job> .Filter.Empty).SingleAsync().ConfigureAwait(false);
Assert.That(result.Result, Is.Not.Null);
Assert.That(result.Result["Datacenter"], Is.EqualTo("PEN"));
}).ConfigureAwait(false);
}
public async Task Subscribe_WillNotCallWorkFuncIfNothingMetTheCriteria()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var exampleOptions = new PeekNextSubscriptionOptions
{
Token = cancel.Token,
PollingInterval = TimeSpan.FromMilliseconds(1),
PeekNextOptions = new PeekNextOptions()
};
var mockPeekNextService = new Mock <IJobPeekNextService>();
mockPeekNextService
.Setup(x => x.PeekFor(exampleOptions.PeekNextOptions))
.ReturnsAsync(Enumerable.Empty <Job>());
var sut = new TaskBasedPeekNextSubscriber(mockPeekNextService.Object);
var numberOfTimesOurDelegateIsInvoked = 0;
await sut.Subscribe(job => Task.FromResult(numberOfTimesOurDelegateIsInvoked += 1), exampleOptions).ConfigureAwait(false);
Thread.Sleep(10);
cancel.Cancel();
Assert.That(numberOfTimesOurDelegateIsInvoked, Is.EqualTo(0));
mockPeekNextService.Verify(x => x.PeekFor(exampleOptions.PeekNextOptions), Times.AtLeastOnce);
}
public async Task Subscribe_WillFindMessages(int numberOfJobs)
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var exampleOptions = new PeekNextSubscriptionOptions
{
Token = cancel.Token,
PollingInterval = TimeSpan.FromMilliseconds(1),
PeekNextOptions = new PeekNextOptions()
};
var mockPeekNextService = new Mock <IJobPeekNextService>();
var numberOfJobsTaken = 0;
mockPeekNextService.Setup(x => x.PeekFor(exampleOptions.PeekNextOptions)).Returns(() =>
{
numberOfJobsTaken += 1;
return(numberOfJobsTaken <= numberOfJobs ? Task.FromResult(new[] { new Job() }.AsEnumerable()) : Task.FromResult(Enumerable.Empty <Job>()));
});
var sut = new TaskBasedPeekNextSubscriber(mockPeekNextService.Object);
var numberOfTimesOurDelegateIsInvoked = 0;
await sut.Subscribe(job => Task.FromResult(numberOfTimesOurDelegateIsInvoked += 1), exampleOptions).ConfigureAwait(false);
do
{
Thread.Sleep(10);
// wait for us to have processed the jobs, or for the max time to be over.
} while (numberOfTimesOurDelegateIsInvoked < numberOfJobs && !cancel.IsCancellationRequested);
cancel.Cancel();
Assert.That(numberOfTimesOurDelegateIsInvoked, Is.EqualTo(numberOfJobs));
mockPeekNextService.Verify(x => x.PeekFor(exampleOptions.PeekNextOptions), Times.AtLeast(numberOfJobsTaken));
}
public async Task ProcessJobsForMyDataCenter_TwoInQueue_QueryThenAcknowledgeApproach()
{
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var spinWidgetJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var fooBarJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "FooBar" },
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished1 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished2 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
var finished3 = await TryProcessOneJobUsingPeekThanAck(database, exampleQueueName, new JobAttributes {
{ "DataCenter", "CAL01" }
}).ConfigureAwait(false);
Assert.That(finished1, Is.Not.Null);
Assert.That(finished2, Is.Not.Null);
Assert.That(finished3, Is.Null);
}).ConfigureAwait(false);
}
public async Task QueryFor_AJArrayOfStringsAndWithInvalidAdhocQuery_ThrowsException()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasAttributes = new JobAttributes
{
{ "name", new JArray {
"DeployApi", "DeployWebsite"
} }
},
AdhocQuery = "SampeInvalidMongoQuery"
};
await RunInMongoLand(database =>
{
var sut = new MongoJobQueryService(database);
var ex = Assert.ThrowsAsync <InvalidOperationException>(async() =>
{
await sut.QueryFor(exampleQuery).ConfigureAwait(false);
});
Assert.That(ex.Message, Does.Contain("Invalid adhocQuery"));
return(Task.FromResult(true));
}).ConfigureAwait(false);
}
protected override async Task EstablishContext()
{
await base.EstablishContext().ConfigureAwait(false);
var queueIds = new List <QueueId>();
for (int i = 0; i < NumberOfQueues; i++)
{
queueIds.Add(QueueId.Parse(Guid.NewGuid().ToString("N")));
}
var rando = new Random(DateTime.UtcNow.Millisecond);
var jobsToInsert = new List <Job>();
for (int i = 0; i < NumberOfExistingNewlyCreatedJobs; i++)
{
var queueIdToUse = queueIds[rando.Next(0, queueIds.Count)];
var newJob = new Job
{
Id = JobId.Generate(),
QueueId = queueIdToUse,
Attributes = AttributeGenerator()
};
jobsToInsert.Add(newJob);
}
for (int i = 0; i < NumberOfExistingAcknowledgedJobs; i++)
{
var queueIdToUse = queueIds[rando.Next(0, queueIds.Count)];
var newJob = new Job
{
Id = JobId.Generate(),
QueueId = queueIdToUse,
Attributes = AttributeGenerator(),
Acknowledgment = AckGenerator(),
};
jobsToInsert.Add(newJob);
}
for (int i = 0; i < NumberOfExistingCompletedJobs; i++)
{
var queueIdToUse = queueIds[rando.Next(0, queueIds.Count)];
var newJob = new Job
{
Id = JobId.Generate(),
QueueId = queueIdToUse,
Attributes = AttributeGenerator(),
Acknowledgment = AckGenerator(),
Result = ResultGenerator()
};
jobsToInsert.Add(newJob);
}
await Database.GetJobCollection().InsertManyAsync(jobsToInsert).ConfigureAwait(false);
GeneratedQueueId = queueIds;
}
public async Task QueryFor_HasResult_YieldsOnlyFinishedJobs()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasResult = true
};
var matchingJob1 = JobId.Generate();
var matchingJob2 = JobId.Generate();
var unmatchedJob1 = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = matchingJob1,
QueueId = exampleQueueId,
Result = new JobResult
{
{ "Result", "Failed" }
}
}, new Job
{
Id = matchingJob2,
QueueId = exampleQueueId,
Result = new JobResult
{
{ "Result", "Success" }
}
}, new Job
{
Id = unmatchedJob1,
QueueId = exampleQueueId,
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryService(database);
var results = (await sut.QueryFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(2));
// ReSharper disable once AssignNullToNotNullAttribute
var foundIds = results.Select(x => x.Id).ToList();
Assert.That(foundIds, Contains.Item(matchingJob1));
Assert.That(foundIds, Contains.Item(matchingJob2));
}).ConfigureAwait(false);
}
public async Task TakeFor_NewJobs_ReturnsAvailableJobAndAcknowledges()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleOptions = new TakeNextOptions
{
QueueId = exampleQueueId,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
};
var newlyCreatedJobId = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Result = new JobResult {
{ "Result", "Success" }
},
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = newlyCreatedJobId,
QueueId = exampleQueueId
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobTakeNextService(database);
var result = await sut.TakeFor(exampleOptions).ConfigureAwait(false);
Assert.That(result, Is.Not.Null);
Assert.That(result?.Id, Is.EqualTo(newlyCreatedJobId));
Assert.That(result?.Acknowledgment, Is.Not.Null);
Assert.That(result?.Acknowledgment["RunnerId"], Is.EqualTo(exampleOptions.Acknowledgment["RunnerId"]));
}).ConfigureAwait(false);
}
public async Task PeekFor_NewJobs_ReturnsAvailableJobs()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new PeekNextOptions
{
QueueId = exampleQueueId
};
var newlyCreatedJobId = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Result = new JobResult {
{ "Result", "Success" }
},
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = newlyCreatedJobId,
QueueId = exampleQueueId
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobPeekNextService(new MongoJobQueryService(database));
var results = (await sut.PeekFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(1));
Assert.That(results?.First()?.Id, Is.EqualTo(newlyCreatedJobId));
}).ConfigureAwait(false);
}
public async Task AddReport_AcknowledgedJobWithReports_AddsReport()
{
var exampleQueue = QueueId.Parse("ExampleQueue");
var myJobId = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = myJobId,
QueueId = exampleQueue,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
},
Reports = new [] { new JobReport {
{ "Start Time", DateTime.UtcNow.ToString("O") }
} }
} };
var exampleReport = new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", "FooBar" }
};
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobReportService(database);
await sut.AddReport(exampleQueue, myJobId, exampleReport).ConfigureAwait(false);
var myJob = await jobs.Find(Builders <Job> .Filter.Eq(x => x.Id, myJobId)).FirstAsync().ConfigureAwait(false);
Assert.That(myJob.Reports, Has.Length.EqualTo(2));
Assert.That(myJob.Reports.Last().Keys, Contains.Item("Timestamp"));
Assert.That(myJob.Reports.Last()["Timestamp"], Is.EqualTo(exampleReport["Timestamp"]));
Assert.That(myJob.Reports.Last().Keys, Contains.Item("Message"));
Assert.That(myJob.Reports.Last()["Message"], Is.EqualTo(exampleReport["Message"]));
}).ConfigureAwait(false);
}
public async Task QueryCount_Skip_YieldSkipped()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
Skip = 2,
};
var existingJobs = new[] { new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryCountService(database);
var results = (await sut.QueryCount(exampleQuery).ConfigureAwait(false));
long expected = 3;
Assert.That(results, Is.Not.Null);
Assert.AreEqual(results, expected);
}).ConfigureAwait(false);
}
public async Task QueryFor_Limit_YieldsOnlyToTheLimit()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
Limit = 2,
};
var existingJobs = new[] { new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryService(database);
var results = (await sut.QueryFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(2));
}).ConfigureAwait(false);
}
public async Task Create_ValidInitialData_Persists()
{
var exampleQueueId = QueueId.Parse(Guid.NewGuid().ToString("N"));
var exampleAttributes = new JobAttributes {
{ "name", "Thing ToDo" }
};
await RunInMongoLand(async database =>
{
var sut = new MongoJobCreationService(database);
var newId = await sut.Create(exampleQueueId, exampleAttributes).ConfigureAwait(false);
Assert.That(newId, Is.Not.Null);
var collection = database.GetCollection <Job>(CollectionNames.Job);
var newlyCreatedJob = await collection.Find(Builders <Job> .Filter.Empty).SingleAsync().ConfigureAwait(false);
Assert.That(newlyCreatedJob.Id, Is.Not.Null);
Assert.That(newlyCreatedJob.Id, Is.Not.EqualTo(JobId.Empty()));
Assert.That(newlyCreatedJob.QueueId, Is.EqualTo(exampleQueueId));
Assert.That(newlyCreatedJob.Attributes, Is.EqualTo(exampleAttributes));
Assert.That(newlyCreatedJob.Attributes["name"], Is.EqualTo("Thing ToDo"));
}).ConfigureAwait(false);
}
public async Task Subscribe_WillCheckMoreThanMoreThanOnce()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var exampleOptions = new PeekNextSubscriptionOptions
{
Token = cancel.Token,
PollingInterval = TimeSpan.FromMilliseconds(1),
PeekNextOptions = new PeekNextOptions()
};
var mockPeekNextService = new Mock <IJobPeekNextService>();
var sut = new TaskBasedPeekNextSubscriber(mockPeekNextService.Object);
var hasBeenCalled = false;
await sut.Subscribe(job => Task.FromResult(hasBeenCalled = true), exampleOptions).ConfigureAwait(false);
Thread.Sleep(10);
cancel.Cancel();
Assert.That(hasBeenCalled, Is.False);
mockPeekNextService.Verify(x => x.PeekFor(exampleOptions.PeekNextOptions), Times.AtLeast(2));
}
public async Task RollingDeploy_ModelSubJobsOrchestratedByParent()
{
var exampleQueueName = QueueId.Parse("Wind");
await RunInMongoLand(async database =>
{
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
var finishedJobs = new ConcurrentBag <Job>();
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, new JobAttributes {
{ "Name", "DeployServer" },
}, cancellationToken).ConfigureAwait(false);
// Create a rolling deploy relevant for my datacenter
var creationService = new MongoJobCreationService(database);
await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "RollingDeploy" },
{ "Environment", "Prod" }
}).ConfigureAwait(false);
// Take Next available job
var takeNextSubscriber = new TaskBasedTakeNextSubscriber(new MongoJobTakeNextService(database));
var standardAck = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
};
var peekQuery = new TakeNextOptions
{
QueueId = exampleQueueName,
HasAttributes = new JobAttributes {
{ "Environment", "Prod" }
},
Acknowledgment = standardAck
};
await takeNextSubscriber.Subscribe(async rollingDeployJob =>
{
// Send Reports
var reportService = new MongoJobReportService(database);
await
reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", "Starting Rolling Deploy" }
}).ConfigureAwait(false);
var queryService = new MongoJobQueryService(database);
IEnumerable servers = new[] { "PROD2", "PROD1" };
foreach (var server in servers)
{
var deployServer = new JobAttributes
{
{ "Name", "DeployServer" },
{ "Server", server }
};
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Requesting Deploy on server {server}" }
}).ConfigureAwait(false);
var deployServerJobId = await creationService.Create(exampleQueueName, deployServer).ConfigureAwait(false);
Job hasResult;
do
{
// replace with detail service
hasResult = (await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { deployServerJobId }, HasResult = true
}).ConfigureAwait(false)).FirstOrDefault();
Thread.Sleep(500);
} while (hasResult == null);
await reportService.AddReport(exampleQueueName, rollingDeployJob.Id,
new JobReport {
{ "Timestamp", DateTime.UtcNow.ToString("O") }, { "Message", $"Deploy on server {server} Completed, {JsonConvert.SerializeObject(hasResult.Result)}" }
}).ConfigureAwait(false);
// inspect result
}
// Send Result
var completionService = new MongoJobCompletionService(database);
await completionService.Complete(exampleQueueName, rollingDeployJob.Id, new JobResult {
{ "Result", "Success" }
}).ConfigureAwait(false);
var finalizedRollingDeployJob = await queryService.QueryFor(new JobQuery {
QueueId = exampleQueueName, JobIds = new[] { rollingDeployJob.Id }
}).ConfigureAwait(false);
cancellationSource.Cancel();
Console.WriteLine($"Finalized Rolling Deploy Job: {JsonConvert.SerializeObject(finalizedRollingDeployJob)}");
Assert.That(finalizedRollingDeployJob.First().Result["Result"], Is.EqualTo("Success"));
}, new TakeNextSubscriptionOptions
{
TakeNextOptions = peekQuery,
Token = cancellationToken,
}).ConfigureAwait(false);
}).ConfigureAwait(false);
}
public async Task TakeFor_NewJobs_OnlyModifiesOneJob()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleOptions = new TakeNextOptions
{
QueueId = exampleQueueId,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
};
var newlyCreatedJobId1 = JobId.Generate();
var newlyCreatedJobId2 = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Result = new JobResult {
{ "Result", "Success" }
},
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = JobId.Generate(),
QueueId = exampleQueueId,
Acknowledgment = new JobAcknowledgment
{
{ "RunnerId", Guid.NewGuid().ToString("N") }
}
}, new Job
{
Id = newlyCreatedJobId1,
QueueId = exampleQueueId
}, new Job
{
Id = newlyCreatedJobId2,
QueueId = exampleQueueId
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobTakeNextService(database);
var result = await sut.TakeFor(exampleOptions).ConfigureAwait(false);
Assert.That(result, Is.Not.Null);
Assert.That(result?.Id, Is.EqualTo(newlyCreatedJobId1));
var jobInDb1 = await jobs.Find(Builders <Job> .Filter.Eq(x => x.Id, newlyCreatedJobId1)).SingleAsync().ConfigureAwait(false);
Assert.That(jobInDb1.Acknowledgment, Is.Not.Null);
Assert.That(jobInDb1.Acknowledgment["RunnerId"], Is.EqualTo(exampleOptions.Acknowledgment["RunnerId"]));
var jobInDb2 = await jobs.Find(Builders <Job> .Filter.Eq(x => x.Id, newlyCreatedJobId2)).SingleAsync().ConfigureAwait(false);
Assert.That(jobInDb2.Acknowledgment, Is.Null);
}).ConfigureAwait(false);
}
public async Task QueryFor_QueryByMultiAttributes_YieldMatchingResults()
{
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasAttributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
{ "Geo", "USA" }
}
};
var matchingJob1 = JobId.Generate();
var unmatchedJob1 = JobId.Generate();
var unmatchedJob2 = JobId.Generate();
var existingJobs = new[] { new Job
{
Id = unmatchedJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
}
}, new Job
{
Id = matchingJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "DataCenter", "CAL01" },
{ "Geo", "USA" }
}
}, new Job
{
Id = unmatchedJob2,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "Geo", "USA" },
}
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryService(database);
var results = (await sut.QueryFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(1));
// ReSharper disable once AssignNullToNotNullAttribute
var foundIds = results.Select(x => x.Id).ToList();
Assert.That(foundIds, Contains.Item(matchingJob1));
}).ConfigureAwait(false);
}
public async Task QueryFor_AJArrayOfStringsAndWithOrStyleAdhocQuery_ReturnsAllJobsMatching()
{
var matchingJob1 = JobId.Generate();
var matchingJob2 = JobId.Generate();
var matchingJob3 = JobId.Generate();
var unmatchedJob1 = JobId.Generate();
var exampleAcknowledgedDateTime1 = new DateTime(2010, 1, 22, 22, 00, 00, DateTimeKind.Utc);
var exampleAcknowledgedDateTime2 = new DateTime(2010, 1, 22, 23, 00, 00, DateTimeKind.Utc);
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasAttributes = new JobAttributes
{
{ "name", new JArray {
"DeployApi", "DeployWebsite"
} }
},
AdhocQuery = "{\"$or\" : [{ \"Acknowledgment.acknowledgedDateTime\": ISODate(\"2010-01-22T22:00:00.000Z\")},{ \"Acknowledgment.acknowledgedDateTime\": ISODate(\"2010-01-22T23:00:00.000Z\")}]}"
};
var existingJobs = new[] { new Job
{
Id = matchingJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "name", "DeployApi" }
}
}, new Job
{
Id = matchingJob2,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "name", "DeployWebsite" }
},
Acknowledgment = new JobAcknowledgment
{
{ "acknowledgedDateTime", exampleAcknowledgedDateTime1 }
}
}, new Job
{
Id = unmatchedJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "name", "DeploySchema" }
}
}, new Job()
{
Id = matchingJob3,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "name", "DeployApi" }
},
Acknowledgment = new JobAcknowledgment
{
{ "acknowledgedDateTime", exampleAcknowledgedDateTime2 }
}
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryService(database);
var results = (await sut.QueryFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(2));
// ReSharper disable once AssignNullToNotNullAttribute
var foundIds = results.Select(x => x.Id).ToList();
Assert.That(foundIds, Contains.Item(matchingJob2));
Assert.That(foundIds, Contains.Item(matchingJob3));
}).ConfigureAwait(false);
}
public async Task QueryFor_AJArrayOfIntegers_ReturnsAllJobsMatching()
{
var matchingJob1 = JobId.Generate();
var matchingJob2 = JobId.Generate();
var unmatchedJob1 = JobId.Generate();
var exampleQueueId = QueueId.Parse("ExampleQueue");
var exampleQuery = new JobQuery
{
QueueId = exampleQueueId,
HasAttributes = new JobAttributes
{
{ "cpu", new JArray {
1, 7
} }
}
};
var existingJobs = new[] { new Job
{
Id = matchingJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "cpu", 1 }
}
}, new Job
{
Id = matchingJob2,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "cpu", 7 }
}
}, new Job
{
Id = unmatchedJob1,
QueueId = exampleQueueId,
Attributes = new JobAttributes
{
{ "cpu", 8 }
}
} };
await RunInMongoLand(async database =>
{
var jobs = database.GetJobCollection();
await jobs.InsertManyAsync(existingJobs).ConfigureAwait(false);
var sut = new MongoJobQueryService(database);
var results = (await sut.QueryFor(exampleQuery).ConfigureAwait(false))?.ToList();
Assert.That(results, Is.Not.Null);
Assert.That(results, Has.Count.EqualTo(2));
// ReSharper disable once AssignNullToNotNullAttribute
var foundIds = results.Select(x => x.Id).ToList();
Assert.That(foundIds, Contains.Item(matchingJob1));
Assert.That(foundIds, Contains.Item(matchingJob2));
}).ConfigureAwait(false);
}
public async Task ProcessJobsForMyDataCenter_XInQueueWithYConsumers_TakeNextApproach(int numberOfJobsToQueue, int numberOfConsumers)
{
var testStartTime = DateTime.Now;
var exampleQueueName = QueueId.Parse("ExampleQueue");
await RunInMongoLand(async database =>
{
// Create two jobs that are relevant for my datacenter
var creationService = new MongoJobCreationService(database);
var jobsInQueue = new List <JobId>();
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < numberOfJobsToQueue; i++)
{
var newJobId = await creationService.Create(exampleQueueName, new JobAttributes
{
{ "Name", "SpinWidget" },
{ "DataCenter", "CAL01" },
}).ConfigureAwait(false);
jobsInQueue.Add(newJobId);
}
stopwatch.Stop();
Console.WriteLine($"Creating {numberOfJobsToQueue} jobs took: {stopwatch.ElapsedMilliseconds} ms");
var finishedJobs = new ConcurrentBag <Job>();
var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(MaxTestTime);
var cancellationToken = cancellationSource.Token;
stopwatch.Reset();
stopwatch.Start();
for (int i = 0; i < numberOfConsumers; i++)
{
var myDataCenter = new JobAttributes {
{ "DataCenter", "CAL01" }
};
await SetupTakeNextWorkerWithSubscription(database, exampleQueueName, finishedJobs, myDataCenter, cancellationToken).ConfigureAwait(false);
}
do
{
Thread.Sleep(100);
} while (finishedJobs.Count < jobsInQueue.Count && DateTime.Now < testStartTime.Add(MaxTestTime));
stopwatch.Stop();
Console.WriteLine($"{numberOfConsumers} consumers processed {numberOfJobsToQueue} jobs in: {stopwatch.ElapsedMilliseconds} ms");
cancellationSource.Cancel();
var finishedJobIds = finishedJobs.Select(x => x.Id).ToList();
foreach (var jobId in jobsInQueue)
{
Assert.That(finishedJobIds, Contains.Item(jobId));
}
}).ConfigureAwait(false);
}