public async Task GetFinishedJobRuns_OutsideTimeWindow_ReturnsNoFinishedJobs()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
DateTimeOffset endWindow = BogusGenerator.Date.SoonOffset();
IEnumerable <Run> includedRuns = CreateRandomRuns(startWindow, endWindow);
IEnumerable <Run> tooLateRuns = CreateRandomRuns(endWindow, BogusGenerator.Date.FutureOffset());
IEnumerable <Run> allRuns = includedRuns.Concat(tooLateRuns);
IEnumerable <Job> jobs = includedRuns.Select(r => new Job
{
JobId = r.JobId,
Settings = new JobSettings {
Name = Guid.NewGuid().ToString()
}
}).ToArray();
DatabricksClient client = DatabricksClientFactory.Create(allRuns, jobs);
var provider = new DatabricksInfoProvider(client, NullLogger.Instance);
// Act
IEnumerable <JobRun> finishedJobs = await provider.GetFinishedJobRunsAsync(BogusGenerator.Date.PastOffset(), startWindow);
// Assert
Assert.Empty(finishedJobs);
}
public async Task GetFinishedJobRuns_WithEndTimeLessThanStartTime_Throws()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.SoonOffset();
DateTimeOffset endWindow = BogusGenerator.Date.RecentOffset();
DatabricksClient client = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
var provider = new DatabricksInfoProvider(client, NullLogger.Instance);
// Act / Assert
await Assert.ThrowsAnyAsync <ArgumentException>(() => provider.GetFinishedJobRunsAsync(startWindow, endWindow));
}
public async Task GetFinishedJobRuns_WithNoAvailableFinishedJobs_ReturnsNoFinishedJobs()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
DateTimeOffset endWindow = BogusGenerator.Date.SoonOffset();
DatabricksClient client = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
var provider = new DatabricksInfoProvider(client, NullLogger.Instance);
// Act
IEnumerable <JobRun> finishedJobs = await provider.GetFinishedJobRunsAsync(startWindow, endWindow);
// Assert
Assert.Empty(finishedJobs);
}
public async Task Run([TimerTrigger("0 */1 * * * *")] TimerInfo timer, ILogger logger)
{
logger.LogInformation($"C# Timer trigger function executed at: {DateTime.UtcNow}");
var metricName = _configuration.GetValue <string>("Arcus:ApplicationInsights:MetricName");
var baseUrl = _configuration.GetValue <string>("Arcus:Databricks:Url");
string secretToken = await _secretProvider.GetRawSecretAsync("Arcus.Databricks.SecretToken");
var startOfWindow = timer.ScheduleStatus.Last;
var endOfWindow = timer.ScheduleStatus.Next;
using var client = DatabricksClient.CreateClient(baseUrl, secretToken);
using (var provider = new DatabricksInfoProvider(client, logger))
{
await provider.MeasureJobOutcomesAsync(metricName, startOfWindow, endOfWindow);
}
}
public async Task MeasureJobOutcomes_WithNoAvailableFinishedJobs_ReturnsNoFinishedJobs()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
DateTimeOffset endWindow = BogusGenerator.Date.SoonOffset();
string metricName = BogusGenerator.Random.Word();
DatabricksClient client = DatabricksClientFactory.Create(Enumerable.Empty <Run>(), Enumerable.Empty <Job>());
var spyLogger = new InMemoryLogger <DatabricksJobMetricsJob>();
var provider = new DatabricksInfoProvider(client, spyLogger);
// Act
await provider.MeasureJobOutcomesAsync(metricName, startWindow, endWindow);
// Assert
Assert.DoesNotContain(spyLogger.Messages, msg => msg.StartsWith("Metric " + metricName));
}
public async Task MeasureJobOutcomes_WithinTimeWindow_OnlyReturnsFinishedJobsWithinTheTimeWindow()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
DateTimeOffset endWindow = BogusGenerator.Date.SoonOffset();
string metricName = BogusGenerator.Random.Word();
IEnumerable <Run> includedRuns = CreateRandomRuns(startWindow, endWindow);
IEnumerable <Run> tooLateRuns = CreateRandomRuns(endWindow, BogusGenerator.Date.FutureOffset());
IEnumerable <Run> allRuns = includedRuns.Concat(tooLateRuns);
IEnumerable <Job> jobs = allRuns.Select(r => new Job
{
JobId = r.JobId,
Settings = new JobSettings {
Name = Guid.NewGuid().ToString()
}
}).ToArray();
DatabricksClient client = DatabricksClientFactory.Create(allRuns, jobs);
var spyLogger = new InMemoryLogger <DatabricksJobMetricsJob>();
var provider = new DatabricksInfoProvider(client, spyLogger);
// Act
await provider.MeasureJobOutcomesAsync(metricName, startWindow, endWindow);
// Assert
Assert.All(includedRuns, run =>
{
Assert.Contains(spyLogger.Messages, msg => msg.Contains(run.RunId.ToString()));
Assert.Contains(spyLogger.Messages, msg => msg.Contains(run.JobId.ToString()));
Job job = Assert.Single(jobs, j => j.JobId == run.JobId);
Assert.NotNull(job);
Assert.Contains(spyLogger.Messages, msg => msg.Contains(job.Settings.Name));
});
Assert.All(tooLateRuns, run =>
{
Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(run.RunId.ToString()));
Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(run.JobId.ToString()));
Job job = Assert.Single(jobs, j => j.JobId == run.JobId);
Assert.NotNull(job);
Assert.DoesNotContain(spyLogger.Messages, msg => msg.Contains(job.Settings.Name));
});
}
public async Task GetFinishedJobRuns_WithinTimeWindow_OnlyReturnsFinishedJobsWithinTheTimeWindow()
{
// Arrange
DateTimeOffset startWindow = BogusGenerator.Date.RecentOffset();
DateTimeOffset endWindow = BogusGenerator.Date.SoonOffset();
IEnumerable <Run> includedRuns = CreateRandomRuns(startWindow, endWindow);
IEnumerable <Run> tooLateRuns = CreateRandomRuns(endWindow, BogusGenerator.Date.FutureOffset());
IEnumerable <Run> allRuns = includedRuns.Concat(tooLateRuns);
IEnumerable <Job> jobs = includedRuns.Select(r => new Job
{
JobId = r.JobId,
Settings = new JobSettings {
Name = Guid.NewGuid().ToString()
}
}).ToArray();
DatabricksClient client = DatabricksClientFactory.Create(allRuns, jobs);
var provider = new DatabricksInfoProvider(client, NullLogger.Instance);
// Act
IEnumerable <JobRun> finishedJobs = await provider.GetFinishedJobRunsAsync(startWindow, endWindow);
// Assert
Assert.NotNull(finishedJobs);
Assert.NotEmpty(finishedJobs);
Assert.Equal(finishedJobs.Count(), includedRuns.Count());
Assert.All(finishedJobs, job =>
{
Assert.Contains(includedRuns, run => run.RunId == job.Run.RunId);
Assert.DoesNotContain(tooLateRuns, run => run.RunId == job.Run.RunId);
Job expectedJob = Assert.Single(jobs, j => j.Settings.Name == job.JobName);
Assert.NotNull(expectedJob);
Assert.Equal(expectedJob.JobId, job.Run.JobId);
});
}
