public async Task RunProject(
string project,
int buildBatchSize,
CancellationToken cancellationToken)
{
DateTimeOffset latest;
DateTimeOffset?latestCandidate = await _timelineTelemetryRepository.GetLatestTimelineBuild(project);
if (latestCandidate.HasValue)
{
latest = latestCandidate.Value;
}
else
{
latest = _systemClock.UtcNow.Subtract(TimeSpan.FromDays(30));
_logger.LogWarning($"No previous time found, using {latest.LocalDateTime:O}");
}
_logger.LogInformation("Reading project {project}", project);
Build[] builds = await GetBuildsAsync(_azureServer, project, latest, buildBatchSize, cancellationToken);
_logger.LogTrace("... found {builds} builds...", builds.Length);
if (builds.Length == 0)
{
_logger.LogTrace("No work to do");
return;
}
List <(int buildId, BuildRequestValidationResult validationResult)> validationResults = builds
.SelectMany(
build => build.ValidationResults,
(build, validationResult) => (build.Id, validationResult))
.ToList();
_logger.LogTrace("Fetching timeline...");
Dictionary <Build, Task <Timeline> > tasks = builds
.ToDictionary(
build => build,
build => _azureServer.GetTimelineAsync(project, build.Id, cancellationToken)
);
await Task.WhenAll(tasks.Select(s => s.Value));
// Identify additional timelines by inspecting each record for a "PreviousAttempt"
// object, then fetching the "timelineId" field.
List <(Build build, Task <Timeline> timelineTask)> retriedTimelineTasks = new List <(Build, Task <Timeline>)>();
foreach ((Build build, Task <Timeline> timelineTask) in tasks)
{
Timeline timeline = await timelineTask;
if (timeline is null)
{
_logger.LogDebug("No timeline found for buildid {buildid}", build.Id);
continue;
}
IEnumerable <string> additionalTimelineIds = timeline.Records
.Where(record => record.PreviousAttempts != null)
.SelectMany(record => record.PreviousAttempts)
.Select(attempt => attempt.TimelineId)
.Distinct();
retriedTimelineTasks.AddRange(
additionalTimelineIds.Select(
timelineId => (build, _azureServer.GetTimelineAsync(project, build.Id, timelineId, cancellationToken))));
}
await Task.WhenAll(retriedTimelineTasks.Select(o => o.timelineTask));
// Only record timelines where their "lastChangedOn" field is after the last
// recorded date. Anything before has already been recorded.
List <(Build build, Task <Timeline> timeline)> allNewTimelines = new List <(Build build, Task <Timeline> timeline)>();
allNewTimelines.AddRange(tasks.Select(t => (t.Key, t.Value)));
allNewTimelines.AddRange(retriedTimelineTasks
.Where(t => t.timelineTask.Result.LastChangedOn > latest));
_logger.LogTrace("... finished timeline");
var records = new List <AugmentedTimelineRecord>();
var issues = new List <AugmentedTimelineIssue>();
var augmentedBuilds = new List <AugmentedBuild>();
_logger.LogTrace("Aggregating results...");
foreach ((Build build, Task <Timeline> timelineTask) in allNewTimelines)
{
using IDisposable buildScope = _logger.BeginScope(KeyValuePair.Create("buildId", build.Id));
augmentedBuilds.Add(CreateAugmentedBuild(build));
Timeline timeline = await timelineTask;
if (timeline?.Records == null)
{
continue;
}
var recordCache =
new Dictionary <string, AugmentedTimelineRecord>();
var issueCache = new List <AugmentedTimelineIssue>();
foreach (TimelineRecord record in timeline.Records)
{
var augRecord = new AugmentedTimelineRecord(build.Id, timeline.Id, record);
recordCache.Add(record.Id, augRecord);
records.Add(augRecord);
if (record.Issues == null)
{
continue;
}
for (int iIssue = 0; iIssue < record.Issues.Length; iIssue++)
{
var augIssue =
new AugmentedTimelineIssue(build.Id, timeline.Id, record.Id, iIssue, record.Issues[iIssue]);
augIssue.Bucket = GetBucket(augIssue);
issueCache.Add(augIssue);
issues.Add(augIssue);
}
}
foreach (AugmentedTimelineRecord record in recordCache.Values)
{
FillAugmentedOrder(record, recordCache);
}
foreach (AugmentedTimelineIssue issue in issueCache)
{
if (recordCache.TryGetValue(issue.RecordId, out AugmentedTimelineRecord record))
{
issue.AugmentedIndex = record.AugmentedOrder + "." + issue.Index.ToString("D3");
}
else
{
issue.AugmentedIndex = "999." + issue.Index.ToString("D3");
}
}
}
_logger.LogInformation("Saving TimelineBuilds...");
await _timelineTelemetryRepository.WriteTimelineBuilds(augmentedBuilds);
_logger.LogInformation("Saving TimelineValidationMessages...");
await _timelineTelemetryRepository.WriteTimelineValidationMessages(validationResults);
_logger.LogInformation("Saving TimelineRecords...");
await _timelineTelemetryRepository.WriteTimelineRecords(records);
_logger.LogInformation("Saving TimelineIssues...");
await _timelineTelemetryRepository.WriteTimelineIssues(issues);
}
