public async Task WriteTimelineBuilds(IEnumerable <AugmentedBuild> augmentedBuilds)
{
await KustoHelpers.WriteDataToKustoInMemoryAsync(
_ingest,
_database,
"TimelineBuilds",
_logger,
augmentedBuilds,
b => new[]
{
new KustoValue("BuildId", b.Build.Id, KustoDataType.Int),
new KustoValue("Status", b.Build.Status, KustoDataType.String),
new KustoValue("Result", b.Build.Result, KustoDataType.String),
new KustoValue("Repository", b.Build.Repository?.Name ?? b.Build.Repository?.Id, KustoDataType.String),
new KustoValue("Reason", b.Build.Reason, KustoDataType.String),
new KustoValue("BuildNumber", b.Build.BuildNumber, KustoDataType.String),
new KustoValue("QueueTime", b.Build.QueueTime, KustoDataType.DateTime),
new KustoValue("StartTime", b.Build.StartTime, KustoDataType.DateTime),
new KustoValue("FinishTime", b.Build.FinishTime, KustoDataType.DateTime),
new KustoValue("Project", b.Build.Project?.Name, KustoDataType.String),
new KustoValue("DefinitionId", b.Build.Definition?.Id.ToString(), KustoDataType.String),
new KustoValue("Definition", $"{b.Build.Definition?.Path}\\{b.Build.Definition?.Name}", KustoDataType.String),
new KustoValue("SourceBranch", GitHelpers.NormalizeBranchName(b.Build.SourceBranch), KustoDataType.String),
new KustoValue("TargetBranch", GitHelpers.NormalizeBranchName(b.TargetBranch), KustoDataType.String),
});
}
public async Task WriteTimelineIssues(IEnumerable <AugmentedTimelineIssue> issues)
{
await KustoHelpers.WriteDataToKustoInMemoryAsync(
_ingest,
_database,
"TimelineIssues",
_logger,
issues,
b => new[]
{
new KustoValue("BuildId", b.BuildId, KustoDataType.Int),
new KustoValue("RecordId", b.RecordId, KustoDataType.String),
new KustoValue("TimelineId", b.TimelineId, KustoDataType.String),
new KustoValue("Index", b.Index, KustoDataType.Int),
new KustoValue("Path", b.AugmentedIndex, KustoDataType.String),
new KustoValue("Type", b.Raw.Type, KustoDataType.String),
new KustoValue("Category", b.Raw.Category, KustoDataType.String),
new KustoValue("Message", b.Raw.Message, KustoDataType.String),
new KustoValue("Bucket", b.Bucket, KustoDataType.String),
});
}
public async Task WriteTimelineRecords(IEnumerable <AugmentedTimelineRecord> records)
{
await KustoHelpers.WriteDataToKustoInMemoryAsync(
_ingest,
_database,
"TimelineRecords",
_logger,
records,
b => new[]
{
new KustoValue("BuildId", b.BuildId, KustoDataType.Int),
new KustoValue("RecordId", b.Raw.Id, KustoDataType.String),
new KustoValue("TimelineId", b.TimelineId, KustoDataType.String),
new KustoValue("Order", b.Raw.Order, KustoDataType.Int),
new KustoValue("Path", b.AugmentedOrder, KustoDataType.String),
new KustoValue("ParentId", b.Raw.ParentId, KustoDataType.String),
new KustoValue("Name", b.Raw.Name, KustoDataType.String),
new KustoValue("StartTime", b.Raw.StartTime, KustoDataType.DateTime),
new KustoValue("FinishTime", b.Raw.FinishTime, KustoDataType.DateTime),
new KustoValue("Result", b.Raw.Result, KustoDataType.String),
new KustoValue("ResultCode", b.Raw.ResultCode, KustoDataType.String),
new KustoValue("ChangeId", b.Raw.ChangeId, KustoDataType.Int),
new KustoValue("LastModified", b.Raw.LastModified, KustoDataType.DateTime),
new KustoValue("WorkerName", b.Raw.WorkerName, KustoDataType.String),
new KustoValue("Details", b.Raw.Details?.Url, KustoDataType.String),
new KustoValue("ErrorCount", b.Raw.ErrorCount, KustoDataType.Int),
new KustoValue("WarningCount", b.Raw.WarningCount, KustoDataType.Int),
new KustoValue("Url", b.Raw.Url, KustoDataType.String),
new KustoValue("LogId", b.Raw.Log?.Id, KustoDataType.Int),
new KustoValue("LogUri", b.Raw.Log?.Url, KustoDataType.String),
new KustoValue("TaskId", b.Raw.Task?.Id, KustoDataType.Int),
new KustoValue("TaskName", b.Raw.Task?.Name, KustoDataType.String),
new KustoValue("TaskVersion", b.Raw.Task?.Version, KustoDataType.String),
new KustoValue("Attempt", b.Raw.Attempt, KustoDataType.Int),
});
}
private async Task RunProject(
AzureDevOpsClient azureServer,
string project,
int buildBatchSize,
AzureDevOpsTimelineOptions options,
CancellationToken cancellationToken)
{
DateTimeOffset latest;
try
{
using (ICslQueryProvider query =
KustoClientFactory.CreateCslQueryProvider(options.KustoQueryConnectionString))
using (IDataReader result = await query.ExecuteQueryAsync(
options.KustoDatabase,
// This isn't use controlled, so I'm not worried about the Kusto injection
$"TimelineBuilds | where Project == '{project}' | summarize max(FinishTime)",
new ClientRequestProperties()
))
{
if (!result.Read())
{
latest = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(30));
_logger.LogWarning($"No previous time found, using {latest.LocalDateTime:O}");
}
else
{
latest = result.GetDateTime(0);
_logger.LogInformation($"... fetched previous time of {latest.LocalDateTime:O}");
}
}
}
catch (SemanticException e) when(e.SemanticErrors == "'where' operator: Failed to resolve column or scalar expression named 'Project'")
{
// The Project column isn't there, we probably reinitalized the tables
latest = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(30));
_logger.LogWarning($"No table column 'Project' found, assumed reinialization: using {latest.LocalDateTime:O}");
}
_logger.LogInformation("Reading project {project}", project);
Build[] builds = await GetBuildsAsync(azureServer, project, latest, buildBatchSize, cancellationToken);
_logger.LogTrace("... found {builds} builds...", builds.Length);
if (builds.Length == 0)
{
_logger.LogTrace("No work to do");
return;
}
List <(int buildId, BuildRequestValidationResult validationResult)> validationResults = builds
.SelectMany(
build => build.ValidationResults,
(build, validationResult) => (build.Id, validationResult))
.ToList();
_logger.LogTrace("Fetching timeline...");
Dictionary <Build, Task <Timeline> > tasks = builds
.ToDictionary(
build => build,
build => azureServer.GetTimelineAsync(project, build.Id, cancellationToken)
);
await Task.WhenAll(tasks.Select(s => s.Value));
_logger.LogTrace("... finished timeline");
var records = new List <AugmentedTimelineRecord>();
var issues = new List <AugmentedTimelineIssue>();
_logger.LogTrace("Aggregating results...");
foreach ((Build build, Task <Timeline> timelineTask) in tasks)
{
Timeline timeline = await timelineTask;
if (timeline?.Records == null)
{
continue;
}
var recordCache =
new Dictionary <string, AugmentedTimelineRecord>();
var issueCache = new List <AugmentedTimelineIssue>();
foreach (TimelineRecord record in timeline.Records)
{
var augRecord = new AugmentedTimelineRecord(build.Id, record);
recordCache.Add(record.Id, augRecord);
records.Add(augRecord);
if (record.Issues == null)
{
continue;
}
for (int iIssue = 0; iIssue < record.Issues.Length; iIssue++)
{
var augIssue =
new AugmentedTimelineIssue(build.Id, record.Id, iIssue, record.Issues[iIssue]);
augIssue.Bucket = GetBucket(augIssue);
issueCache.Add(augIssue);
issues.Add(augIssue);
}
}
foreach (AugmentedTimelineRecord record in recordCache.Values)
{
FillAugmentedOrder(record, recordCache);
}
foreach (AugmentedTimelineIssue issue in issueCache)
{
if (recordCache.TryGetValue(issue.RecordId, out AugmentedTimelineRecord record))
{
issue.AugmentedIndex = record.AugmentedOrder + "." + issue.Index.ToString("D3");
}
else
{
issue.AugmentedIndex = "999." + issue.Index.ToString("D3");
}
}
}
if (string.IsNullOrEmpty(options.KustoIngestConnectionString))
{
_logger.LogError("No KustoIngestConnectionString set");
return;
}
IKustoIngestClient ingest =
KustoIngestFactory.CreateQueuedIngestClient(options.KustoIngestConnectionString);
_logger.LogInformation("Saving TimelineBuilds...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineBuilds",
_logger,
builds,
b => new[]
{
new KustoValue("BuildId", b.Id.ToString(), KustoDataTypes.Int),
new KustoValue("Status", b.Status, KustoDataTypes.String),
new KustoValue("Result", b.Result, KustoDataTypes.String),
new KustoValue("Repository", b.Repository?.Name ?? b.Repository?.Id, KustoDataTypes.String),
new KustoValue("Reason", b.Reason, KustoDataTypes.String),
new KustoValue("BuildNumber", b.BuildNumber, KustoDataTypes.String),
new KustoValue("QueueTime", b.QueueTime, KustoDataTypes.DateTime),
new KustoValue("StartTime", b.StartTime, KustoDataTypes.DateTime),
new KustoValue("FinishTime", b.FinishTime, KustoDataTypes.DateTime),
new KustoValue("Project", b.Project?.Name, KustoDataTypes.String),
new KustoValue("DefinitionId", b.Definition?.Id.ToString(), KustoDataTypes.String),
new KustoValue("Definition", $"{b.Definition?.Path}\\{b.Definition?.Name}", KustoDataTypes.String),
});
_logger.LogInformation("Saving TimelineValidationMessages...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineIssues",
_logger,
validationResults,
b => new[]
{
new KustoValue("BuildId", b.buildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", null, KustoDataTypes.String),
new KustoValue("Index", null, KustoDataTypes.Int),
new KustoValue("Path", null, KustoDataTypes.String),
new KustoValue("Type", b.validationResult.Result, KustoDataTypes.String),
new KustoValue("Category", "ValidationResult", KustoDataTypes.String),
new KustoValue("Message", b.validationResult.Message, KustoDataTypes.String),
new KustoValue("Bucket", "ValidationResult", KustoDataTypes.String),
});
_logger.LogInformation("Saving TimelineRecords...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineRecords",
_logger,
records,
b => new[]
{
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.Raw.Id, KustoDataTypes.String),
new KustoValue("Order", b.Raw.Order.ToString(), KustoDataTypes.Int),
new KustoValue("Path", b.AugmentedOrder, KustoDataTypes.String),
new KustoValue("ParentId", b.Raw.ParentId, KustoDataTypes.String),
new KustoValue("Name", b.Raw.Name, KustoDataTypes.String),
new KustoValue("StartTime", b.Raw.StartTime, KustoDataTypes.DateTime),
new KustoValue("FinishTime", b.Raw.FinishTime, KustoDataTypes.DateTime),
new KustoValue("Result", b.Raw.Result, KustoDataTypes.String),
new KustoValue("ResultCode", b.Raw.ResultCode, KustoDataTypes.String),
new KustoValue("ChangeId", b.Raw.ChangeId.ToString(), KustoDataTypes.Int),
new KustoValue("LastModified", b.Raw.LastModified, KustoDataTypes.DateTime),
new KustoValue("WorkerName", b.Raw.WorkerName, KustoDataTypes.String),
new KustoValue("Details", b.Raw.Details?.Url, KustoDataTypes.String),
new KustoValue("ErrorCount", b.Raw.ErrorCount.ToString(), KustoDataTypes.Int),
new KustoValue("WarningCount", b.Raw.WarningCount.ToString(), KustoDataTypes.Int),
new KustoValue("Url", b.Raw.Url, KustoDataTypes.String),
new KustoValue("LogId", b.Raw.Log?.Id.ToString(), KustoDataTypes.Int),
new KustoValue("LogUri", b.Raw.Log?.Url, KustoDataTypes.String),
new KustoValue("TaskId", b.Raw.Task?.Id, KustoDataTypes.Int),
new KustoValue("TaskName", b.Raw.Task?.Name, KustoDataTypes.String),
new KustoValue("TaskVersion", b.Raw.Task?.Version, KustoDataTypes.String),
new KustoValue("Attempt", b.Raw.Attempt.ToString(), KustoDataTypes.Int),
});
_logger.LogInformation("Saving TimelineIssues...");
await KustoHelpers.WriteDataToKustoInMemoryAsync(
ingest,
options.KustoDatabase,
"TimelineIssues",
_logger,
issues,
b => new[]
{
new KustoValue("BuildId", b.BuildId.ToString(), KustoDataTypes.Int),
new KustoValue("RecordId", b.RecordId, KustoDataTypes.String),
new KustoValue("Index", b.Index.ToString(), KustoDataTypes.Int),
new KustoValue("Path", b.AugmentedIndex, KustoDataTypes.String),
new KustoValue("Type", b.Raw.Type, KustoDataTypes.String),
new KustoValue("Category", b.Raw.Category, KustoDataTypes.String),
new KustoValue("Message", b.Raw.Message, KustoDataTypes.String),
new KustoValue("Bucket", b.Bucket, KustoDataTypes.String),
});
}
