public void Roots()
{
var timeline = ResourceUtil.GetTimeline("timeline1.json");
var tree = TimelineTree.Create(timeline);
Assert.Single(tree.Roots);
}
public void Jobs()
{
var timeline = ResourceUtil.GetTimeline("timeline1.json");
var tree = TimelineTree.Create(timeline);
var jobs = tree.Nodes.Where(x => tree.IsJob(x.TimelineRecord.Id));
Assert.Equal(142, jobs.Count());
}
public TimelineRecordItem(
TimelineRecord record,
TimelineTree tree)
{
Debug.Assert(tree.TryGetNode(record.Id, out _));
Record = record;
Tree = tree;
}
public void TryGetJob()
{
var timeline = ResourceUtil.GetTimeline("timeline1.json");
var tree = TimelineTree.Create(timeline);
Assert.True(tree.TryGetNode("dfefcd06-03ef-5951-c8ec-02f90019bee7", out var node));
Assert.True(tree.TryGetJob(node !.TimelineRecord, out var job));
Assert.Equal("CoreCLR Common Pri0 Test Build Windows_NT arm64 checked", job !.Name);
}
public async Task <List <TimelineRecord> > ListFailedJobs(Build build)
{
var timeline = await Server.GetTimelineAsync(build).ConfigureAwait(false);
if (timeline is null)
{
return(new List <TimelineRecord>());
}
var timelineTree = TimelineTree.Create(timeline);
return(timelineTree
.Nodes
.Where(n => !n.TimelineRecord.IsAnySuccess() && timelineTree.IsJob(n.TimelineRecord.Id))
.Select(x => x.TimelineRecord)
.ToList());
}
public IEnumerable <SearchTimelineResult> SearchTimeline(
BuildResultInfo buildInfo,
Timeline timeline,
Regex text,
Regex?name = null,
Regex?task = null,
int?attempt = null)
{
var timelineTree = TimelineTree.Create(timeline);
var records = timeline.Records
.Where(r => name is null || name.IsMatch(r.Name))
.Where(r => r.Task is null || task is null || task.IsMatch(r.Task.Name));
foreach (var record in records)
{
if (record.Issues is null)
{
continue;
}
string?line = null;
foreach (var issue in record.Issues)
{
if (text.IsMatch(issue.Message))
{
line = issue.Message;
break;
}
}
if (line is object)
{
yield return(new SearchTimelineResult(
new TimelineRecordItem(record, timelineTree),
buildInfo,
line));
}
}
}
internal static TimelineTree Create(Timeline timeline)
{
var records = timeline.Records;
var map = new Dictionary <string, TimelineNode>();
// Each stage will have a different root
var roots = new List <TimelineNode>();
foreach (var record in records)
{
var node = GetOrCreateNode(record.Id);
node.TimelineRecord = record;
if (string.IsNullOrEmpty(record.ParentId))
{
roots.Add(node);
}
else
{
var parentNode = GetOrCreateNode(record.ParentId);
parentNode.Children.Add(node);
node.ParentNode = parentNode;
}
}
// Now look for hidden roots
foreach (var value in map.Values)
{
if (value.ParentNode is null && !roots.Contains(value))
{
roots.Add(value);
}
}
// TODO sort by start time, not name. The tree should reflect execution order
var comparer = StringComparer.OrdinalIgnoreCase;
foreach (var value in map.Values)
{
value.Children.Sort(Compare);
}
roots.Sort(Compare);
var tree = new TimelineTree(timeline, roots);
Debug.Assert(tree.Count == timeline.Records.Length);
return(tree);
TimelineNode GetOrCreateNode(string id)
{
TimelineNode node;
if (!map.TryGetValue(id, out node))
{
node = new TimelineNode();
map.Add(id, node);
}
return(node);
}
public async Task <ModelBuildAttempt> EnsureBuildAttemptAsync(ModelBuild modelBuild, BuildResult buildResult, Timeline timeline)
{
var attempt = timeline.GetAttempt();
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object && !modelBuildAttempt.IsTimelineMissing)
{
return(modelBuildAttempt);
}
var startTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.StartTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var startTime = startTimeQuery.Any()
? startTimeQuery.Min()
: modelBuild.StartTime;
var finishTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.FinishTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var finishTime = finishTimeQuery.Any()
? finishTimeQuery.Max()
: modelBuild.FinishTime;
if (modelBuildAttempt is object)
{
modelBuildAttempt.BuildResult = buildResult;
modelBuildAttempt.StartTime = startTime;
modelBuildAttempt.FinishTime = finishTime;
modelBuildAttempt.IsTimelineMissing = false;
}
else
{
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = buildResult,
StartTime = startTime,
FinishTime = finishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
}
var timelineTree = TimelineTree.Create(timeline);
foreach (var record in timeline.Records)
{
if (record.Issues is null ||
!timelineTree.TryGetJob(record, out var job))
{
continue;
}
foreach (var issue in record.Issues)
{
var timelineIssue = new ModelTimelineIssue()
{
Attempt = attempt,
JobName = job.Name,
RecordName = record.Name,
TaskName = record.Task?.Name ?? "",
RecordId = record.Id,
Message = issue.Message,
ModelBuild = modelBuild,
IssueType = issue.Type,
ModelBuildAttempt = modelBuildAttempt,
};
Context.ModelTimelineIssues.Add(timelineIssue);
}
}
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
public async Task <List <SearchBuildLogsResult> > SearchBuildLogsAsync(
IEnumerable <BuildResultInfo> builds,
SearchBuildLogsRequest request,
Action <Exception>?onError = null)
{
if (request.Text is null)
{
throw new ArgumentException("Need text to search for", nameof(request));
}
var nameRegex = CreateSearchRegex(request.LogName);
var textRegex = CreateSearchRegex(request.Text);
var list = new List <(BuildResultInfo BuildInfo, TimelineTree Tree, TimelineRecord TimelineRecord, BuildLogReference BuildLogReference)>();
foreach (var buildInfo in builds)
{
try
{
var timeline = await Server.GetTimelineAsync(buildInfo.Project, buildInfo.Number).ConfigureAwait(false);
if (timeline is object)
{
var tree = TimelineTree.Create(timeline);
var records = timeline.Records.Where(r => nameRegex is null || nameRegex.IsMatch(r.Name));
foreach (var record in records)
{
if (record.Log is { } log)
{
list.Add((buildInfo, tree, record, log));
}
}
}
}
catch (Exception ex)
{
onError?.Invoke(ex);
}
}
if (list.Count > request.Limit)
{
onError?.Invoke(new Exception($"Limiting the {list.Count} logs to first {request.Limit}"));
list = list.Take(request.Limit).ToList();
}
var resultTasks = list
.AsParallel()
.Select(async x =>
{
var match = await SearchFileForFirstMatchAsync(x.BuildLogReference.Url, textRegex, onError).ConfigureAwait(false);
var line = match is object && match.Success
? match.Value
: null;
return(Query: x, Line: line);
});
var results = new List <SearchBuildLogsResult>();
foreach (var task in resultTasks)
{
try
{
var result = await task.ConfigureAwait(false);
if (result.Line is object)
{
string jobName = "";
if (result.Query.Tree.TryGetJob(result.Query.TimelineRecord, out var jobRecord))
{
jobName = jobRecord.Name;
}
results.Add(new SearchBuildLogsResult(result.Query.BuildInfo, jobName, result.Query.TimelineRecord, result.Query.BuildLogReference, result.Line));
}
}
catch (Exception ex)
{
onError?.Invoke(ex);
}
}
return(results);
}
