internal static async Task <BuildTestInfoCollection> ListBuildTestInfosAsync(this DotNetQueryUtil queryUtil, BuildSearchOptionSet optionSet, bool includeAllTests = false)
{
TestOutcome[]? outcomes = includeAllTests
? null
: new[] { TestOutcome.Failed };
var list = new List <BuildTestInfo>();
foreach (var build in await queryUtil.ListBuildsAsync(optionSet).ConfigureAwait(false))
{
try
{
var collection = await DotNetUtil.ListDotNetTestRunsAsync(queryUtil.Server, build, outcomes);
var buildTestInfo = new BuildTestInfo(build, collection.SelectMany(x => x.TestCaseResults).ToList());
list.Add(buildTestInfo);
}
catch (Exception ex)
{
Console.WriteLine($"Cannot get test info for {build.Id} {DevOpsUtil.GetBuildUri(build)}");
Console.WriteLine(ex.Message);
}
}
return(new BuildTestInfoCollection(new ReadOnlyCollection <BuildTestInfo>(list)));
}
public async Task <List <NGenAssemblyData> > GetNGenAssemblyDataAsync(Build build)
{
VerifyBuild(build);
var uri = DevOpsUtil.GetBuildUri(build);
Logger.LogInformation($"Processing {build.Id} - {uri}");
// Newer builds have the NGEN logs in a separate artifact altogether to decrease the time needed
// to downloaad them. Try that first and fall back to the diagnostic logs if it doesn't exist.
MemoryStream stream;
Func <ZipArchiveEntry, bool> predicate;
try
{
Logger.LogInformation("Downloading NGEN logs");
stream = await DevOpsServer.DownloadArtifactAsync(ProjectName, build.Id, "NGen Logs");
predicate = e => !string.IsNullOrEmpty(e.Name);
}
catch (Exception)
{
Logger.LogInformation("Falling back to diagnostic logs");
stream = await DevOpsServer.DownloadArtifactAsync(ProjectName, build.Id, "Build Diagnostic Files");
predicate = e => !string.IsNullOrEmpty(e.Name) && e.FullName.StartsWith("Build Diagnostic Files/ngen/");
}
return(await GetFromStream(stream, predicate));
}
private async Task DoSearchTimeline(Build build, IEnumerable <ModelTimelineQuery> timelineQueries)
{
Logger.LogInformation($"Searching {DevOpsUtil.GetBuildUri(build)}");
var buildKey = build.GetBuildKey();
Timeline timeline;
try
{
timeline = await Server.GetTimelineAsync(build);
if (timeline is null)
{
Logger.LogWarning("Error: No timeline");
return;
}
}
catch (Exception ex)
{
Logger.LogWarning($"Error getting timeline: {ex.Message}");
return;
}
var buildInfo = build.GetBuildInfo();
var modelBuild = TriageContextUtil.EnsureBuild(buildInfo);
foreach (var modelTimelineQuery in timelineQueries)
{
DoSearchTimeline(build, timeline, modelBuild, modelTimelineQuery);
}
}
public void GetTargetBranch(int buildNumber, string targetBranch)
{
var json = ResourceUtil.GetJsonFile("build-list2.json");
var builds = AzureJsonUtil.GetArray <Build>(json);
var build = builds.Single(x => x.Id == buildNumber);
Assert.Equal(targetBranch, DevOpsUtil.GetTargetBranch(build));
}
private async Task UploadBuild(Build build)
{
try
{
var uri = DevOpsUtil.GetBuildUri(build);
if (build.Status != BuildStatus.Completed)
{
Logger.LogInformation($"Build is not completed {uri}");
return;
}
if (await IsBuildUploadedAsync(build.Id))
{
Logger.LogInformation($"Build already uploaded {uri}");
return;
}
Logger.LogInformation($"Getting timeline {uri}");
var jobs = await GetJobCloneTimesAsync(build);
if (jobs.Count == 0)
{
Logger.LogInformation("Found no jobs");
return;
}
Logger.LogInformation($"Uploading {uri}");
if (build.StartTime is null)
{
Logger.LogError("Found no start time");
return;
}
var buildStartTime = DateTimeOffset.Parse(build.StartTime);
await DotNetUtil.DoWithTransactionAsync(SqlConnection, $"Upload Clone {build.Id}", async transaction =>
{
foreach (var job in jobs)
{
await UploadJobCloneTime(transaction, build.Id, build.Definition.Id, buildStartTime, uri, job);
}
var minDuration = jobs.Min(x => x.Duration);
var maxDuration = jobs.Max(x => x.Duration);
var totalFetchSize = jobs.Sum(x => x.FetchSize);
var minFetchSpeed = jobs.Min(x => x.MinFetchSpeed);
var maxFetchSpeed = jobs.Max(x => x.MaxFetchSpeed);
await UploadBuildCloneTime(transaction, build.Id, build.Definition.Id, minDuration, maxDuration, buildStartTime, uri, totalFetchSize, minFetchSpeed, maxFetchSpeed);
});
Logger.LogInformation("Build upload complete");
}
catch (Exception ex)
{
Logger.LogError($"Error {ex.Message}");
throw;
}
}
public void GetOrganizationTests(string url, string organization)
{
var build = new Build()
{
Url = url,
};
Assert.Equal(organization, DevOpsUtil.GetOrganization(build));
}
public TestResultsDisplay(IEnumerable <ModelTestResult> modelTestResults)
{
var anyHelix = false;
string?gitHubRepository = null;
var emptyDictionary = new Dictionary <string, string>();
foreach (var modelTestResult in modelTestResults)
{
anyHelix = anyHelix || modelTestResult.IsHelixTestResult;
gitHubRepository ??= modelTestResult.ModelBuild.GitHubRepository;
var routeData = new Dictionary <string, string>();
if (BuildsRequest is object)
{
routeData["bq"] = BuildsRequest.GetQueryString();
}
var request = new SearchTestsRequest()
{
Name = modelTestResult.TestFullName
};
routeData["tq"] = request.GetQueryString();
var testResultInfo = new TestResultInfo()
{
BuildNumber = modelTestResult.ModelBuild.BuildNumber,
BuildUri = DevOpsUtil.GetBuildUri(modelTestResult.ModelBuild.AzureOrganization, modelTestResult.ModelBuild.AzureProject, modelTestResult.ModelBuild.BuildNumber),
Kind = modelTestResult.ModelBuild.GetModelBuildKind().GetDisplayString(),
TestRun = modelTestResult.ModelTestRun.Name,
TestFullName = modelTestResult.TestFullName,
TestFullNameRouteData = routeData,
ErrorMessage = modelTestResult.ErrorMessage,
HelixConsoleUri = modelTestResult.HelixConsoleUri,
HelixRunClientUri = modelTestResult.HelixRunClientUri,
HelixCoreDumpUri = modelTestResult.HelixCoreDumpUri,
HelixTestResultsUri = modelTestResult.HelixTestResultsUri,
};
Results.Add(testResultInfo);
}
IncludeHelixColumns = anyHelix;
GitHubRepository = gitHubRepository;
}
static int Compare(TimelineNode x, TimelineNode y)
{
var xStart = DevOpsUtil.ConvertRestTime(x.TimelineRecord.StartTime);
var yStart = DevOpsUtil.ConvertRestTime(y.TimelineRecord.StartTime);
if (xStart is null)
{
if (yStart is null)
{
return(0);
}
return(-1);
}
if (yStart is null)
{
return(1);
}
return(xStart.Value.CompareTo(yStart.Value));
}
private static async Task DumpTestTimes()
{
using var util = new RunTestsUtil(await GetToken("scratch-db"));
foreach (var build in (await util.ListBuildsAsync(top: 20)).Where(x => x.Result == BuildResult.Succeeded))
{
Console.Write(DevOpsUtil.GetBuildUri(build));
Console.Write(" ");
try
{
var buildTestTime = await util.GetBuildTestTimeAsync(build);
var milliseconds = buildTestTime.Jobs.Sum(x => x.Duration.TotalMilliseconds);
Console.Write(TimeSpan.FromMilliseconds(milliseconds));
Console.Write(" ");
var max = buildTestTime.Jobs.Max(x => x.Duration.TotalMilliseconds);
Console.WriteLine(TimeSpan.FromMilliseconds(max));
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
await util.UpdateDatabaseAsync(top : 100);
}
public async Task UploadBuild(Build build)
{
VerifyBuild(build);
var uri = DevOpsUtil.GetBuildUri(build).ToString();
Logger.LogInformation($"Uploading {build.Id} - {uri}");
if (await IsBuildUploadedAsync(build.Id))
{
Logger.LogInformation("Build already uploaded");
return;
}
var branchName = DotNetUtil.NormalizeBranchName(build.SourceBranch);
var list = await GetNGenAssemblyDataAsync(build);
await DotNetUtil.DoWithTransactionAsync(SqlConnection, $"Uploading {build.Id}", async transaction =>
{
foreach (var ngenAssemblyData in list)
{
await UploadNGenAssemblyDataAsync(transaction, build.Id, branchName, uri, ngenAssemblyData);
}
});
}
public async Task <IActionResult> OnGet()
{
if (!(Number is { } number))
{
return(Page());
}
var buildKey = GetBuildKey(number);
var project = buildKey.Project;
var organization = buildKey.Organization;
var buildId = TriageContextUtil.GetModelBuildId(buildKey);
var modelBuild = await PopulateBuildInfo();
await PopulateTimeline();
await PopulateTests();
return(Page());
async Task <ModelBuild?> PopulateBuildInfo()
{
var modelBuild = await TriageContextUtil
.GetModelBuildQuery(buildKey)
.Include(x => x.ModelGitHubIssues)
.FirstOrDefaultAsync();
if (modelBuild is null)
{
return(null);
}
BuildUri = DevOpsUtil.GetBuildUri(organization, project, number);
BuildResult = modelBuild.BuildResult ?? BuildResult.None;
Repository = $"{modelBuild.GitHubOrganization}/{modelBuild.GitHubRepository}";
RepositoryUri = $"https://{modelBuild.GitHubOrganization}/{modelBuild.GitHubRepository}";
DefinitionName = modelBuild.DefinitionName;
TargetBranch = modelBuild.GitHubTargetBranch;
GitHubIssues.Clear();
GitHubIssues.AddRange(modelBuild.ModelGitHubIssues.Select(x => x.GetGitHubIssueKey()));
if (modelBuild.PullRequestNumber is { } prNumber)
{
PullRequestKey = new GitHubPullRequestKey(
modelBuild.GitHubOrganization,
modelBuild.GitHubRepository,
prNumber);
}
return(modelBuild);
}
async Task PopulateTimeline()
{
var query = TriageContextUtil
.Context
.ModelTimelineIssues
.Where(x => x.ModelBuildId == buildId)
.Include(x => x.ModelBuild);
TimelineIssuesDisplay = await TimelineIssuesDisplay.Create(
query,
includeBuildColumn : false,
includeIssueTypeColumn : true,
includeAttemptColumn : true);
Attempts = TimelineIssuesDisplay.Issues.Count > 0
? TimelineIssuesDisplay.Issues.Max(x => x.Attempt)
: 1;
}
async Task PopulateTests()
{
var query = TriageContextUtil
.Context
.ModelTestResults
.Where(x => x.ModelBuildId == buildId)
.Include(x => x.ModelTestRun)
.Include(x => x.ModelBuild);
var modelTestResults = await query.ToListAsync();
TestResultsDisplay = new TestResultsDisplay(modelTestResults)
{
IncludeBuildColumn = false,
IncludeBuildKindColumn = false,
IncludeTestFullNameColumn = true,
IncludeTestFullNameLinks = true,
IncludeErrorMessageColumn = true,
};
if (modelBuild is object)
{
TestResultsDisplay.BuildsRequest = new SearchBuildsRequest()
{
Definition = modelBuild.DefinitionName,
Started = new DateRequestValue(dayQuery: 7)
};
}
}
}
private static async Task ListStaleChecks()
{
var gitHub = new GitHubClient(new ProductHeaderValue("MyAmazingApp"));
gitHub.Credentials = new Credentials(await GetToken("github"));
var apiConnection = new ApiConnection(gitHub.Connection);
var checksClient = new ChecksClient(apiConnection);
var server = new DevOpsServer("dnceng");
var list = new List <string>();
foreach (var build in await server.ListBuildsAsync("public", new[] { 196 }, top: 500))
{
if (build.Status == BuildStatus.Completed &&
build.Reason == BuildReason.PullRequest &&
build.FinishTime is object &&
DateTimeOffset.UtcNow - DateTimeOffset.Parse(build.FinishTime) > TimeSpan.FromMinutes(5))
{
try
{
Console.WriteLine($"Checking {build.Repository.Id} {build.SourceVersion}");
// Build is complete for at least five minutes. Results should be available
var name = build.Repository.Id.Split("/");
var pullRequestId = int.Parse(build.SourceBranch.Split("/")[2]);
var prUri = $"https://github.com/{build.Repository.Id}/pull/{pullRequestId}";
var repository = await gitHub.Repository.Get(name[0], name[1]);
var pullRequest = await gitHub.PullRequest.Get(repository.Id, pullRequestId);
if (pullRequest.MergeableState.Value == MergeableState.Dirty ||
pullRequest.MergeableState.Value == MergeableState.Unknown)
{
// There are merge conflicts. This seems to confuse things a bit below.
continue;
}
// Need to use the HEAD of the PR not Build.SourceVersion here. The Build.SourceVersion
// is the HEAD of the PR merged into HEAD of the target branch. The check suites only track
// the HEAD of PR
var response = await checksClient.Suite.GetAllForReference(repository.Id, pullRequest.Head.Sha);
var devOpsResponses = response.CheckSuites.Where(x => x.App.Name == "Azure Pipelines").ToList();
var allDone = devOpsResponses.All(x => x.Status.Value == CheckStatus.Completed);
if (!allDone)
{
// There are merge conflicts. This seems to confuse things a bit below.
Console.WriteLine($"\t{DevOpsUtil.GetBuildUri(build)}");
Console.WriteLine($"\t{prUri}");
list.Add(prUri);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
foreach (var uri in list.ToHashSet())
{
Console.WriteLine(uri);
}
}
internal async Task <int> PrintSearchHelix(IEnumerable <string> args)
{
string text = null;
var optionSet = new BuildSearchOptionSet()
{
{ "v|value=", "text to search for", t => text = t },
};
ParseAll(optionSet, args);
if (text is null)
{
Console.WriteLine("Must provide a text argument to search for");
optionSet.WriteOptionDescriptions(Console.Out);
return(ExitFailure);
}
var badLogList = new List <string>();
var textRegex = new Regex(text, RegexOptions.Compiled | RegexOptions.IgnoreCase);
var collection = await QueryUtil.ListBuildTestInfosAsync(optionSet);
var found = collection
.AsParallel()
.Select(async b => (b.Build, await SearchBuild(b)));
Console.WriteLine("|Build|Kind|Console Log|");
Console.WriteLine("|---|---|---|");
foreach (var task in found)
{
var(build, helixLogInfo) = await task;
if (helixLogInfo is null)
{
continue;
}
var kind = "Rolling";
if (DevOpsUtil.TryGetPullRequestKey(build, out var pullRequestKey))
{
kind = $"PR {pullRequestKey.PullRequestUri}";
}
Console.WriteLine($"|[{build.Id}]({DevOpsUtil.GetBuildUri(build)})|{kind}|[console.log]({helixLogInfo.ConsoleUri})|");
}
foreach (var line in badLogList)
{
Console.WriteLine(line);
}
return(ExitSuccess);
async Task <HelixLogInfo> SearchBuild(BuildTestInfo buildTestInfo)
{
var build = buildTestInfo.Build;
foreach (var workItem in buildTestInfo.GetHelixWorkItems())
{
try
{
var logInfo = await GetHelixLogInfoAsync(workItem);
if (logInfo.ConsoleUri is object)
{
using var stream = await Server.DownloadFileAsync(logInfo.ConsoleUri);
if (IsMatch(stream, textRegex))
{
return(logInfo);
}
}
}
catch
{
badLogList.Add($"Unable to download helix logs for {build.Id} {workItem.HelixInfo.JobId}");
}
}
return(null);
}
public async Task <ModelBuildAttempt> EnsureBuildAttemptAsync(ModelBuild modelBuild, BuildResult buildResult, Timeline timeline)
{
var attempt = timeline.GetAttempt();
var modelBuildAttempt = await Context.ModelBuildAttempts
.Where(x => x.ModelBuildId == modelBuild.Id && x.Attempt == attempt)
.FirstOrDefaultAsync().ConfigureAwait(false);
if (modelBuildAttempt is object && !modelBuildAttempt.IsTimelineMissing)
{
return(modelBuildAttempt);
}
var startTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.StartTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var startTime = startTimeQuery.Any()
? startTimeQuery.Min()
: modelBuild.StartTime;
var finishTimeQuery = timeline
.Records
.Where(x => x.Attempt == attempt)
.Select(x => DevOpsUtil.ConvertFromRestTime(x.FinishTime))
.SelectNullableValue()
.Select(x => (DateTime?)x.DateTime);
var finishTime = finishTimeQuery.Any()
? finishTimeQuery.Max()
: modelBuild.FinishTime;
if (modelBuildAttempt is object)
{
modelBuildAttempt.BuildResult = buildResult;
modelBuildAttempt.StartTime = startTime;
modelBuildAttempt.FinishTime = finishTime;
modelBuildAttempt.IsTimelineMissing = false;
}
else
{
modelBuildAttempt = new ModelBuildAttempt()
{
Attempt = attempt,
BuildResult = buildResult,
StartTime = startTime,
FinishTime = finishTime,
ModelBuild = modelBuild,
IsTimelineMissing = false,
};
Context.ModelBuildAttempts.Add(modelBuildAttempt);
}
var timelineTree = TimelineTree.Create(timeline);
foreach (var record in timeline.Records)
{
if (record.Issues is null ||
!timelineTree.TryGetJob(record, out var job))
{
continue;
}
foreach (var issue in record.Issues)
{
var timelineIssue = new ModelTimelineIssue()
{
Attempt = attempt,
JobName = job.Name,
RecordName = record.Name,
TaskName = record.Task?.Name ?? "",
RecordId = record.Id,
Message = issue.Message,
ModelBuild = modelBuild,
IssueType = issue.Type,
ModelBuildAttempt = modelBuildAttempt,
};
Context.ModelTimelineIssues.Add(timelineIssue);
}
}
await Context.SaveChangesAsync().ConfigureAwait(false);
return(modelBuildAttempt);
}
public async Task OnGetAsync(int id, int pageNumber = 0)
{
var issue = await Context.ModelTrackingIssues
.Where(x => x.Id == id)
.Include(x => x.ModelBuildDefinition)
.SingleAsync();
ModelTrackingIssueId = id;
IssueTitle = issue.IssueTitle;
SearchQuery = issue.SearchQuery;
TrackingKind = issue.TrackingKind.ToString();
GitHubIssueUri = issue.GetGitHubIssueKey()?.IssueUri;
IsActive = issue.IsActive;
Definition = issue.ModelBuildDefinition?.DefinitionName;
const int pageSize = 10;
var totalPages = await Context.ModelTrackingIssueMatches
.Where(x => x.ModelTrackingIssueId == issue.Id)
.CountAsync() / pageSize;
PaginationDisplay = new PaginationDisplay(
"/Tracking/Issue",
new Dictionary <string, string>()
{
{ nameof(id), id.ToString() }
},
pageNumber,
totalPages);
var dateTimeUtil = new DateTimeUtil();
Results = await Context.ModelTrackingIssueMatches
.Where(x => x.ModelTrackingIssueId == issue.Id)
.Include(x => x.ModelBuildAttempt)
.ThenInclude(x => x.ModelBuild)
.OrderByDescending(x => x.ModelBuildAttempt.ModelBuild.BuildNumber)
.Skip(pageNumber * pageSize)
.Take(pageSize)
.Select(x => new Result()
{
BuildNumber = x.ModelBuildAttempt.ModelBuild.BuildNumber,
BuildUri = DevOpsUtil.GetBuildUri(x.ModelBuildAttempt.ModelBuild.AzureOrganization, x.ModelBuildAttempt.ModelBuild.AzureProject, x.ModelBuildAttempt.ModelBuild.BuildNumber),
BuildKind = x.ModelBuildAttempt.ModelBuild.PullRequestNumber is object? "Pull Request" : "Rolling",
JobName = x.JobName,
Attempt = x.ModelBuildAttempt.Attempt,
RepositoryName = x.ModelBuildAttempt.ModelBuild.GitHubRepository,
RepositoryUri = x.ModelBuildAttempt.ModelBuild.GitHubRepository is object
?$"https://github.com/{x.ModelBuildAttempt.ModelBuild.GitHubOrganization}/{x.ModelBuildAttempt.ModelBuild.GitHubRepository}"
: "",
Queued = dateTimeUtil.ConvertDateTime(x.ModelBuildAttempt.ModelBuild.QueueTime),
})
.AsNoTracking()
.ToListAsync();
var now = dateTimeUtil.Now;
HitCount = new HitCountInfo()
{
Today = await GetHitCount(now - TimeSpan.FromDays(1)),
Week = await GetHitCount(now - TimeSpan.FromDays(7)),
Month = await GetHitCount(now - TimeSpan.FromDays(30)),
};
async Task <int> GetHitCount(DateTime before) => await Context
.ModelTrackingIssueResults
.Where(x => x.ModelTrackingIssueId == ModelTrackingIssueId && x.IsPresent && x.ModelBuildAttempt.ModelBuild.QueueTime > before)
.CountAsync();
}
