public async Task <List <NGenAssemblyData> > GetNGenAssemblyDataAsync(Build build)
{
VerifyBuild(build);
var uri = DevOpsUtil.GetBuildUri(build);
Logger.LogInformation($"Processing {build.Id} - {uri}");
// Newer builds have the NGEN logs in a separate artifact altogether to decrease the time needed
// to downloaad them. Try that first and fall back to the diagnostic logs if it doesn't exist.
MemoryStream stream;
Func <ZipArchiveEntry, bool> predicate;
try
{
Logger.LogInformation("Downloading NGEN logs");
stream = await DevOpsServer.DownloadArtifactAsync(ProjectName, build.Id, "NGen Logs");
predicate = e => !string.IsNullOrEmpty(e.Name);
}
catch (Exception)
{
Logger.LogInformation("Falling back to diagnostic logs");
stream = await DevOpsServer.DownloadArtifactAsync(ProjectName, build.Id, "Build Diagnostic Files");
predicate = e => !string.IsNullOrEmpty(e.Name) && e.FullName.StartsWith("Build Diagnostic Files/ngen/");
}
return(await GetFromStream(stream, predicate));
}
public async Task <List <JobCloneTime> > GetJobCloneTimesAsync(Build build)
{
var list = new List <JobCloneTime>();
var timeline = await DevOpsServer.GetTimelineAsync(ProjectName, build.Id);
if (timeline is null)
{
return(list);
}
foreach (var record in timeline.Records.Where(x => x.Name == "Checkout" && x.FinishTime is object && x.StartTime is object))
{
var duration = DateTime.Parse(record.FinishTime) - DateTime.Parse(record.StartTime);
var startTime = DateTimeOffset.Parse(record.StartTime);
var parent = timeline.Records.Single(x => x.Id == record.ParentId);
var(fetchSize, minFetchSpeed, maxFetchSpeed, averageFetchSpeed) = await GetSizesAsync(record);
var jobCloneTime = new JobCloneTime(
parent.Name,
startTime,
duration,
fetchSize,
minFetchSpeed: minFetchSpeed,
maxFetchSpeed: maxFetchSpeed,
averageFetchSpeed: averageFetchSpeed);
list.Add(jobCloneTime);
}
return(list);
}
public async IAsyncEnumerable <(PullRequest PullReuqest, Build Build)> EnumerateMergedPullRequestBuilds(
DevOpsServer server,
GitHubBuildInfo gitHubInfo,
string project,
int[]?definitions)
{
await foreach (var pullRequest in EnumerateClosedPullRequests(gitHubInfo.Organization, gitHubInfo.Repository).ConfigureAwait(false))
{
var prKey = new GitHubPullRequestKey(gitHubInfo.Organization, gitHubInfo.Repository, pullRequest.Number);
Build?build = null;
try
{
var builds = (await server.ListPullRequestBuildsAsync(prKey, project, definitions).ConfigureAwait(false))
.OrderByDescending(b => b.BuildNumber)
.Where(x => x.Status == BuildStatus.Completed && x.Result != BuildResult.Canceled)
.ToList();
if (builds.Count > 0)
{
build = builds[0];
}
}
catch (Exception)
{
// Error enumerating builds, continue to the next one
}
if (build is object)
{
yield return(pullRequest, build);
}
}
}
private static async Task DumpTestTimesCsv()
{
var server = new DevOpsServer("dnceng", await GetToken("dnceng"));
var all = await server.ListTestRunsAsync("public", 585853);
var debug = all.Where(x => x.Name == "Windows Desktop Debug Test32").First();
var spanish = all.Where(x => x.Name == "Windows Desktop Spanish").First();
await Write(debug, @"p:\temp\data-debug-class.csv");
await Write(spanish, @"p:\temp\data-spanish-class.csv");
async Task Write(TestRun testRun, string filePath)
{
var testCases = await server.ListTestResultsAsync("public", testRun.Id);
var sorted = testCases
.Select(x => (ClassName: GetClassName(x), Duration: x.DurationInMs))
.GroupBy(x => x.ClassName)
.OrderBy(x => x.Key);
var builder = new StringBuilder();
foreach (var group in sorted)
{
var time = TimeSpan.FromMilliseconds(group.Sum(x => x.Duration));
builder.AppendLine($"{group.Key},{time}");
}
File.WriteAllText(filePath, builder.ToString());
}
public async Task GetAttemptOneMultiPatch()
{
var handler = new TestableHttpMessageHandler();
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/799195/timeline?api-version=5.0",
ResourceUtil.GetJsonFile("timeline2.json"));
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/799195/timeline/5c657fa2-65d2-55e4-f9c9-8fbd97185e37?api-version=5.0",
ResourceUtil.GetJsonFile("timeline2-part1.json"));
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/799195/timeline/0254bc91-8160-5748-ab06-8ddd0ae17493?api-version=5.0",
ResourceUtil.GetJsonFile("timeline2-part2.json"));
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/799195/timeline/b02ddad0-c847-5898-a3e9-29c2612003f0?api-version=5.0",
ResourceUtil.GetJsonFile("timeline2-part3.json"));
var server = new DevOpsServer("dnceng", httpClient: new HttpClient(handler));
var timeline = await server.GetTimelineAttemptAsync("public", 799195, attempt : 1);
Assert.NotNull(timeline);
var timelineTree = TimelineTree.Create(timeline !);
Assert.NotNull(timelineTree);
}
public Functions(DevOpsServer server, GitHubClient gitHubClient, TriageContext context)
{
Server = server;
GitHubClient = gitHubClient;
Context = context;
TriageContextUtil = new TriageContextUtil(context);
}
private static async Task ListBuildsFullAsync()
{
var server = new DevOpsServer("dnceng");
var builds1 = await server.ListBuildsAsync("public", new[] { 15 });
var builds2 = await server.ListBuildsAsync("public", top : 10);
}
public async Task OnPullRequestMergedAsync(
DevOpsServer server,
TriageContextUtil triageContextUtil,
GitHubPullRequestKey prKey,
string project,
CancellationToken cancellationToken = default)
{
// Pull requests can trigger builds in multiple definitions. Need to calculate the merged PR build
// for each of them
var allBuilds = await server.ListPullRequestBuildsAsync(prKey, project).ConfigureAwait(false);
foreach (var group in allBuilds.GroupBy(x => x.Definition.Id))
{
var mergedBuild = group
.Where(x => x.Status == BuildStatus.Completed && x.Result != BuildResult.Canceled)
.OrderByDescending(x => x.Id)
.FirstOrDefault();
if (mergedBuild is object)
{
var modelBuild = await triageContextUtil.EnsureBuildAsync(mergedBuild.GetBuildResultInfo()).ConfigureAwait(false);
modelBuild.IsMergedPullRequest = true;
await triageContextUtil.Context.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
}
}
}
private static async Task Scratch2()
{
var server = new DevOpsServer("dnceng", await GetToken("dnceng"));
var builds = await server.ListBuildsAsync("public", definitions : new[] { 686 }, top : 3000);
var buildTimes = new List <(int BuildNumber, DateTime StartTime, DateTime EndTime)>();
GetBuildTimes();
var firstDay = buildTimes.Min(x => x.StartTime).Date;
var lastDay = DateTime.UtcNow.Date;
var maxCapacity = GetCapacity();
Console.WriteLine($"Max capacity is {maxCapacity}");
void GetBuildTimes()
{
using var writer = new StreamWriter(@"p:\temp\builds.csv", append: false);
writer.WriteLine("Build Number,Start Time, End Time");
foreach (var build in builds)
{
if (build.FinishTime is null)
{
continue;
}
if (!DateTime.TryParse(build.StartTime, out var startTime) ||
!DateTime.TryParse(build.FinishTime, out var endTime))
{
continue;
}
writer.WriteLine($"{build.Id},{build.StartTime},{build.FinishTime}");
buildTimes.Add((build.Id, startTime, endTime));
}
}
int GetCapacity()
{
using var writer = new StreamWriter(@"p:\temp\capacity.csv", append: false);
writer.WriteLine("Time,Build Count");
var current = firstDay;
var max = 0;
while (current.Date <= lastDay)
{
var count = buildTimes.Count(x => current >= x.StartTime && current <= x.EndTime);
if (count > max)
{
max = count;
}
writer.WriteLine($"{current},{count}");
current = current.AddMinutes(15);
}
return(max);
}
}
private DotNetQueryUtil CreateForServer(DevOpsServer server)
{
// https://github.com/jaredpar/devops-util/issues/19
// Consider using a cache here
var azureUtil = new AzureUtil(server);
return(new DotNetQueryUtil(server, azureUtil));
}
static async Task <DotNetTestRun> GetDotNetTestRunAsync(DevOpsServer server, Build build, TestRun testRun, TestOutcome[] outcomes)
{
var all = await server.ListTestResultsAsync(build.Project.Name, testRun.Id, outcomes : outcomes).ConfigureAwait(false);
var info = new TestRunInfo(build, testRun);
var list = ToDotNetTestCaseResult(info, all.ToList());
return(new DotNetTestRun(info, new ReadOnlyCollection <DotNetTestCaseResult>(list)));
}
public BuildRetryUtil(
DevOpsServer server,
TriageContext context,
ILogger logger)
{
Server = server;
TriageContextUtil = new TriageContextUtil(context);
Logger = logger;
}
public Functions(DevOpsServer server, TriageContext context, GitHubClientFactory gitHubClientFactory)
{
Server = server;
Context = context;
TriageContextUtil = new TriageContextUtil(context);
GitHubClientFactory = gitHubClientFactory;
SiteLinkUtil = SiteLinkUtil.Published;
HelixServer = new HelixServer();
}
private async Task <(double?FetchSize, double?MinFetchSpeed, double?MaxFetchSpeed, double?AverageFetchSpeed)> GetSizesAsync(TimelineRecord record)
{
try
{
using var stream = await DevOpsServer.DownloadFileAsync(record.Log.Url);
using var reader = new StreamReader(stream);
var sizeAtom = @"[\d.]+\s+[GMK]iB";
var regex = new Regex($@"Receiving objects:\s+\d+% .*,\s+({sizeAtom})\s+\|\s+({sizeAtom})/s(,\s*done)?", RegexOptions.IgnoreCase | RegexOptions.Compiled);
var fetchSpeeds = new List <double>();
double?fetchSize = null;
do
{
var line = await reader.ReadLineAsync();
if (line is null)
{
break;
}
var match = regex.Match(line);
if (match.Success)
{
var size = parseSize(match.Groups[1].Value);
var speed = parseSize(match.Groups[2].Value);
if (speed.HasValue)
{
fetchSpeeds.Add(speed.Value);
}
updateIfBigger(ref fetchSize, size);
if (!string.IsNullOrEmpty(match.Groups[3].Value))
{
break;
}
}
} while (true);
if (fetchSpeeds.Count == 0)
{
return(fetchSize, null, null, null);
}
return(
fetchSize,
fetchSpeeds.Min(),
fetchSpeeds.Max(),
fetchSpeeds.Average());
}
catch (Exception ex)
{
Logger.LogInformation($"Error parsing fetch times: {ex.Message}");
return(null, null, null, null);
}
public DotNetQueryUtil(DevOpsServer server, IAzureUtil?azureUtil = null)
{
if (azureUtil is object && server.Organization != azureUtil.Organization)
{
throw new ArgumentException();
}
Server = server;
AzureUtil = azureUtil ?? new AzureUtil(server);
}
internal RuntimeInfo(
DevOpsServer devopsServer,
HelixServer helixServer,
IAzureUtil azureUtil)
{
DevOpsServer = devopsServer;
HelixServer = helixServer;
AzureUtil = azureUtil;
QueryUtil = new DotNetQueryUtil(DevOpsServer, azureUtil);
}
public AutoTriageUtil(
DevOpsServer server,
GitHubClient gitHubClient,
TriageContext context,
ILogger logger)
{
Server = server;
GitHubClient = gitHubClient;
QueryUtil = new DotNetQueryUtil(server);
TriageContextUtil = new TriageContextUtil(context);
Logger = logger;
}
public async Task UpdateDatabaseAsync(int?top = null)
{
var builds = await DevOpsServer.ListBuildsAsync(ProjectName, top : top);
foreach (var build in builds)
{
try
{
await UploadBuild(build);
}
catch (Exception ex)
{
Logger.LogError($"Unable to upload {build.Id}: {ex.Message}");
}
}
}
/// <summary>
/// List the set of <see cref="TestCaseResult"/> for a given <see cref="TestRun"/>.
///
/// Ideally we could include sub results in the ListTestResultsAsync call because it's a supported parameter of
/// the ListTestResults AzDO REST API call. Unfortunately there is a bug right now where they are not respecting
/// that parameter.
///
/// As a fallback for now we simply iterate the results, look for ones that are likely theories, and then request the
/// results individually. This means we also limit to only failed outcomes bc otherwise we'd be making an additional
/// API call for every single result. That could be extremely expensive
/// </summary>
public static async Task <List <TestCaseResult> > ListTestResultsAsync(
this DevOpsServer server,
string project,
int testRunId,
TestOutcome[]?outcomes,
bool includeSubResults,
Action <Exception>?onError = null)
{
var testCaseResults = await server.ListTestResultsAsync(project, testRunId, outcomes : outcomes).ConfigureAwait(false);
if (includeSubResults)
{
var comparer = StringComparer.OrdinalIgnoreCase;
for (int i = 0; i < testCaseResults.Count; i++)
{
var testCaseResult = testCaseResults[i];
if (string.IsNullOrEmpty(testCaseResult.ErrorMessage) && IsFailedOutcome(testCaseResult.Outcome))
{
try
{
var otherResult = await server.GetTestCaseResultAsync(project, testRunId, testCaseResult.Id, ResultDetail.SubResults).ConfigureAwait(false);
if (otherResult.SubResults is object)
{
otherResult.SubResults = otherResult
.SubResults
.Where(x => IsFailedOutcome(x.Outcome))
.ToArray();
}
testCaseResults[i] = otherResult;
}
catch (Exception ex)
{
// If we can't fetch the sub-result then just continue
onError?.Invoke(ex);
}
}
bool IsFailedOutcome(string outcome) => DotNetUtil.FailedTestOutcomes.Any(x => comparer.Equals(x.ToString(), outcome));
}
}
return(testCaseResults);
}
public async Task GetAttemptOneMissingRecords()
{
var handler = new TestableHttpMessageHandler();
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/795146/timeline?api-version=5.0",
ResourceUtil.GetJsonFile("timeline3.json"));
handler.AddJson(
"https://dev.azure.com/dnceng/public/_apis/build/builds/795146/timeline/96ac2280-8cb4-5df5-99de-dd2da759617d?api-version=5.0",
ResourceUtil.GetJsonFile("timeline3-part1.json"));
var server = new DevOpsServer("dnceng", httpClient: new HttpClient(handler));
var timeline = await server.GetTimelineAttemptAsync("public", 795146, attempt : 1);
Assert.NotNull(timeline);
var timelineTree = TimelineTree.Create(timeline !);
Assert.NotNull(timelineTree);
Assert.True(timelineTree.TryGetNode("711d63f1-27de-5afd-fdbe-3b9edb784e9f", out var node));
Assert.Equal(TaskResult.Failed, node !.TimelineRecord.Result);
}
public static async Task <List <DotNetTestRun> > ListDotNetTestRunsAsync(DevOpsServer server, Build build, params TestOutcome[] outcomes)
{
var testRuns = await server.ListTestRunsAsync(build.Project.Name, build.Id).ConfigureAwait(false);
var taskList = new List <Task <DotNetTestRun> >();
foreach (var testRun in testRuns)
{
taskList.Add(GetDotNetTestRunAsync(server, build, testRun, outcomes));
}
await Task.WhenAll(taskList);
var list = new List <DotNetTestRun>();
foreach (var task in taskList)
{
list.Add(task.Result);
}
return(list);
private static async Task Scratch()
{
var server = new DevOpsServer("dnceng", await GetToken("dnceng"));
var all = await server.ListArtifactsAsync("internal", 572208);
var sum = all
.Select(x => x.GetByteSize())
.Where(x => x.HasValue)
.Select(x => (double)x)
.Sum();
var kb = 1_024;
var mb = kb * kb;
var gb = mb * kb;
Console.WriteLine(gb);
Console.WriteLine(sum / gb);
foreach (var a in all)
{
Console.WriteLine($"{a.Name} {a.GetByteSize()}");
}
}
static async Task <(Build Build, HelixLogInfo?LogInfo, List <string> BadLogs)> SearchBuild(
DevOpsServer server,
DotNetQueryUtil queryUtil,
Regex textRegex,
Build build)
{
var badLogList = new List <string>();
try
{
var workItems = await queryUtil
.ListHelixWorkItemsAsync(build, DotNetUtil.FailedTestOutcomes)
.ConfigureAwait(false);
foreach (var workItem in workItems)
{
var logInfo = await HelixUtil.GetHelixLogInfoAsync(server, workItem);
if (logInfo.ConsoleUri is object)
{
var isMatch = await queryUtil.SearchFileForAnyMatchAsync(
logInfo.ConsoleUri,
textRegex,
ex => badLogList.Add($"Unable to search helix logs {build.Id} {workItem.HelixInfo.JobId}, {logInfo.ConsoleUri}: {ex.Message}")).ConfigureAwait(false);
if (isMatch)
{
return(build, logInfo, badLogList);
}
}
}
}
catch (Exception ex)
{
badLogList.Add($"Unable to search helix logs for {build.Id}: {ex.Message}");
}
return(build, null, badLogList);
}
public StandardTestBase(ITestOutputHelper testOutputHelper)
{
Connection = CreateInMemoryDatabase();
var options = new DbContextOptionsBuilder <TriageContext>()
.UseSqlite(Connection)
.Options;
Context = new TriageContext(options);
TriageContextUtil = new TriageContextUtil(Context);
TestableHttpMessageHandler = new TestableHttpMessageHandler();
TestableLogger = new TestableLogger(testOutputHelper);
TestableGitHubClientFactory = new TestableGitHubClientFactory();
var httpClient = new HttpClient(TestableHttpMessageHandler);
Server = new DevOpsServer("random", httpClient: httpClient);
HelixServer = new HelixServer(httpClient: httpClient);
QueryUtil = new DotNetQueryUtil(Server);
Context.Database.EnsureDeleted();
Context.Database.EnsureCreated();
}
public async Task UploadBuildAsync(int buildId)
{
var build = await DevOpsServer.GetBuildAsync(ProjectName, buildId);
await UploadBuild(build);
}
public static async Task <Dictionary <HelixInfo, HelixLogInfo> > GetHelixMapAsync(this DevOpsServer server, IEnumerable <DotNetTestCaseResult> testCaseResults)
{
var query = testCaseResults
.Where(x => x.HelixWorkItem.HasValue)
.Select(x => x.HelixWorkItem !.Value)
.GroupBy(x => x.HelixInfo)
.ToList()
.AsParallel()
.Select(async g => (g.Key, await HelixUtil.GetHelixLogInfoAsync(server, g.First())));
await Task.WhenAll(query).ConfigureAwait(false);
return(query.ToDictionary(x => x.Result.Key, x => x.Result.Item2));
}
public static Task <Dictionary <HelixInfo, HelixLogInfo> > GetHelixMapAsync(this DevOpsServer server, DotNetTestRun testRun) => GetHelixMapAsync(server, testRun.TestCaseResults);
public static Task <List <Timeline> > ListTimelineAttemptsAsync(this DevOpsServer server, string project, int buildNumber)
{
IAzureUtil azureUtil = new AzureUtil(server);
return(azureUtil.ListTimelineAttemptsAsync(project, buildNumber));
}
public CloneTimeUtil(string sqlConnectionString, ILogger logger = null)
{
Logger = logger ?? DotNetUtil.CreateConsoleLogger();
DevOpsServer = new DevOpsServer("dnceng");
SqlConnection = new SqlConnection(sqlConnectionString);
}
public async Task <List <Build> > ListBuildsAsync(int top) => await DevOpsServer.ListBuildsAsync(ProjectName, new[] { BuildDefinitionId }, top : top);
