internal RuntimeInfo(string personalAccessToken = null, bool cacheable = false)
{
Server = cacheable
? new CachingDevOpsServer(RuntimeInfoUtil.CacheDirectory, "dnceng", personalAccessToken)
: new DevOpsServer("dnceng", personalAccessToken);
QueryUtil = new DotNetQueryUtil(Server);
}
internal static async Task <BuildTestInfoCollection> ListBuildTestInfosAsync(this DotNetQueryUtil queryUtil, BuildSearchOptionSet optionSet, bool includeAllTests = false)
{
TestOutcome[]? outcomes = includeAllTests
? null
: new[] { TestOutcome.Failed };
var list = new List <BuildTestInfo>();
foreach (var build in await queryUtil.ListBuildsAsync(optionSet).ConfigureAwait(false))
{
try
{
var collection = await DotNetUtil.ListDotNetTestRunsAsync(queryUtil.Server, build, outcomes);
var buildTestInfo = new BuildTestInfo(build, collection.SelectMany(x => x.TestCaseResults).ToList());
list.Add(buildTestInfo);
}
catch (Exception ex)
{
Console.WriteLine($"Cannot get test info for {build.Id} {DevOpsUtil.GetBuildUri(build)}");
Console.WriteLine(ex.Message);
}
}
return(new BuildTestInfoCollection(new ReadOnlyCollection <BuildTestInfo>(list)));
}
public ModelDataUtil(
DotNetQueryUtil queryUtil,
TriageContextUtil triageContextUtil,
ILogger logger)
{
Server = queryUtil.Server;
QueryUtil = queryUtil;
TriageContextUtil = triageContextUtil;
Logger = logger;
}
internal RuntimeInfo(
DevOpsServer devopsServer,
HelixServer helixServer,
IAzureUtil azureUtil)
{
DevOpsServer = devopsServer;
HelixServer = helixServer;
AzureUtil = azureUtil;
QueryUtil = new DotNetQueryUtil(DevOpsServer, azureUtil);
}
public TrackingIssueUtil(
HelixServer helixServer,
DotNetQueryUtil queryUtil,
TriageContextUtil triageContextUtil,
ILogger logger)
{
HelixServer = helixServer;
QueryUtil = queryUtil;
TriageContextUtil = triageContextUtil;
Logger = logger;
}
public AutoTriageUtil(
DevOpsServer server,
GitHubClient gitHubClient,
TriageContext context,
ILogger logger)
{
Server = server;
GitHubClient = gitHubClient;
QueryUtil = new DotNetQueryUtil(server);
TriageContextUtil = new TriageContextUtil(context);
Logger = logger;
}
public void ParseQueryString(string userQuery)
{
foreach (var tuple in DotNetQueryUtil.TokenizeQueryPairs(userQuery))
{
switch (tuple.Name.ToLower())
{
case "definition":
Definition = tuple.Value;
break;
default:
throw new Exception("Invalid option: '{tuple.Name}'");
}
}
}
public void ParseQueryString(string userQuery)
{
foreach (var tuple in DotNetQueryUtil.TokenizeQueryPairs(userQuery))
{
switch (tuple.Name.ToLower())
{
case "logkind":
switch (tuple.Value)
{
case "console":
MaybeAdd(HelixLogKind.Console);
break;
case "runclient":
MaybeAdd(HelixLogKind.RunClient);
break;
case "testresults":
MaybeAdd(HelixLogKind.TestResults);
break;
}
break;
case "text":
Text = tuple.Value.Trim('"');
break;
case "limit":
Limit = int.Parse(tuple.Value);
break;
default:
throw new Exception($"Invalid option {tuple.Name}");
}
}
void MaybeAdd(HelixLogKind kind)
{
if (!HelixLogKinds.Contains(kind))
{
HelixLogKinds.Add(kind);
}
}
}
static async Task <(Build Build, HelixLogInfo?LogInfo, List <string> BadLogs)> SearchBuild(
DevOpsServer server,
DotNetQueryUtil queryUtil,
Regex textRegex,
Build build)
{
var badLogList = new List <string>();
try
{
var workItems = await queryUtil
.ListHelixWorkItemsAsync(build, DotNetUtil.FailedTestOutcomes)
.ConfigureAwait(false);
foreach (var workItem in workItems)
{
var logInfo = await HelixUtil.GetHelixLogInfoAsync(server, workItem);
if (logInfo.ConsoleUri is object)
{
var isMatch = await queryUtil.SearchFileForAnyMatchAsync(
logInfo.ConsoleUri,
textRegex,
ex => badLogList.Add($"Unable to search helix logs {build.Id} {workItem.HelixInfo.JobId}, {logInfo.ConsoleUri}: {ex.Message}")).ConfigureAwait(false);
if (isMatch)
{
return(build, logInfo, badLogList);
}
}
}
}
catch (Exception ex)
{
badLogList.Add($"Unable to search helix logs for {build.Id}: {ex.Message}");
}
return(build, null, badLogList);
}
public StandardTestBase(ITestOutputHelper testOutputHelper)
{
Connection = CreateInMemoryDatabase();
var options = new DbContextOptionsBuilder <TriageContext>()
.UseSqlite(Connection)
.Options;
Context = new TriageContext(options);
TriageContextUtil = new TriageContextUtil(Context);
TestableHttpMessageHandler = new TestableHttpMessageHandler();
TestableLogger = new TestableLogger(testOutputHelper);
TestableGitHubClientFactory = new TestableGitHubClientFactory();
var httpClient = new HttpClient(TestableHttpMessageHandler);
Server = new DevOpsServer("random", httpClient: httpClient);
HelixServer = new HelixServer(httpClient: httpClient);
QueryUtil = new DotNetQueryUtil(Server);
Context.Database.EnsureDeleted();
Context.Database.EnsureCreated();
}
public async Task BuildCompleteAsync(
[QueueTrigger(QueueNameBuildComplete, Connection = ConfigurationAzureBlobConnectionString)] string message,
[Queue(QueueNameTriageBuildAttempt, Connection = ConfigurationAzureBlobConnectionString)] IAsyncCollector <string> triageCollector,
[Queue(QueueNameBuildRetry, Connection = ConfigurationAzureBlobConnectionString)] IAsyncCollector <string> retryCollector,
ILogger logger)
{
var buildInfoMessage = JsonConvert.DeserializeObject <BuildInfoMessage>(message);
var projectName = buildInfoMessage.ProjectName;
if (projectName is null)
{
var projectId = buildInfoMessage.ProjectId;
if (projectId is null)
{
logger.LogError("Both project name and id are null");
return;
}
buildInfoMessage.ProjectName = await Server.ConvertProjectIdToNameAsync(projectId);
}
logger.LogInformation($"Gathering data for build {buildInfoMessage.ProjectName} {buildInfoMessage.BuildNumber}");
if (buildInfoMessage.ProjectName != "public")
{
logger.LogError($"Asked to gather data from {buildInfoMessage.ProjectName} which is not 'public'");
return;
}
var build = await Server.GetBuildAsync(buildInfoMessage.ProjectName !, buildInfoMessage.BuildNumber);
var queryUtil = new DotNetQueryUtil(Server);
var modelDataUtil = new ModelDataUtil(queryUtil, TriageContextUtil, logger);
var buildAttemptKey = await modelDataUtil.EnsureModelInfoAsync(build);
await triageCollector.AddAsync(JsonConvert.SerializeObject(new BuildAttemptMessage(buildAttemptKey)));
await retryCollector.AddAsync(JsonConvert.SerializeObject(new BuildAttemptMessage(buildAttemptKey)));
}
public void ParseQueryString(string userQuery)
{
foreach (var tuple in DotNetQueryUtil.TokenizeQueryPairs(userQuery))
{
switch (tuple.Name.ToLower())
{
case "logname":
LogName = tuple.Value.Trim('"');
break;
case "text":
Text = tuple.Value.Trim('"');
break;
case "limit":
Limit = int.Parse(tuple.Value);
break;
default:
throw new Exception($"Invalid option {tuple.Name}");
}
}
}
internal static async Task <List <Build> > ListBuildsAsync(this DotNetQueryUtil queryUtil, BuildSearchOptionSet optionSet)
{
if (optionSet.BuildIds.Count > 0 && optionSet.Definitions.Count > 0)
{
OptionFailure("Cannot specify builds and definitions", optionSet);
throw CreateBadOptionException();
}
var searchCount = optionSet.SearchCount ?? BuildSearchOptionSet.DefaultSearchCount;
var repository = optionSet.Repository;
var branch = optionSet.Branch;
var before = optionSet.Before;
var after = optionSet.After;
if (branch is object && !branch.StartsWith("refs"))
{
branch = $"refs/heads/{branch}";
}
var builds = new List <Build>();
if (optionSet.BuildIds.Count > 0)
{
if (optionSet.Repository is object)
{
OptionFailure("Cannot specify builds and repository", optionSet);
throw CreateBadOptionException();
}
if (optionSet.Branch is object)
{
OptionFailure("Cannot specify builds and branch", optionSet);
throw CreateBadOptionException();
}
if (optionSet.SearchCount is object)
{
OptionFailure("Cannot specify builds and count", optionSet);
throw CreateBadOptionException();
}
foreach (var buildInfo in optionSet.BuildIds)
{
if (!TryGetBuildId(optionSet, buildInfo, out var buildProject, out var buildId))
{
OptionFailure($"Cannot convert {buildInfo} to build id", optionSet);
throw CreateBadOptionException();
}
var build = await queryUtil.Server.GetBuildAsync(buildProject, buildId).ConfigureAwait(false);
builds.Add(build);
}
}
else
{
var(project, definitions) = GetProjectAndDefinitions();
var collection = await queryUtil.ListBuildsAsync(
searchCount,
project,
definitions : definitions,
repositoryId : repository,
branch : branch,
includePullRequests : optionSet.IncludePullRequests,
before : optionSet.Before,
after : optionSet.After);
builds.AddRange(collection);
}
// Exclude out the builds that are complicating results
foreach (var excludedBuildId in optionSet.ExcludedBuildIds)
{
builds = builds.Where(x => x.Id != excludedBuildId).ToList();
}
return(builds);
(string Project, int[] Definitions) GetProjectAndDefinitions()
{
if (optionSet.Definitions.Count == 0)
{
return(optionSet.Project ?? DotNetUtil.DefaultAzureProject, Array.Empty <int>());
}
string?project = null;
var list = new List <int>();
foreach (var definition in optionSet.Definitions)
{
if (!DotNetUtil.TryGetDefinitionId(definition, out var definitionProject, out var definitionId))
{
OptionFailureDefinition(definition, optionSet);
throw CreateBadOptionException();
}
if (definitionProject is object)
{
if (project is null)
{
project = definitionProject;
}
else if (!StringComparer.OrdinalIgnoreCase.Equals(definitionProject, project))
{
throw new InvalidOperationException($"Conflicting project names {project} and {definitionProject}");
}
}
list.Add(definitionId);
}
project ??= DotNetUtil.DefaultAzureProject;
return(project, list.ToArray());
}
static bool TryGetBuildId(BuildSearchOptionSet optionSet, string build, out string project, out int buildId)
{
var defaultProject = optionSet.Project ?? DotNetUtil.DefaultAzureProject;
return(DotNetQueryUtil.TryGetBuildId(build, defaultProject, out project !, out buildId));
}
public void TokenizeQuery(string query, string[] tokens)
{
Assert.Equal(tokens, DotNetQueryUtil.TokenizeQuery(query));
}
// internal static async Task<int> Main(string[] args) => await MainCore(args.ToList());
internal static async Task <int> MainCore(TriageContext context, List <string> args)
{
var azdoToken = Environment.GetEnvironmentVariable("RUNFO_AZURE_TOKEN");
var gitHubToken = Environment.GetEnvironmentVariable("RUNFO_GITHUB_TOKEN");
var cacheDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "runfo", "json");
var server = new CachingDevOpsServer(cacheDirectory, "dnceng", azdoToken);
var gitHubClient = new GitHubClient(new ProductHeaderValue("RuntimeStatusPage"));
var queryUtil = new DotNetQueryUtil(server);
using var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole());
// TODO: should not hard code jaredpar here
gitHubClient.Credentials = new Credentials("jaredpar", gitHubToken);
string command;
if (args.Count == 0)
{
command = "list";
}
else
{
command = args[0];
args = args.Skip(1).ToList();
}
var autoTriageUtil = new AutoTriageUtil(server, gitHubClient, context, loggerFactory.CreateLogger <AutoTriageUtil>());
switch (command)
{
case "auto":
await RunAutoTriage(args);
break;
case "issues":
await RunIssues();
break;
case "rebuild":
await RunRebuild();
break;
case "scratch":
await RunScratch();
break;
default:
Console.WriteLine($"Unrecognized option {command}");
break;
}
return(ExitSuccess);
async Task RunAutoTriage(List <string> args)
{
autoTriageUtil.EnsureTriageIssues();
await autoTriageUtil.Triage("-d runtime -c 100 -pr");
await autoTriageUtil.Triage("-d runtime-official -c 20 -pr");
await autoTriageUtil.UpdateQueryIssues();
await autoTriageUtil.UpdateStatusIssue();
}
async Task RunIssues()
{
await autoTriageUtil.UpdateQueryIssues();
await autoTriageUtil.UpdateStatusIssue();
}
async Task RunRebuild()
{
autoTriageUtil.EnsureTriageIssues();
await autoTriageUtil.Triage("-d runtime -c 1000 -pr");
await autoTriageUtil.Triage("-d aspnet -c 1000 -pr");
await autoTriageUtil.Triage("-d runtime-official -c 50 -pr");
await autoTriageUtil.Triage("-d aspnet-official -c 50 -pr");
}
async Task RunScratch()
{
// autoTriageUtil.EnsureTriageIssues();
// await autoTriageUtil.Triage("-d runtime -c 500 -pr");
// await autoTriageUtil.Triage("-d runtime-official -c 50 -pr");
// await autoTriageUtil.UpdateQueryIssues();
await autoTriageUtil.UpdateStatusIssue();
}
}
