private static async Task RequestBenchmark(PRBenchmarkRequest prBenchmarkRequest, string newJobFileName)
{
await using var baseJobStream = File.OpenRead(BaseJobPath);
var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream);
jsonDictionary[nameof(BuildInstructions)] = new BuildInstructions
{
BuildCommands = BaseBuildInstructions.BuildCommands,
ExtraDriverArgs = BaseBuildInstructions.ExtraDriverArgs,
PullRequestNumber = prBenchmarkRequest.PullRequest.Number,
BaselineSHA = prBenchmarkRequest.PullRequest.Base.Sha,
PullRequestSHA = prBenchmarkRequest.PullRequest.Head.Sha,
ScenarioName = prBenchmarkRequest.ScenarioName,
};
using var newJobStream = new MemoryStream();
await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions
{
WriteIndented = true,
});
newJobStream.Position = 0;
await JobFileSystem.WriteFile(newJobStream, newJobFileName);
}
private static async Task <BenchmarkResult> WaitForBenchmarkResults(string newJobFileName)
{
var startTicks = Environment.TickCount64;
var expectedProcessedJobPath = Path.Combine(ProcessedDirectoryName, newJobFileName);
while (!await JobFileSystem.FileExists(expectedProcessedJobPath))
{
if (Environment.TickCount64 - startTicks > BenchmarkTimeout.TotalMilliseconds)
{
throw new TimeoutException($"Benchmark results for job {newJobFileName} were not published to {ProcessedDirectoryName} within {BenchmarkTimeout}.");
}
await Task.Delay(1000);
}
Console.WriteLine($"Found '{newJobFileName}'");
using var processedJsonStream = await JobFileSystem.ReadFile(expectedProcessedJobPath);
using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream);
foreach (var element in jsonDocument.RootElement.EnumerateObject())
{
if (element.NameEquals(nameof(BenchmarkResult)))
{
return(JsonSerializer.Deserialize <BenchmarkResult>(element.Value.GetRawText()));
}
}
throw new InvalidDataException($"Processed benchmark job '{newJobFileName}' did not include a top-level '{nameof(BenchmarkResult)}' property.");
}
private static async Task <bool> WaitForCompleteJsonFile(string jsonFilePath, TimeSpan timeout)
{
var startTicks = Environment.TickCount64;
while (!await JobFileSystem.FileExists(jsonFilePath))
{
if (Environment.TickCount64 - startTicks > timeout.TotalMilliseconds)
{
return(false);
}
await Task.Delay(1000);
}
// Wait up to 5 seconds for the Json file to be fully parsable.
for (int i = 0; i < 5; i++)
{
try
{
using var processedJsonStream = await JobFileSystem.ReadFile(jsonFilePath);
using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream);
return(true);
}
catch (JsonException)
{
if (i == 4)
{
throw;
}
await Task.Delay(1000);
}
}
return(false);
}
private static async Task <BenchmarkResult> WaitForBenchmarkResults(string newJobFileName)
{
var startTicks = Environment.TickCount64;
var expectedProcessedJobPath = Path.Combine(ProcessedDirectoryName, newJobFileName);
if (!await WaitForCompleteJsonFile(expectedProcessedJobPath, BenchmarkTimeout))
{
throw new TimeoutException($"Benchmark results for job {newJobFileName} were not published to {ProcessedDirectoryName} within {BenchmarkTimeout}.");
}
Console.WriteLine($"Found '{newJobFileName}'");
using var processedJsonStream = await JobFileSystem.ReadFile(expectedProcessedJobPath);
using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream);
if (!jsonDocument.RootElement.TryGetProperty(nameof(BenchmarkResult), out var benchmarkResultElement))
{
throw new InvalidDataException($"Processed benchmark job '{newJobFileName}' did not include a top-level '{nameof(BenchmarkResult)}' property.");
}
return(JsonSerializer.Deserialize <BenchmarkResult>(benchmarkResultElement.GetRawText()));
}
private static async Task RequestBenchmark(PullRequest pr, string newJobFileName)
{
await using var baseJobStream = File.OpenRead(BaseJobPath);
var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream);
jsonDictionary["BuildInstructions"] = new BuildInstructions
{
BuildCommands = BuildCommands,
BaselineSHA = pr.Base.Sha,
PullRequestSHA = pr.Head.Sha,
};
using var newJobStream = new MemoryStream();
await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions
{
WriteIndented = true,
});
newJobStream.Position = 0;
await JobFileSystem.WriteFile(newJobStream, newJobFileName);
}
public static int Main(string[] args)
{
var app = new CommandLineApplication();
app.HelpOption("-h|--help");
var baseJobPath = app.Option("-b|--base-job <PATH>", "The base job file path", CommandOptionType.SingleValue).IsRequired();
var jobsPath = app.Option("-j|--jobs-path <PATH>", "The path where jobs are created", CommandOptionType.SingleValue);
var azureStorageConnectionString = app.Option("-c|--azure-storage-connection-string <CONNECTIONSTRING>", "The Azure Storage connection string", CommandOptionType.SingleValue);
var azureStorageFileShareName = app.Option("-f|--azure-storage-file-share-name <NAME>", "The Azure Storage file share name", CommandOptionType.SingleValue);
var githubUser = app.Option("-u|--github-user <NAME>", "The GitHub user name for the bot", CommandOptionType.SingleValue);
var githubUserToken = app.Option("-t|--github-user-token <TOKEN>", "The GitHub token for the bot", CommandOptionType.SingleValue);
var githubAppId = app.Option("-a|--github-app-id <ID>", "The GitHub App ID for the bot", CommandOptionType.SingleValue);
var githubAppKeyPath = app.Option("-k|--github-app-key-file <PATH>", "The GitHub App pem file path", CommandOptionType.SingleValue);
var githubAppInstallationId = app.Option("-i|--github-app-install-id <ID>", "The GitHub App installation ID for the repo. E.g. 'https://github.com/settings/installations/{Id}'", CommandOptionType.SingleValue);
app.OnExecuteAsync(async cancellationToken =>
{
BaseJobPath = baseJobPath.Value();
GitHubClient client;
string botLoginName;
if (githubUser.HasValue())
{
if (!githubUserToken.HasValue())
{
Console.WriteLine("--github-user was provided with no --github-token.");
return(-1);
}
botLoginName = githubUser.Value();
client = GetClientForUser(botLoginName, githubUserToken.Value());
}
else if (githubAppId.HasValue())
{
if (!githubAppKeyPath.HasValue())
{
Console.WriteLine("--github-app-id was provided with no --github-app-key-file.");
return(-1);
}
if (!githubAppInstallationId.HasValue())
{
Console.WriteLine("--github-app-id was provided with no --github-app-install-id.");
return(-1);
}
if (!long.TryParse(githubAppInstallationId.Value(), out long installId))
{
Console.WriteLine("--github-app-install-id is not a valid long.");
}
botLoginName = AppName + "[bot]";
client = await GetClientForApp(githubAppId.Value(), githubAppKeyPath.Value(), installId);
}
else
{
Console.WriteLine("Cannot authenticate with GitHub. Neither a --github-user nor a --github-app-id has been provided.");
return(-1);
}
if (jobsPath.HasValue())
{
JobFileSystem = new LocalFileSystem(jobsPath.Value());
}
else if (azureStorageConnectionString.HasValue())
{
if (!azureStorageFileShareName.HasValue())
{
Console.WriteLine("--azure-storage-connection-string was provided with no --azure-storage-file-share.");
return(-1);
}
var cloudDir = await GetCloudFileDirectory(azureStorageConnectionString.Value(), azureStorageFileShareName.Value());
JobFileSystem = new AzureStorageFileSystem(cloudDir);
}
else
{
Console.WriteLine("Neither a --jobs-path nor an --azure-storage-connection-string has been provided.");
return(-1);
}
await JobFileSystem.CreateDirectoryIfNotExists(ProcessedDirectoryName);
BaseBuildInstructions = await GetBuildInstructions();
Console.WriteLine($"Scanning for benchmark requests in {Owner}/{Repo}.");
await foreach (var prBenchmarkRequest in GetPRsToBenchmark(client, botLoginName))
{
var pr = prBenchmarkRequest.PullRequest;
try
{
var session = Guid.NewGuid().ToString("n");
var newJobFileName = $"{session}.{Path.GetFileName(BaseJobPath)}";
var startingCommentText = string.Format(StartingBencmarkComment, prBenchmarkRequest.ScenarioName, session);
Console.WriteLine($"Requesting {prBenchmarkRequest.ScenarioName} benchmark for PR #{pr.Number}.");
Console.WriteLine($"Benchmark starting comment: {startingCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, startingCommentText);
await RequestBenchmark(prBenchmarkRequest, newJobFileName);
Console.WriteLine($"Benchmark requested for PR #{pr.Number}. Waiting up to {BenchmarkTimeout} for results.");
var results = await WaitForBenchmarkResults(newJobFileName);
string FormatOutput(string stdout, string stderr)
{
return(string.IsNullOrEmpty(stderr) ? stdout : $"stdout: {results.BaselineStdout}\nstderr: {results.BaselineStderr}");
}
var baselineOutput = FormatOutput(results.BaselineStdout, results.BaselineStderr);
var prOutput = FormatOutput(results.PullRequestStdout, results.PullRequestStderr);
var resultCommentText = string.Format(CompletedBenchmarkCommentTemplate, baselineOutput, prOutput);
Console.WriteLine($"Benchmark results received for PR #{pr.Number}. Posting results to {pr.Url}.");
Console.WriteLine($"Benchmark results comment: {resultCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, resultCommentText);
}
catch (Exception ex)
{
var errorCommentText = $"Failed to benchmark PR #{pr.Number}. Skipping... Details:\n```\n{ex}\n```";
Console.WriteLine($"Benchmark error comment: {errorCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, errorCommentText);
}
}
Console.WriteLine($"Done scanning for benchmark requests.");
return(0);
});
return(app.Execute(args));
}
