private static async Task RequestBenchmark(PRBenchmarkRequest prBenchmarkRequest, string newJobFileName) { await using var baseJobStream = File.OpenRead(BaseJobPath); var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream); jsonDictionary[nameof(BuildInstructions)] = new BuildInstructions { BuildCommands = BaseBuildInstructions.BuildCommands, ExtraDriverArgs = BaseBuildInstructions.ExtraDriverArgs, PullRequestNumber = prBenchmarkRequest.PullRequest.Number, BaselineSHA = prBenchmarkRequest.PullRequest.Base.Sha, PullRequestSHA = prBenchmarkRequest.PullRequest.Head.Sha, ScenarioName = prBenchmarkRequest.ScenarioName, }; using var newJobStream = new MemoryStream(); await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions { WriteIndented = true, }); newJobStream.Position = 0; await JobFileSystem.WriteFile(newJobStream, newJobFileName); }
private static async Task <BenchmarkResult> WaitForBenchmarkResults(string newJobFileName) { var startTicks = Environment.TickCount64; var expectedProcessedJobPath = Path.Combine(ProcessedDirectoryName, newJobFileName); while (!await JobFileSystem.FileExists(expectedProcessedJobPath)) { if (Environment.TickCount64 - startTicks > BenchmarkTimeout.TotalMilliseconds) { throw new TimeoutException($"Benchmark results for job {newJobFileName} were not published to {ProcessedDirectoryName} within {BenchmarkTimeout}."); } await Task.Delay(1000); } Console.WriteLine($"Found '{newJobFileName}'"); using var processedJsonStream = await JobFileSystem.ReadFile(expectedProcessedJobPath); using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream); foreach (var element in jsonDocument.RootElement.EnumerateObject()) { if (element.NameEquals(nameof(BenchmarkResult))) { return(JsonSerializer.Deserialize <BenchmarkResult>(element.Value.GetRawText())); } } throw new InvalidDataException($"Processed benchmark job '{newJobFileName}' did not include a top-level '{nameof(BenchmarkResult)}' property."); }
private static async Task <bool> WaitForCompleteJsonFile(string jsonFilePath, TimeSpan timeout) { var startTicks = Environment.TickCount64; while (!await JobFileSystem.FileExists(jsonFilePath)) { if (Environment.TickCount64 - startTicks > timeout.TotalMilliseconds) { return(false); } await Task.Delay(1000); } // Wait up to 5 seconds for the Json file to be fully parsable. for (int i = 0; i < 5; i++) { try { using var processedJsonStream = await JobFileSystem.ReadFile(jsonFilePath); using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream); return(true); } catch (JsonException) { if (i == 4) { throw; } await Task.Delay(1000); } } return(false); }
private static async Task <BenchmarkResult> WaitForBenchmarkResults(string newJobFileName) { var startTicks = Environment.TickCount64; var expectedProcessedJobPath = Path.Combine(ProcessedDirectoryName, newJobFileName); if (!await WaitForCompleteJsonFile(expectedProcessedJobPath, BenchmarkTimeout)) { throw new TimeoutException($"Benchmark results for job {newJobFileName} were not published to {ProcessedDirectoryName} within {BenchmarkTimeout}."); } Console.WriteLine($"Found '{newJobFileName}'"); using var processedJsonStream = await JobFileSystem.ReadFile(expectedProcessedJobPath); using var jsonDocument = await JsonDocument.ParseAsync(processedJsonStream); if (!jsonDocument.RootElement.TryGetProperty(nameof(BenchmarkResult), out var benchmarkResultElement)) { throw new InvalidDataException($"Processed benchmark job '{newJobFileName}' did not include a top-level '{nameof(BenchmarkResult)}' property."); } return(JsonSerializer.Deserialize <BenchmarkResult>(benchmarkResultElement.GetRawText())); }
private static async Task RequestBenchmark(PullRequest pr, string newJobFileName) { await using var baseJobStream = File.OpenRead(BaseJobPath); var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream); jsonDictionary["BuildInstructions"] = new BuildInstructions { BuildCommands = BuildCommands, BaselineSHA = pr.Base.Sha, PullRequestSHA = pr.Head.Sha, }; using var newJobStream = new MemoryStream(); await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions { WriteIndented = true, }); newJobStream.Position = 0; await JobFileSystem.WriteFile(newJobStream, newJobFileName); }
public static int Main(string[] args) { var app = new CommandLineApplication(); app.HelpOption("-h|--help"); var baseJobPath = app.Option("-b|--base-job <PATH>", "The base job file path", CommandOptionType.SingleValue).IsRequired(); var jobsPath = app.Option("-j|--jobs-path <PATH>", "The path where jobs are created", CommandOptionType.SingleValue); var azureStorageConnectionString = app.Option("-c|--azure-storage-connection-string <CONNECTIONSTRING>", "The Azure Storage connection string", CommandOptionType.SingleValue); var azureStorageFileShareName = app.Option("-f|--azure-storage-file-share-name <NAME>", "The Azure Storage file share name", CommandOptionType.SingleValue); var githubUser = app.Option("-u|--github-user <NAME>", "The GitHub user name for the bot", CommandOptionType.SingleValue); var githubUserToken = app.Option("-t|--github-user-token <TOKEN>", "The GitHub token for the bot", CommandOptionType.SingleValue); var githubAppId = app.Option("-a|--github-app-id <ID>", "The GitHub App ID for the bot", CommandOptionType.SingleValue); var githubAppKeyPath = app.Option("-k|--github-app-key-file <PATH>", "The GitHub App pem file path", CommandOptionType.SingleValue); var githubAppInstallationId = app.Option("-i|--github-app-install-id <ID>", "The GitHub App installation ID for the repo. E.g. 'https://github.com/settings/installations/{Id}'", CommandOptionType.SingleValue); app.OnExecuteAsync(async cancellationToken => { BaseJobPath = baseJobPath.Value(); GitHubClient client; string botLoginName; if (githubUser.HasValue()) { if (!githubUserToken.HasValue()) { Console.WriteLine("--github-user was provided with no --github-token."); return(-1); } botLoginName = githubUser.Value(); client = GetClientForUser(botLoginName, githubUserToken.Value()); } else if (githubAppId.HasValue()) { if (!githubAppKeyPath.HasValue()) { Console.WriteLine("--github-app-id was provided with no --github-app-key-file."); return(-1); } if (!githubAppInstallationId.HasValue()) { Console.WriteLine("--github-app-id was provided with no --github-app-install-id."); return(-1); } if (!long.TryParse(githubAppInstallationId.Value(), out long installId)) { Console.WriteLine("--github-app-install-id is not a valid long."); } botLoginName = AppName + "[bot]"; client = await GetClientForApp(githubAppId.Value(), githubAppKeyPath.Value(), installId); } else { Console.WriteLine("Cannot authenticate with GitHub. Neither a --github-user nor a --github-app-id has been provided."); return(-1); } if (jobsPath.HasValue()) { JobFileSystem = new LocalFileSystem(jobsPath.Value()); } else if (azureStorageConnectionString.HasValue()) { if (!azureStorageFileShareName.HasValue()) { Console.WriteLine("--azure-storage-connection-string was provided with no --azure-storage-file-share."); return(-1); } var cloudDir = await GetCloudFileDirectory(azureStorageConnectionString.Value(), azureStorageFileShareName.Value()); JobFileSystem = new AzureStorageFileSystem(cloudDir); } else { Console.WriteLine("Neither a --jobs-path nor an --azure-storage-connection-string has been provided."); return(-1); } await JobFileSystem.CreateDirectoryIfNotExists(ProcessedDirectoryName); BaseBuildInstructions = await GetBuildInstructions(); Console.WriteLine($"Scanning for benchmark requests in {Owner}/{Repo}."); await foreach (var prBenchmarkRequest in GetPRsToBenchmark(client, botLoginName)) { var pr = prBenchmarkRequest.PullRequest; try { var session = Guid.NewGuid().ToString("n"); var newJobFileName = $"{session}.{Path.GetFileName(BaseJobPath)}"; var startingCommentText = string.Format(StartingBencmarkComment, prBenchmarkRequest.ScenarioName, session); Console.WriteLine($"Requesting {prBenchmarkRequest.ScenarioName} benchmark for PR #{pr.Number}."); Console.WriteLine($"Benchmark starting comment: {startingCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, startingCommentText); await RequestBenchmark(prBenchmarkRequest, newJobFileName); Console.WriteLine($"Benchmark requested for PR #{pr.Number}. Waiting up to {BenchmarkTimeout} for results."); var results = await WaitForBenchmarkResults(newJobFileName); string FormatOutput(string stdout, string stderr) { return(string.IsNullOrEmpty(stderr) ? stdout : $"stdout: {results.BaselineStdout}\nstderr: {results.BaselineStderr}"); } var baselineOutput = FormatOutput(results.BaselineStdout, results.BaselineStderr); var prOutput = FormatOutput(results.PullRequestStdout, results.PullRequestStderr); var resultCommentText = string.Format(CompletedBenchmarkCommentTemplate, baselineOutput, prOutput); Console.WriteLine($"Benchmark results received for PR #{pr.Number}. Posting results to {pr.Url}."); Console.WriteLine($"Benchmark results comment: {resultCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, resultCommentText); } catch (Exception ex) { var errorCommentText = $"Failed to benchmark PR #{pr.Number}. Skipping... Details:\n```\n{ex}\n```"; Console.WriteLine($"Benchmark error comment: {errorCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, errorCommentText); } } Console.WriteLine($"Done scanning for benchmark requests."); return(0); }); return(app.Execute(args)); }