private static async Task RequestBenchmark(PRBenchmarkRequest prBenchmarkRequest, string newJobFileName) { await using var baseJobStream = File.OpenRead(BaseJobPath); var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream); jsonDictionary[nameof(BuildInstructions)] = new BuildInstructions { BuildCommands = BaseBuildInstructions.BuildCommands, ExtraDriverArgs = BaseBuildInstructions.ExtraDriverArgs, PullRequestNumber = prBenchmarkRequest.PullRequest.Number, BaselineSHA = prBenchmarkRequest.PullRequest.Base.Sha, PullRequestSHA = prBenchmarkRequest.PullRequest.Head.Sha, ScenarioName = prBenchmarkRequest.ScenarioName, }; using var newJobStream = new MemoryStream(); await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions { WriteIndented = true, }); newJobStream.Position = 0; await JobFileSystem.WriteFile(newJobStream, newJobFileName); }
private static void RunBuildCommands(BuildInstructions buildRules) { foreach (var buildCommand in buildRules.BuildCommands) { RunCommand(buildCommand); } }
private static void RunBuildCommands(BuildInstructions buildRules) { foreach (var buildCommand in buildRules.BuildCommands) { var splitCommand = buildCommand.Split(' ', 2); Process.Start(splitCommand[0], splitCommand.Length == 2 ? splitCommand[1] : string.Empty).WaitForExit(); } }
private static string GetDriverArguments( string jobsFilePath, string sessionId, string sdkVersion, BuildInstructions buildInstructions, bool isBaseline) { var argumentsBuilder = new StringBuilder($"{DriverPath} --server {ServerUrl} --client {ClientUrl} --jobs {jobsFilePath} --session {sessionId}"); argumentsBuilder.Append(" --self-contained --aspNetCoreVersion Latest --runtimeVersion Latest --quiet"); if (!string.IsNullOrWhiteSpace(buildInstructions.ScenarioName) && !string.Equals("Default", buildInstructions.ScenarioName, StringComparison.OrdinalIgnoreCase)) { argumentsBuilder.Append(" --scenario "); argumentsBuilder.Append(buildInstructions.ScenarioName); } if (!string.IsNullOrWhiteSpace(sdkVersion)) { argumentsBuilder.Append(" --sdk "); argumentsBuilder.Append(sdkVersion); } if (!string.IsNullOrWhiteSpace(buildInstructions.ExtraDriverArgs)) { argumentsBuilder.Append(" "); argumentsBuilder.Append(buildInstructions.ExtraDriverArgs); } if (isBaseline) { argumentsBuilder.Append(" --save baseline --description Before"); } else { argumentsBuilder.Append(" --diff baseline --description After"); } return(argumentsBuilder.ToString()); }
private static async Task RequestBenchmark(PullRequest pr, string newJobFileName) { await using var baseJobStream = File.OpenRead(BaseJobPath); var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream); jsonDictionary["BuildInstructions"] = new BuildInstructions { BuildCommands = BuildCommands, BaselineSHA = pr.Base.Sha, PullRequestSHA = pr.Head.Sha, }; using var newJobStream = new MemoryStream(); await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions { WriteIndented = true, }); newJobStream.Position = 0; await JobFileSystem.WriteFile(newJobStream, newJobFileName); }
private static async Task <string[]> GetBuildCommands() { using var processingJsonStream = File.OpenRead(BaseJobPath); using var jsonDocument = await JsonDocument.ParseAsync(processingJsonStream); BuildInstructions buildInstructions = null; foreach (var element in jsonDocument.RootElement.EnumerateObject()) { if (element.NameEquals(nameof(BuildInstructions))) { buildInstructions = JsonSerializer.Deserialize <BuildInstructions>(element.Value.GetRawText()); break; } } if (buildInstructions is null) { throw new InvalidDataException($"Job file {Path.GetFileName(BaseJobPath)} doesn't include a top-level '{nameof(BuildInstructions)}' property."); } return(buildInstructions.BuildCommands); }
private static async Task RequestBenchmark(PullRequest pr, string session) { var baseFile = new FileInfo(BaseJobPath); var newJobPath = Path.Combine(Path.GetTempPath(), $"{session}.{baseFile.Name}"); var newJobFile = new FileInfo(newJobPath); baseFile.CopyTo(newJobPath); try { using (var newJobStream = File.Open(newJobPath, System.IO.FileMode.Open)) { var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(newJobStream); jsonDictionary["BuildInstructions"] = new BuildInstructions { BuildCommands = BuildCommands, BaselineSHA = pr.Base.Sha, PullRequestSHA = pr.Head.Sha, }; // Clear file and reset position to 0 newJobStream.SetLength(0); await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions { WriteIndented = true, }); } newJobFile.MoveTo(Path.Combine(JobsPath, newJobFile.Name)); } catch { newJobFile.Delete(); throw; } }
public static int Main(string[] args) { var app = new CommandLineApplication(); app.HelpOption("-h|--help"); var baseJobPath = app.Option("-b|--base-job <PATH>", "The base job file path", CommandOptionType.SingleValue).IsRequired(); var jobsPath = app.Option("-j|--jobs-path <PATH>", "The path where jobs are created", CommandOptionType.SingleValue); var azureStorageConnectionString = app.Option("-c|--azure-storage-connection-string <CONNECTIONSTRING>", "The Azure Storage connection string", CommandOptionType.SingleValue); var azureStorageFileShareName = app.Option("-f|--azure-storage-file-share-name <NAME>", "The Azure Storage file share name", CommandOptionType.SingleValue); var githubUser = app.Option("-u|--github-user <NAME>", "The GitHub user name for the bot", CommandOptionType.SingleValue); var githubUserToken = app.Option("-t|--github-user-token <TOKEN>", "The GitHub token for the bot", CommandOptionType.SingleValue); var githubAppId = app.Option("-a|--github-app-id <ID>", "The GitHub App ID for the bot", CommandOptionType.SingleValue); var githubAppKeyPath = app.Option("-k|--github-app-key-file <PATH>", "The GitHub App pem file path", CommandOptionType.SingleValue); var githubAppInstallationId = app.Option("-i|--github-app-install-id <ID>", "The GitHub App installation ID for the repo. E.g. 'https://github.com/settings/installations/{Id}'", CommandOptionType.SingleValue); app.OnExecuteAsync(async cancellationToken => { BaseJobPath = baseJobPath.Value(); GitHubClient client; string botLoginName; if (githubUser.HasValue()) { if (!githubUserToken.HasValue()) { Console.WriteLine("--github-user was provided with no --github-token."); return(-1); } botLoginName = githubUser.Value(); client = GetClientForUser(botLoginName, githubUserToken.Value()); } else if (githubAppId.HasValue()) { if (!githubAppKeyPath.HasValue()) { Console.WriteLine("--github-app-id was provided with no --github-app-key-file."); return(-1); } if (!githubAppInstallationId.HasValue()) { Console.WriteLine("--github-app-id was provided with no --github-app-install-id."); return(-1); } if (!long.TryParse(githubAppInstallationId.Value(), out long installId)) { Console.WriteLine("--github-app-install-id is not a valid long."); } botLoginName = AppName + "[bot]"; client = await GetClientForApp(githubAppId.Value(), githubAppKeyPath.Value(), installId); } else { Console.WriteLine("Cannot authenticate with GitHub. Neither a --github-user nor a --github-app-id has been provided."); return(-1); } if (jobsPath.HasValue()) { JobFileSystem = new LocalFileSystem(jobsPath.Value()); } else if (azureStorageConnectionString.HasValue()) { if (!azureStorageFileShareName.HasValue()) { Console.WriteLine("--azure-storage-connection-string was provided with no --azure-storage-file-share."); return(-1); } var cloudDir = await GetCloudFileDirectory(azureStorageConnectionString.Value(), azureStorageFileShareName.Value()); JobFileSystem = new AzureStorageFileSystem(cloudDir); } else { Console.WriteLine("Neither a --jobs-path nor an --azure-storage-connection-string has been provided."); return(-1); } await JobFileSystem.CreateDirectoryIfNotExists(ProcessedDirectoryName); BaseBuildInstructions = await GetBuildInstructions(); Console.WriteLine($"Scanning for benchmark requests in {Owner}/{Repo}."); await foreach (var prBenchmarkRequest in GetPRsToBenchmark(client, botLoginName)) { var pr = prBenchmarkRequest.PullRequest; try { var session = Guid.NewGuid().ToString("n"); var newJobFileName = $"{session}.{Path.GetFileName(BaseJobPath)}"; var startingCommentText = string.Format(StartingBencmarkComment, prBenchmarkRequest.ScenarioName, session); Console.WriteLine($"Requesting {prBenchmarkRequest.ScenarioName} benchmark for PR #{pr.Number}."); Console.WriteLine($"Benchmark starting comment: {startingCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, startingCommentText); await RequestBenchmark(prBenchmarkRequest, newJobFileName); Console.WriteLine($"Benchmark requested for PR #{pr.Number}. Waiting up to {BenchmarkTimeout} for results."); var results = await WaitForBenchmarkResults(newJobFileName); string FormatOutput(string stdout, string stderr) { return(string.IsNullOrEmpty(stderr) ? stdout : $"stdout: {results.BaselineStdout}\nstderr: {results.BaselineStderr}"); } var baselineOutput = FormatOutput(results.BaselineStdout, results.BaselineStderr); var prOutput = FormatOutput(results.PullRequestStdout, results.PullRequestStderr); var resultCommentText = string.Format(CompletedBenchmarkCommentTemplate, baselineOutput, prOutput); Console.WriteLine($"Benchmark results received for PR #{pr.Number}. Posting results to {pr.Url}."); Console.WriteLine($"Benchmark results comment: {resultCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, resultCommentText); } catch (Exception ex) { var errorCommentText = $"Failed to benchmark PR #{pr.Number}. Skipping... Details:\n```\n{ex}\n```"; Console.WriteLine($"Benchmark error comment: {errorCommentText}"); await client.Issue.Comment.Create(Owner, Repo, pr.Number, errorCommentText); } } Console.WriteLine($"Done scanning for benchmark requests."); return(0); }); return(app.Execute(args)); }