private static async Task RequestBenchmark(PRBenchmarkRequest prBenchmarkRequest, string newJobFileName)
{
await using var baseJobStream = File.OpenRead(BaseJobPath);
var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream);
jsonDictionary[nameof(BuildInstructions)] = new BuildInstructions
{
BuildCommands = BaseBuildInstructions.BuildCommands,
ExtraDriverArgs = BaseBuildInstructions.ExtraDriverArgs,
PullRequestNumber = prBenchmarkRequest.PullRequest.Number,
BaselineSHA = prBenchmarkRequest.PullRequest.Base.Sha,
PullRequestSHA = prBenchmarkRequest.PullRequest.Head.Sha,
ScenarioName = prBenchmarkRequest.ScenarioName,
};
using var newJobStream = new MemoryStream();
await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions
{
WriteIndented = true,
});
newJobStream.Position = 0;
await JobFileSystem.WriteFile(newJobStream, newJobFileName);
}
private static void RunBuildCommands(BuildInstructions buildRules)
{
foreach (var buildCommand in buildRules.BuildCommands)
{
RunCommand(buildCommand);
}
}
private static void RunBuildCommands(BuildInstructions buildRules)
{
foreach (var buildCommand in buildRules.BuildCommands)
{
var splitCommand = buildCommand.Split(' ', 2);
Process.Start(splitCommand[0], splitCommand.Length == 2 ? splitCommand[1] : string.Empty).WaitForExit();
}
}
private static string GetDriverArguments(
string jobsFilePath,
string sessionId,
string sdkVersion,
BuildInstructions buildInstructions,
bool isBaseline)
{
var argumentsBuilder = new StringBuilder($"{DriverPath} --server {ServerUrl} --client {ClientUrl} --jobs {jobsFilePath} --session {sessionId}");
argumentsBuilder.Append(" --self-contained --aspNetCoreVersion Latest --runtimeVersion Latest --quiet");
if (!string.IsNullOrWhiteSpace(buildInstructions.ScenarioName) && !string.Equals("Default", buildInstructions.ScenarioName, StringComparison.OrdinalIgnoreCase))
{
argumentsBuilder.Append(" --scenario ");
argumentsBuilder.Append(buildInstructions.ScenarioName);
}
if (!string.IsNullOrWhiteSpace(sdkVersion))
{
argumentsBuilder.Append(" --sdk ");
argumentsBuilder.Append(sdkVersion);
}
if (!string.IsNullOrWhiteSpace(buildInstructions.ExtraDriverArgs))
{
argumentsBuilder.Append(" ");
argumentsBuilder.Append(buildInstructions.ExtraDriverArgs);
}
if (isBaseline)
{
argumentsBuilder.Append(" --save baseline --description Before");
}
else
{
argumentsBuilder.Append(" --diff baseline --description After");
}
return(argumentsBuilder.ToString());
}
private static async Task RequestBenchmark(PullRequest pr, string newJobFileName)
{
await using var baseJobStream = File.OpenRead(BaseJobPath);
var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(baseJobStream);
jsonDictionary["BuildInstructions"] = new BuildInstructions
{
BuildCommands = BuildCommands,
BaselineSHA = pr.Base.Sha,
PullRequestSHA = pr.Head.Sha,
};
using var newJobStream = new MemoryStream();
await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions
{
WriteIndented = true,
});
newJobStream.Position = 0;
await JobFileSystem.WriteFile(newJobStream, newJobFileName);
}
private static async Task <string[]> GetBuildCommands()
{
using var processingJsonStream = File.OpenRead(BaseJobPath);
using var jsonDocument = await JsonDocument.ParseAsync(processingJsonStream);
BuildInstructions buildInstructions = null;
foreach (var element in jsonDocument.RootElement.EnumerateObject())
{
if (element.NameEquals(nameof(BuildInstructions)))
{
buildInstructions = JsonSerializer.Deserialize <BuildInstructions>(element.Value.GetRawText());
break;
}
}
if (buildInstructions is null)
{
throw new InvalidDataException($"Job file {Path.GetFileName(BaseJobPath)} doesn't include a top-level '{nameof(BuildInstructions)}' property.");
}
return(buildInstructions.BuildCommands);
}
private static async Task RequestBenchmark(PullRequest pr, string session)
{
var baseFile = new FileInfo(BaseJobPath);
var newJobPath = Path.Combine(Path.GetTempPath(), $"{session}.{baseFile.Name}");
var newJobFile = new FileInfo(newJobPath);
baseFile.CopyTo(newJobPath);
try
{
using (var newJobStream = File.Open(newJobPath, System.IO.FileMode.Open))
{
var jsonDictionary = await JsonSerializer.DeserializeAsync <Dictionary <string, object> >(newJobStream);
jsonDictionary["BuildInstructions"] = new BuildInstructions
{
BuildCommands = BuildCommands,
BaselineSHA = pr.Base.Sha,
PullRequestSHA = pr.Head.Sha,
};
// Clear file and reset position to 0
newJobStream.SetLength(0);
await JsonSerializer.SerializeAsync(newJobStream, jsonDictionary, new JsonSerializerOptions
{
WriteIndented = true,
});
}
newJobFile.MoveTo(Path.Combine(JobsPath, newJobFile.Name));
}
catch
{
newJobFile.Delete();
throw;
}
}
public static int Main(string[] args)
{
var app = new CommandLineApplication();
app.HelpOption("-h|--help");
var baseJobPath = app.Option("-b|--base-job <PATH>", "The base job file path", CommandOptionType.SingleValue).IsRequired();
var jobsPath = app.Option("-j|--jobs-path <PATH>", "The path where jobs are created", CommandOptionType.SingleValue);
var azureStorageConnectionString = app.Option("-c|--azure-storage-connection-string <CONNECTIONSTRING>", "The Azure Storage connection string", CommandOptionType.SingleValue);
var azureStorageFileShareName = app.Option("-f|--azure-storage-file-share-name <NAME>", "The Azure Storage file share name", CommandOptionType.SingleValue);
var githubUser = app.Option("-u|--github-user <NAME>", "The GitHub user name for the bot", CommandOptionType.SingleValue);
var githubUserToken = app.Option("-t|--github-user-token <TOKEN>", "The GitHub token for the bot", CommandOptionType.SingleValue);
var githubAppId = app.Option("-a|--github-app-id <ID>", "The GitHub App ID for the bot", CommandOptionType.SingleValue);
var githubAppKeyPath = app.Option("-k|--github-app-key-file <PATH>", "The GitHub App pem file path", CommandOptionType.SingleValue);
var githubAppInstallationId = app.Option("-i|--github-app-install-id <ID>", "The GitHub App installation ID for the repo. E.g. 'https://github.com/settings/installations/{Id}'", CommandOptionType.SingleValue);
app.OnExecuteAsync(async cancellationToken =>
{
BaseJobPath = baseJobPath.Value();
GitHubClient client;
string botLoginName;
if (githubUser.HasValue())
{
if (!githubUserToken.HasValue())
{
Console.WriteLine("--github-user was provided with no --github-token.");
return(-1);
}
botLoginName = githubUser.Value();
client = GetClientForUser(botLoginName, githubUserToken.Value());
}
else if (githubAppId.HasValue())
{
if (!githubAppKeyPath.HasValue())
{
Console.WriteLine("--github-app-id was provided with no --github-app-key-file.");
return(-1);
}
if (!githubAppInstallationId.HasValue())
{
Console.WriteLine("--github-app-id was provided with no --github-app-install-id.");
return(-1);
}
if (!long.TryParse(githubAppInstallationId.Value(), out long installId))
{
Console.WriteLine("--github-app-install-id is not a valid long.");
}
botLoginName = AppName + "[bot]";
client = await GetClientForApp(githubAppId.Value(), githubAppKeyPath.Value(), installId);
}
else
{
Console.WriteLine("Cannot authenticate with GitHub. Neither a --github-user nor a --github-app-id has been provided.");
return(-1);
}
if (jobsPath.HasValue())
{
JobFileSystem = new LocalFileSystem(jobsPath.Value());
}
else if (azureStorageConnectionString.HasValue())
{
if (!azureStorageFileShareName.HasValue())
{
Console.WriteLine("--azure-storage-connection-string was provided with no --azure-storage-file-share.");
return(-1);
}
var cloudDir = await GetCloudFileDirectory(azureStorageConnectionString.Value(), azureStorageFileShareName.Value());
JobFileSystem = new AzureStorageFileSystem(cloudDir);
}
else
{
Console.WriteLine("Neither a --jobs-path nor an --azure-storage-connection-string has been provided.");
return(-1);
}
await JobFileSystem.CreateDirectoryIfNotExists(ProcessedDirectoryName);
BaseBuildInstructions = await GetBuildInstructions();
Console.WriteLine($"Scanning for benchmark requests in {Owner}/{Repo}.");
await foreach (var prBenchmarkRequest in GetPRsToBenchmark(client, botLoginName))
{
var pr = prBenchmarkRequest.PullRequest;
try
{
var session = Guid.NewGuid().ToString("n");
var newJobFileName = $"{session}.{Path.GetFileName(BaseJobPath)}";
var startingCommentText = string.Format(StartingBencmarkComment, prBenchmarkRequest.ScenarioName, session);
Console.WriteLine($"Requesting {prBenchmarkRequest.ScenarioName} benchmark for PR #{pr.Number}.");
Console.WriteLine($"Benchmark starting comment: {startingCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, startingCommentText);
await RequestBenchmark(prBenchmarkRequest, newJobFileName);
Console.WriteLine($"Benchmark requested for PR #{pr.Number}. Waiting up to {BenchmarkTimeout} for results.");
var results = await WaitForBenchmarkResults(newJobFileName);
string FormatOutput(string stdout, string stderr)
{
return(string.IsNullOrEmpty(stderr) ? stdout : $"stdout: {results.BaselineStdout}\nstderr: {results.BaselineStderr}");
}
var baselineOutput = FormatOutput(results.BaselineStdout, results.BaselineStderr);
var prOutput = FormatOutput(results.PullRequestStdout, results.PullRequestStderr);
var resultCommentText = string.Format(CompletedBenchmarkCommentTemplate, baselineOutput, prOutput);
Console.WriteLine($"Benchmark results received for PR #{pr.Number}. Posting results to {pr.Url}.");
Console.WriteLine($"Benchmark results comment: {resultCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, resultCommentText);
}
catch (Exception ex)
{
var errorCommentText = $"Failed to benchmark PR #{pr.Number}. Skipping... Details:\n```\n{ex}\n```";
Console.WriteLine($"Benchmark error comment: {errorCommentText}");
await client.Issue.Comment.Create(Owner, Repo, pr.Number, errorCommentText);
}
}
Console.WriteLine($"Done scanning for benchmark requests.");
return(0);
});
return(app.Execute(args));
}