public static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, string referencePath, bool withConsole = true)
{
var referenceResultText = File.ReadAllText(referencePath);
var referenceResult = JsonConvert.DeserializeObject <BenchmarkScenarioResult>(referenceResultText);
return(IsEqualToReference(scenarioResult, referenceResult, withConsole));
}
public void SaveResultDefaultLocation(BenchmarkScenarioResult scenarioResult, string name = null, string directory = "BenchmarkResults/", bool withDatetime = true)
{
// TODO: ugly?
var datetime = withDatetime ? new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds() + "_" : "";
var path = Path.Combine(directory, $"{datetime}{name ?? scenarioResult.Name ?? string.Empty}.json");
SaveResult(scenarioResult, path);
}
private static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, BenchmarkScenarioResult referenceResult, bool withConsole = true)
{
if (withConsole)
{
Console.WriteLine();
Console.WriteLine("Compare to reference result");
}
var allEqual = true;
foreach (var benchmarkResult in scenarioResult.BenchmarkResults)
{
var referenceBenchmarkResult = referenceResult.BenchmarkResults.SingleOrDefault(x => x.InputName == benchmarkResult.InputName);
if (referenceBenchmarkResult == null)
{
Console.WriteLine($"{benchmarkResult.InputName} - equal: not found");
continue;
}
var runsEqual = RunsEqual(benchmarkResult.Runs.Cast <IGeneratorRun>().ToList(), referenceBenchmarkResult.Runs.Cast <IGeneratorRun>().ToList()); // TODO: ugly
var averageTime = benchmarkResult.Runs.Average(x => x.Time);
var referenceAverageTime = referenceBenchmarkResult.Runs.Average(x => x.Time);
var averageIterations = benchmarkResult.Runs.Average(x => x.Iterations) / 1000;
var referenceAverageIterations = referenceBenchmarkResult.Runs.Average(x => x.Iterations) / 1000;
if (withConsole)
{
Console.WriteLine($"{benchmarkResult.InputName} - equal: {runsEqual}, time average {referenceAverageTime / 1000:##.00}s -> {averageTime / 1000:##.00}s, iterations average {referenceAverageIterations:F}k -> {averageIterations:F}k");
}
if (!runsEqual)
{
allEqual = false;
}
}
if (withConsole)
{
var originalColor = Console.ForegroundColor;
Console.WriteLine();
Console.ForegroundColor = allEqual ? ConsoleColor.DarkGreen : ConsoleColor.DarkRed;
Console.WriteLine($"All equal: {allEqual}");
Console.ForegroundColor = originalColor;
}
return(allEqual);
}
public ManualResult(BenchmarkScenarioResult scenarioResult, ManualInfo manualInfo)
{
if (scenarioResult == null)
{
throw new ArgumentNullException(nameof(scenarioResult));
}
if (manualInfo == null)
{
throw new ArgumentNullException(nameof(manualInfo));
}
InputResults = scenarioResult.BenchmarkResults;
Name = manualInfo.Name ?? scenarioResult.Name ?? throw new ArgumentNullException(nameof(Name));
Group = manualInfo.Group ?? throw new ArgumentNullException(nameof(Group));
}
public async Task UploadManualResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, ManualInfo manualInfo)
{
if (scenarioResult == null)
{
throw new ArgumentNullException(nameof(scenarioResult));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (manualInfo == null)
{
throw new ArgumentNullException(nameof(manualInfo));
}
await UploadResult(new ManualResult(scenarioResult, manualInfo), config);
}
public async Task UploadCommitResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, CommitInfo commitInfo)
{
if (scenarioResult == null)
{
throw new ArgumentNullException(nameof(scenarioResult));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (commitInfo == null)
{
throw new ArgumentNullException(nameof(commitInfo));
}
await UploadResult(new CommitResult(scenarioResult, commitInfo), config);
}
public void SaveResult(BenchmarkScenarioResult scenarioResult, string path)
{
if (scenarioResult == null)
{
throw new ArgumentNullException(nameof(scenarioResult));
}
var json = JsonConvert.SerializeObject(scenarioResult, Formatting.Indented);
var directory = Path.GetDirectoryName(path);
if (directory != null)
{
Directory.CreateDirectory(directory);
}
File.WriteAllText(path, json);
}
public CommitResult(BenchmarkScenarioResult scenarioResult, CommitInfo commitInfo)
{
if (scenarioResult == null)
{
throw new ArgumentNullException(nameof(scenarioResult));
}
if (commitInfo == null)
{
throw new ArgumentNullException(nameof(commitInfo));
}
InputResults = scenarioResult.BenchmarkResults;
Commit = commitInfo.Commit;
CommitMessage = commitInfo.CommitMessage;
Branch = commitInfo.Branch;
BuildNumber = commitInfo.BuildNumber;
PullRequest = commitInfo.PullRequest;
}
public static List <double> GetTimeDifferences(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, double successRateThreshold = 0d)
{
var times = result.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList();
var timesReference = referenceResult.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList();
for (var i = 0; i < result.BenchmarkResults.Count; i++)
{
var benchmarkResult = result.BenchmarkResults[i];
var benchmarkResultReference = referenceResult.BenchmarkResults[i];
var successRate = benchmarkResult.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResult.Runs.Count;
var successRateReference = benchmarkResultReference.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResultReference.Runs.Count;
if (successRate < successRateThreshold && successRateReference < successRateThreshold)
{
times[i] = 1;
timesReference[i] = 1;
}
}
var differences = times.Zip(timesReference, StatisticsUtils.DifferenceToReference).ToList();
return(differences);
}
public static BoxPlotValues GetBoxPlotValues(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, bool excludeOutliers = true)
{
var timeDifferences = GetTimeDifferences(result, referenceResult);
return(GetBoxPlotValues(timeDifferences, excludeOutliers));
}
public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group)
{
if (resultSaver == null)
{
throw new ArgumentNullException(nameof(resultSaver));
}
if (name == null)
{
throw new ArgumentNullException(nameof(name));
}
if (group == null)
{
throw new ArgumentNullException(nameof(group));
}
resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}");
var uploadConfig = GetDefaultUploadConfig();
await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo()
{
Group = group,
Name = name,
});
}
