public static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, string referencePath, bool withConsole = true) { var referenceResultText = File.ReadAllText(referencePath); var referenceResult = JsonConvert.DeserializeObject <BenchmarkScenarioResult>(referenceResultText); return(IsEqualToReference(scenarioResult, referenceResult, withConsole)); }
public void SaveResultDefaultLocation(BenchmarkScenarioResult scenarioResult, string name = null, string directory = "BenchmarkResults/", bool withDatetime = true) { // TODO: ugly? var datetime = withDatetime ? new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds() + "_" : ""; var path = Path.Combine(directory, $"{datetime}{name ?? scenarioResult.Name ?? string.Empty}.json"); SaveResult(scenarioResult, path); }
private static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, BenchmarkScenarioResult referenceResult, bool withConsole = true) { if (withConsole) { Console.WriteLine(); Console.WriteLine("Compare to reference result"); } var allEqual = true; foreach (var benchmarkResult in scenarioResult.BenchmarkResults) { var referenceBenchmarkResult = referenceResult.BenchmarkResults.SingleOrDefault(x => x.InputName == benchmarkResult.InputName); if (referenceBenchmarkResult == null) { Console.WriteLine($"{benchmarkResult.InputName} - equal: not found"); continue; } var runsEqual = RunsEqual(benchmarkResult.Runs.Cast <IGeneratorRun>().ToList(), referenceBenchmarkResult.Runs.Cast <IGeneratorRun>().ToList()); // TODO: ugly var averageTime = benchmarkResult.Runs.Average(x => x.Time); var referenceAverageTime = referenceBenchmarkResult.Runs.Average(x => x.Time); var averageIterations = benchmarkResult.Runs.Average(x => x.Iterations) / 1000; var referenceAverageIterations = referenceBenchmarkResult.Runs.Average(x => x.Iterations) / 1000; if (withConsole) { Console.WriteLine($"{benchmarkResult.InputName} - equal: {runsEqual}, time average {referenceAverageTime / 1000:##.00}s -> {averageTime / 1000:##.00}s, iterations average {referenceAverageIterations:F}k -> {averageIterations:F}k"); } if (!runsEqual) { allEqual = false; } } if (withConsole) { var originalColor = Console.ForegroundColor; Console.WriteLine(); Console.ForegroundColor = allEqual ? ConsoleColor.DarkGreen : ConsoleColor.DarkRed; Console.WriteLine($"All equal: {allEqual}"); Console.ForegroundColor = originalColor; } return(allEqual); }
public ManualResult(BenchmarkScenarioResult scenarioResult, ManualInfo manualInfo) { if (scenarioResult == null) { throw new ArgumentNullException(nameof(scenarioResult)); } if (manualInfo == null) { throw new ArgumentNullException(nameof(manualInfo)); } InputResults = scenarioResult.BenchmarkResults; Name = manualInfo.Name ?? scenarioResult.Name ?? throw new ArgumentNullException(nameof(Name)); Group = manualInfo.Group ?? throw new ArgumentNullException(nameof(Group)); }
public async Task UploadManualResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, ManualInfo manualInfo) { if (scenarioResult == null) { throw new ArgumentNullException(nameof(scenarioResult)); } if (config == null) { throw new ArgumentNullException(nameof(config)); } if (manualInfo == null) { throw new ArgumentNullException(nameof(manualInfo)); } await UploadResult(new ManualResult(scenarioResult, manualInfo), config); }
public async Task UploadCommitResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, CommitInfo commitInfo) { if (scenarioResult == null) { throw new ArgumentNullException(nameof(scenarioResult)); } if (config == null) { throw new ArgumentNullException(nameof(config)); } if (commitInfo == null) { throw new ArgumentNullException(nameof(commitInfo)); } await UploadResult(new CommitResult(scenarioResult, commitInfo), config); }
public void SaveResult(BenchmarkScenarioResult scenarioResult, string path) { if (scenarioResult == null) { throw new ArgumentNullException(nameof(scenarioResult)); } var json = JsonConvert.SerializeObject(scenarioResult, Formatting.Indented); var directory = Path.GetDirectoryName(path); if (directory != null) { Directory.CreateDirectory(directory); } File.WriteAllText(path, json); }
public CommitResult(BenchmarkScenarioResult scenarioResult, CommitInfo commitInfo) { if (scenarioResult == null) { throw new ArgumentNullException(nameof(scenarioResult)); } if (commitInfo == null) { throw new ArgumentNullException(nameof(commitInfo)); } InputResults = scenarioResult.BenchmarkResults; Commit = commitInfo.Commit; CommitMessage = commitInfo.CommitMessage; Branch = commitInfo.Branch; BuildNumber = commitInfo.BuildNumber; PullRequest = commitInfo.PullRequest; }
public static List <double> GetTimeDifferences(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, double successRateThreshold = 0d) { var times = result.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList(); var timesReference = referenceResult.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList(); for (var i = 0; i < result.BenchmarkResults.Count; i++) { var benchmarkResult = result.BenchmarkResults[i]; var benchmarkResultReference = referenceResult.BenchmarkResults[i]; var successRate = benchmarkResult.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResult.Runs.Count; var successRateReference = benchmarkResultReference.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResultReference.Runs.Count; if (successRate < successRateThreshold && successRateReference < successRateThreshold) { times[i] = 1; timesReference[i] = 1; } } var differences = times.Zip(timesReference, StatisticsUtils.DifferenceToReference).ToList(); return(differences); }
public static BoxPlotValues GetBoxPlotValues(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, bool excludeOutliers = true) { var timeDifferences = GetTimeDifferences(result, referenceResult); return(GetBoxPlotValues(timeDifferences, excludeOutliers)); }
public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group) { if (resultSaver == null) { throw new ArgumentNullException(nameof(resultSaver)); } if (name == null) { throw new ArgumentNullException(nameof(name)); } if (group == null) { throw new ArgumentNullException(nameof(group)); } resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}"); var uploadConfig = GetDefaultUploadConfig(); await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo() { Group = group, Name = name, }); }