Exemplo n.º 1
0
        public static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, string referencePath, bool withConsole = true)
        {
            var referenceResultText = File.ReadAllText(referencePath);
            var referenceResult     = JsonConvert.DeserializeObject <BenchmarkScenarioResult>(referenceResultText);

            return(IsEqualToReference(scenarioResult, referenceResult, withConsole));
        }
Exemplo n.º 2
0
        public void SaveResultDefaultLocation(BenchmarkScenarioResult scenarioResult, string name = null, string directory = "BenchmarkResults/", bool withDatetime = true)
        {
            // TODO: ugly?
            var datetime = withDatetime ? new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds() + "_" : "";
            var path     = Path.Combine(directory, $"{datetime}{name ?? scenarioResult.Name ?? string.Empty}.json");

            SaveResult(scenarioResult, path);
        }
Exemplo n.º 3
0
        private static bool IsEqualToReference(BenchmarkScenarioResult scenarioResult, BenchmarkScenarioResult referenceResult, bool withConsole = true)
        {
            if (withConsole)
            {
                Console.WriteLine();
                Console.WriteLine("Compare to reference result");
            }

            var allEqual = true;

            foreach (var benchmarkResult in scenarioResult.BenchmarkResults)
            {
                var referenceBenchmarkResult = referenceResult.BenchmarkResults.SingleOrDefault(x => x.InputName == benchmarkResult.InputName);

                if (referenceBenchmarkResult == null)
                {
                    Console.WriteLine($"{benchmarkResult.InputName} - equal: not found");
                    continue;
                }

                var runsEqual = RunsEqual(benchmarkResult.Runs.Cast <IGeneratorRun>().ToList(), referenceBenchmarkResult.Runs.Cast <IGeneratorRun>().ToList()); // TODO: ugly

                var averageTime          = benchmarkResult.Runs.Average(x => x.Time);
                var referenceAverageTime = referenceBenchmarkResult.Runs.Average(x => x.Time);

                var averageIterations          = benchmarkResult.Runs.Average(x => x.Iterations) / 1000;
                var referenceAverageIterations = referenceBenchmarkResult.Runs.Average(x => x.Iterations) / 1000;

                if (withConsole)
                {
                    Console.WriteLine($"{benchmarkResult.InputName} - equal: {runsEqual}, time average {referenceAverageTime / 1000:##.00}s -> {averageTime / 1000:##.00}s, iterations average {referenceAverageIterations:F}k -> {averageIterations:F}k");
                }

                if (!runsEqual)
                {
                    allEqual = false;
                }
            }

            if (withConsole)
            {
                var originalColor = Console.ForegroundColor;
                Console.WriteLine();
                Console.ForegroundColor = allEqual ? ConsoleColor.DarkGreen : ConsoleColor.DarkRed;
                Console.WriteLine($"All equal: {allEqual}");
                Console.ForegroundColor = originalColor;
            }

            return(allEqual);
        }
Exemplo n.º 4
0
        public ManualResult(BenchmarkScenarioResult scenarioResult, ManualInfo manualInfo)
        {
            if (scenarioResult == null)
            {
                throw new ArgumentNullException(nameof(scenarioResult));
            }
            if (manualInfo == null)
            {
                throw new ArgumentNullException(nameof(manualInfo));
            }

            InputResults = scenarioResult.BenchmarkResults;
            Name         = manualInfo.Name ?? scenarioResult.Name ?? throw new ArgumentNullException(nameof(Name));
            Group        = manualInfo.Group ?? throw new ArgumentNullException(nameof(Group));
        }
Exemplo n.º 5
0
        public async Task UploadManualResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, ManualInfo manualInfo)
        {
            if (scenarioResult == null)
            {
                throw new ArgumentNullException(nameof(scenarioResult));
            }
            if (config == null)
            {
                throw new ArgumentNullException(nameof(config));
            }
            if (manualInfo == null)
            {
                throw new ArgumentNullException(nameof(manualInfo));
            }

            await UploadResult(new ManualResult(scenarioResult, manualInfo), config);
        }
Exemplo n.º 6
0
        public async Task UploadCommitResult(BenchmarkScenarioResult scenarioResult, UploadConfig config, CommitInfo commitInfo)
        {
            if (scenarioResult == null)
            {
                throw new ArgumentNullException(nameof(scenarioResult));
            }
            if (config == null)
            {
                throw new ArgumentNullException(nameof(config));
            }
            if (commitInfo == null)
            {
                throw new ArgumentNullException(nameof(commitInfo));
            }

            await UploadResult(new CommitResult(scenarioResult, commitInfo), config);
        }
Exemplo n.º 7
0
        public void SaveResult(BenchmarkScenarioResult scenarioResult, string path)
        {
            if (scenarioResult == null)
            {
                throw new ArgumentNullException(nameof(scenarioResult));
            }

            var json      = JsonConvert.SerializeObject(scenarioResult, Formatting.Indented);
            var directory = Path.GetDirectoryName(path);

            if (directory != null)
            {
                Directory.CreateDirectory(directory);
            }

            File.WriteAllText(path, json);
        }
Exemplo n.º 8
0
        public CommitResult(BenchmarkScenarioResult scenarioResult, CommitInfo commitInfo)
        {
            if (scenarioResult == null)
            {
                throw new ArgumentNullException(nameof(scenarioResult));
            }
            if (commitInfo == null)
            {
                throw new ArgumentNullException(nameof(commitInfo));
            }

            InputResults  = scenarioResult.BenchmarkResults;
            Commit        = commitInfo.Commit;
            CommitMessage = commitInfo.CommitMessage;
            Branch        = commitInfo.Branch;
            BuildNumber   = commitInfo.BuildNumber;
            PullRequest   = commitInfo.PullRequest;
        }
Exemplo n.º 9
0
        public static List <double> GetTimeDifferences(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, double successRateThreshold = 0d)
        {
            var times          = result.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList();
            var timesReference = referenceResult.BenchmarkResults.Select(x => x.Runs.Average(y => y.Time)).ToList();

            for (var i = 0; i < result.BenchmarkResults.Count; i++)
            {
                var benchmarkResult          = result.BenchmarkResults[i];
                var benchmarkResultReference = referenceResult.BenchmarkResults[i];

                var successRate          = benchmarkResult.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResult.Runs.Count;
                var successRateReference = benchmarkResultReference.Runs.Count(x => x.IsSuccessful) / (double)benchmarkResultReference.Runs.Count;

                if (successRate < successRateThreshold && successRateReference < successRateThreshold)
                {
                    times[i]          = 1;
                    timesReference[i] = 1;
                }
            }

            var differences = times.Zip(timesReference, StatisticsUtils.DifferenceToReference).ToList();

            return(differences);
        }
Exemplo n.º 10
0
        public static BoxPlotValues GetBoxPlotValues(BenchmarkScenarioResult result, BenchmarkScenarioResult referenceResult, bool excludeOutliers = true)
        {
            var timeDifferences = GetTimeDifferences(result, referenceResult);

            return(GetBoxPlotValues(timeDifferences, excludeOutliers));
        }
Exemplo n.º 11
0
        public static async Task SaveAndUpload(this BenchmarkResultSaver resultSaver, BenchmarkScenarioResult scenarioResult, string name, string group)
        {
            if (resultSaver == null)
            {
                throw new ArgumentNullException(nameof(resultSaver));
            }
            if (name == null)
            {
                throw new ArgumentNullException(nameof(name));
            }
            if (group == null)
            {
                throw new ArgumentNullException(nameof(group));
            }

            resultSaver.SaveResultDefaultLocation(scenarioResult, $"{group}_{name}");

            var uploadConfig = GetDefaultUploadConfig();
            await resultSaver.UploadManualResult(scenarioResult, uploadConfig, new ManualInfo()
            {
                Group = group,
                Name  = name,
            });
        }