public void PercentileValuesTest() { var summary = new Statistics(Enumerable.Range(1, 30)); Print(summary); Assert.Equal(1, summary.Percentiles.P0); Assert.Equal(8.25, summary.Percentiles.P25); Assert.Equal(15.5, summary.Percentiles.P50); Assert.Equal(20.43, summary.Percentiles.P67, 4); Assert.Equal(24.2, summary.Percentiles.P80, 4); Assert.Equal(25.65, summary.Percentiles.P85); Assert.Equal(27.1, summary.Percentiles.P90); Assert.Equal(28.55, summary.Percentiles.P95, 4); Assert.Equal(30, summary.Percentiles.P100); var a = Enumerable.Range(1, 30); var b = Enumerable.Concat(Enumerable.Repeat(0, 30), a); var c = Enumerable.Concat(b, Enumerable.Repeat(31, 30)); summary = new Statistics(c); Print(summary); Assert.Equal(0, summary.Percentiles.P0); Assert.Equal(0, summary.Percentiles.P25); Assert.Equal(15.5, summary.Percentiles.P50); Assert.Equal(30.63, summary.Percentiles.P67, 4); Assert.Equal(31, summary.Percentiles.P80); Assert.Equal(31, summary.Percentiles.P85); Assert.Equal(31, summary.Percentiles.P90); Assert.Equal(31, summary.Percentiles.P95); Assert.Equal(31, summary.Percentiles.P100); }
private string Format(Statistics statistics, TimeUnit timeUnit) { if (statistics == null) return "NA"; var value = calc(statistics); return isTimeColumn ? value.ToTimeStr(timeUnit) : value.ToStr(); }
public void ConfidenceIntervalTest() { var summary = new Statistics(Enumerable.Range(1, 30)); Print(summary); Assert.Equal(95, summary.ConfidenceInterval.Level.ToPercent()); Assert.Equal(15.5, summary.ConfidenceInterval.Mean); Assert.Equal(summary.StandardError, summary.ConfidenceInterval.Error); Assert.Equal(12.34974, summary.ConfidenceInterval.Lower, 4); Assert.Equal(18.65026, summary.ConfidenceInterval.Upper, 4); }
public void RankTest() { var s1 = new Statistics(100, 101, 100, 101); var s2 = new Statistics(300, 301, 300, 301); var s3 = new Statistics(200.3279, 200.3178, 200.4046); var s4 = new Statistics(200.2298, 200.5738, 200.3582); var s5 = new Statistics(195, 196, 195, 196); var actualRanks = RankHelper.GetRanks(s1, s2, s3, s4, s5); var expectedRanks = new[] { 1, 4, 3, 3, 2 }; Assert.Equal(expectedRanks, actualRanks); }
private static int[] GetPlaces(Statistics[] s) { var n = s.Length; int[] places = new int[n]; places[0] = 1; for (int i = 1; i < n; i++) if (LookSame(s[i - 1], s[i])) places[i] = places[i - 1]; else places[i] = places[i - 1] + 1; return places; }
/// <summary> /// Welch's Two Sample t-test /// </summary> public static WelchTTest Calc(Statistics x, Statistics y) { int n1 = x.N, n2 = y.N; double v1 = x.Variance, v2 = y.Variance, m1 = x.Mean, m2 = y.Mean; var se = Sqrt((v1 / n1) + (v2 / n2)); var t = (m1 - m2) / se; var df = (v1 / n1 + v2 / n2).Sqr() / ((v1 / n1).Sqr() / (n1 - 1) + (v2 / n2).Sqr() / (n2 - 1)); var pValue = MathHelper.Student(t, df); return new WelchTTest(t, df, pValue); }
private void Print(Statistics summary) { output.WriteLine("Min = " + summary.Min); output.WriteLine("LowerFence = " + summary.LowerFence); output.WriteLine("Q1 = " + summary.Q1); output.WriteLine("Median = " + summary.Median); output.WriteLine("Mean = " + summary.Mean); output.WriteLine("Q3 = " + summary.Q3); output.WriteLine("UpperFence = " + summary.UpperFence); output.WriteLine("Max = " + summary.Max); output.WriteLine("InterquartileRange = " + summary.InterquartileRange); output.WriteLine("StandardDeviation = " + summary.StandardDeviation); output.WriteLine("Outlier = [" + string.Join("; ", summary.Outliers) + "]"); output.WriteLine("CI = " + summary.ConfidenceInterval.ToStr()); }
public void Test2() { var summary = new Statistics(1, 2); Print(summary); Assert.Equal(1, summary.Min); Assert.Equal(-0.5, summary.LowerFence); Assert.Equal(1, summary.Q1); Assert.Equal(1.5, summary.Median); Assert.Equal(1.5, summary.Mean); Assert.Equal(2, summary.Q3); Assert.Equal(3.5, summary.UpperFence); Assert.Equal(2, summary.Max); Assert.Equal(1, summary.InterquartileRange); Assert.Equal(0.70711, summary.StandardDeviation, 4); Assert.Equal(new double[0], summary.Outliers); }
public void Test3() { var summary = new Statistics(1, 2, 4); Print(summary); Assert.Equal(1, summary.Min); Assert.Equal(-3.5, summary.LowerFence); Assert.Equal(1, summary.Q1); Assert.Equal(2, summary.Median); Assert.Equal(2.333333, summary.Mean, 5); Assert.Equal(4, summary.Q3); Assert.Equal(8.5, summary.UpperFence); Assert.Equal(4, summary.Max); Assert.Equal(3, summary.InterquartileRange); Assert.Equal(1.52753, summary.StandardDeviation, 4); Assert.Equal(new double[0], summary.Outliers); }
/// <summary> /// Welch's Two Sample t-test /// </summary> public static WelchTTest Calc(Statistics x, Statistics y) { int n1 = x.N, n2 = y.N; if (x.N < 2) throw new ArgumentException("x should contains at least 2 elements", nameof(x)); if (y.N < 2) throw new ArgumentException("y should contains at least 2 elements", nameof(y)); double v1 = x.Variance, v2 = y.Variance, m1 = x.Mean, m2 = y.Mean; double se = Sqrt((v1 / n1) + (v2 / n2)); double t = (m1 - m2) / se; double df = (v1 / n1 + v2 / n2).Sqr() / ((v1 / n1).Sqr() / (n1 - 1) + (v2 / n2).Sqr() / (n2 - 1)); double pValue = MathHelper.Student(t, df); return new WelchTTest(t, df, pValue); }
// TODO: rewrite without allocations public IEnumerable<Measurement> GetMeasurements() { double overhead = Idle == null ? 0.0 : new Statistics(Idle.Select(m => m.Nanoseconds)).Mean; var mainStats = new Statistics(Main.Select(m => m.Nanoseconds)); int resultIndex = 0; foreach (var measurement in Main) { if (removeOutliers && mainStats.IsOutlier(measurement.Nanoseconds)) continue; double value = Math.Max(0, measurement.Nanoseconds - overhead); if (IsSuspiciouslySmall(value)) value = 0; yield return new Measurement( measurement.LaunchIndex, IterationMode.Result, ++resultIndex, measurement.Operations, value); } }
public void Test1() { var summary = new Statistics(1); Print(summary); Assert.Equal(1, summary.Min); Assert.Equal(1, summary.LowerFence); Assert.Equal(1, summary.Q1); Assert.Equal(1, summary.Median); Assert.Equal(1, summary.Mean); Assert.Equal(1, summary.Q3); Assert.Equal(1, summary.UpperFence); Assert.Equal(1, summary.Max); Assert.Equal(0, summary.InterquartileRange); Assert.Equal(0, summary.StandardDeviation); Assert.Equal(new double[0], summary.Outliers); Assert.Equal(1, summary.Percentiles.P0); Assert.Equal(1, summary.Percentiles.P25); Assert.Equal(1, summary.Percentiles.P50); Assert.Equal(1, summary.Percentiles.P85); Assert.Equal(1, summary.Percentiles.P95); Assert.Equal(1, summary.Percentiles.P100); }
/// <summary> /// Mean for [X*Y]. /// </summary> public static double MulMean(Statistics x, Statistics y) => x.Mean * y.Mean;
public void Test7() { var summary = new Statistics(1, 2, 4, 8, 16, 32, 64); Print(summary); Assert.Equal(1, summary.Min); Assert.Equal(-43, summary.LowerFence); Assert.Equal(2, summary.Q1); Assert.Equal(8, summary.Median); Assert.Equal(18.1428571429, summary.Mean, 5); Assert.Equal(32, summary.Q3); Assert.Equal(77, summary.UpperFence); Assert.Equal(64, summary.Max); Assert.Equal(30, summary.InterquartileRange); Assert.Equal(22.9378, summary.StandardDeviation, 4); Assert.Equal(new double[0], summary.Outliers); }
private static bool LookSame(Statistics s1, Statistics s2) => s1.Mean + 3*s1.StandardDeviation > s2.Mean && s2.Mean - 3*s1.StandardDeviation < s1.Mean;
/// <summary> /// Variance for [X*Y]. /// </summary> public static double MulVariance(Statistics x, Statistics y) { return x.Sqr().Mean * y.Sqr().Mean - x.Mean.Sqr() * y.Mean.Sqr(); }
public void OutlierTest() { var summary = new Statistics(1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 10, 10.1); Print(summary); Assert.Equal(new[] { 10, 10.1 }, summary.Outliers); }
/// <summary> /// Variance for [X/Y]. /// </summary> public static double DivVariance(Statistics x, Statistics y) { var yInvert = y.Invert(); if (yInvert == null) throw new DivideByZeroException(); return MulVariance(x, yInvert); }
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config, IResolver resolver) { var executeResults = new List<ExecuteResult>(); logger.WriteLineInfo("// *** Execute ***"); bool analyzeRunToRunVariance = benchmark.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver); bool autoLaunchCount = !benchmark.Job.HasValue(RunMode.LaunchCountCharacteristic); int defaultValue = analyzeRunToRunVariance ? 2 : 1; int launchCount = Math.Max( 1, autoLaunchCount ? defaultValue: benchmark.Job.Run.LaunchCount); for (int launchIndex = 0; launchIndex < launchCount; launchIndex++) { string printedLaunchCount = (analyzeRunToRunVariance && autoLaunchCount && launchIndex < 2) ? "" : " / " + launchCount; logger.WriteLineInfo($"// Launch: {launchIndex + 1}{printedLaunchCount}"); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); if (executeResult.ExitCode != 0) logger.WriteLineError("ExitCode != 0"); executeResults.Add(executeResult); var measurements = executeResults .SelectMany(r => r.Data) .Select(line => Measurement.Parse(logger, line, 0)) .Where(r => r.IterationMode != IterationMode.Unknown). ToArray(); if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } if (autoLaunchCount && launchIndex == 1 && analyzeRunToRunVariance) { // TODO: improve this logic var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median; var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median; var percent = idleApprox / mainApprox * 100; launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula } } logger.WriteLine(); // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results if (config.GetDiagnosers().Any()) { logger.WriteLineInfo("// Run, Diagnostic"); var compositeDiagnoser = config.GetCompositeDiagnoser(); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, resolver, compositeDiagnoser); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); compositeDiagnoser.ProcessResults(benchmark, report); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); logger.WriteLine(); } return executeResults; }
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config) { var executeResults = new List<ExecuteResult>(); logger.WriteLineInfo("// *** Execute ***"); var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value); for (int processNumber = 0; processNumber < launchCount; processNumber++) { var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString(); logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}"); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); executeResults.Add(executeResult); var measurements = executeResults .SelectMany(r => r.Data) .Select(line => Measurement.Parse(logger, line, 0)) .Where(r => r.IterationMode != IterationMode.Unknown). ToArray(); if (!measurements.Any()) { // Something went wrong during the benchmark, don't bother doing more runs logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!"); break; } if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1) { var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median; var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median; var percent = idleApprox / mainApprox * 100; launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula } } logger.WriteLine(); // Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results if (config.GetDiagnosers().Count() > 0) { logger.WriteLineInfo($"// Run, Diagnostic"); config.GetCompositeDiagnoser().Start(benchmark); var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser()); var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList(); var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns); config.GetCompositeDiagnoser().Stop(benchmark, report); if (!executeResult.FoundExecutable) logger.WriteLineError("Executable not found"); logger.WriteLine(); } return executeResults; }
/// <summary> /// Variance for [X*Y]. /// </summary> public static double MulVariance(Statistics x, Statistics y) { return(x.Sqr().Mean *y.Sqr().Mean - x.Mean.Sqr() * y.Mean.Sqr()); }