public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, ImmutableList <SourceLogEntry> log) { Left = left; Right = right; var leftResults = left.AllResults.ToList(); var rightResults = right.AllResults.ToList(); LeftOnly = leftResults.ExceptBy(rightResults, result => result.FullMethod); RightOnly = rightResults.ExceptBy(leftResults, result => result.FullMethod); var pairs = (from l in leftResults join r in rightResults on l.FullMethod equals r.FullMethod select new BenchmarkPair(l, r)).ToList(); LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList(); RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList(); bool leftEarlier = BenchmarkRepository.BuildForLabel(left.Label) < BenchmarkRepository.BuildForLabel(right.Label); var earlier = leftEarlier ? left : right; var later = leftEarlier ? right : left; var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label); var laterHash = BenchmarkRepository.HashForLabel(later.Label); LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList(); }
public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, ImmutableList<SourceLogEntry> log) { Left = left; Right = right; var leftResults = left.AllResults.ToList(); var rightResults = right.AllResults.ToList(); LeftOnly = leftResults.ExceptBy(rightResults, result => result.FullMethod); RightOnly = rightResults.ExceptBy(leftResults, result => result.FullMethod); var pairs = (from l in leftResults join r in rightResults on l.FullMethod equals r.FullMethod select new BenchmarkPair(l, r)).ToList(); LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList(); RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList(); bool leftEarlier = BenchmarkRepository.BuildForLabel(left.Label) < BenchmarkRepository.BuildForLabel(right.Label); var earlier = leftEarlier ? left : right; var later = leftEarlier ? right : left; var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label); var laterHash = BenchmarkRepository.HashForLabel(later.Label); LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList(); }
public ActionResult BenchmarkRun(string machine, string label) { BenchmarkRun run = null; string nextLabel = null; // In descending label order, so earlier... foreach (var candidate in repository.RunsByMachine[machine]) { if (run != null) { nextLabel = candidate.Label; break; } if (candidate.Label == label) { run = candidate; } } ImmutableList <SourceLogEntry> changes = null; if (nextLabel != null) { string earlierHash = BenchmarkRepository.HashForLabel(nextLabel); string thisHash = BenchmarkRepository.HashForLabel(label); changes = log.EntriesBetween(earlierHash, thisHash).ToImmutableList(); } return(View(new BenchmarkRunAndSourceLogs(run, changes))); }
public void ShouldDisposeAllCounters() { var testCollector1 = new TestMetricCollector(new CounterMetricName("foo"), "bar"); var measureBucket1 = new MeasureBucket(testCollector1); var testCollector2 = new TestMetricCollector(new CounterMetricName("foo"), "bar"); var measureBucket2 = new MeasureBucket(testCollector2); var benchmarkRun = new BenchmarkRun(new List <MeasureBucket>(new[] { measureBucket1, measureBucket2 }), new List <Counter>()); Assert.False(testCollector1.WasDisposed); Assert.False(testCollector2.WasDisposed); benchmarkRun.Dispose(); Assert.True(testCollector1.WasDisposed); Assert.True(testCollector2.WasDisposed); }
public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, MercurialLog log) { Left = left; Right = right; LeftOnly = left.Results.ExceptBy(right.Results, result => result.FullyQualifiedMethod); RightOnly = right.Results.ExceptBy(left.Results, result => result.FullyQualifiedMethod); var pairs = (from l in left.Results join r in right.Results on l.FullyQualifiedMethod equals r.FullyQualifiedMethod select new BenchmarkPair(l, r)).ToList(); LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList(); RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList(); var earlier = left.StartTime < right.StartTime ? left : right; var later = left.StartTime < right.StartTime ? right : left; var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label); var laterHash = BenchmarkRepository.HashForLabel(later.Label); LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList(); }
public void AddRun(BenchmarkRun run) { runs.Add(run); }
/// <summary> /// Pre-allocate all of the objects we're going to need for this benchmark /// </summary> private void Allocate() { _currentRun = Builder.NewRun(WarmupData); }
public BenchmarkRunAndSourceLogs(BenchmarkRun benchmarkRun, ImmutableList<SourceLogEntry> changes) { this.Run = benchmarkRun; this.Changes = changes; }
public Entry(BenchmarkRun run, string description) { this.Run = run; this.Description = description; }
private BenchmarkRun?GetPreviousRun(BenchmarkRun run) => repository.GetEnvironment(run.BenchmarkEnvironmentId)?.Runs .SkipWhile(r => r.BenchmarkRunId != run.BenchmarkRunId) .Skip(1) .FirstOrDefault();
public BenchmarkRunAndSourceLogs(BenchmarkRun benchmarkRun, ImmutableList <SourceLogEntry> changes) { this.Run = benchmarkRun; this.Changes = changes; }
public BenchmarkRunAndMercurialLogs(BenchmarkRun benchmarkRun, ImmutableList <MercurialLogEntry> changes) { this.Run = benchmarkRun; this.Changes = changes; }
private static void ProcessRun(BenchmarkRepository environmentRepository, string runDirectory) { var outputFile = Path.Combine(runDirectory, "benchmarks.pb"); // If the output file already exists, assume we're done. if (File.Exists(outputFile)) { return; } var startFile = Path.Combine(runDirectory, "start.txt"); var endFile = Path.Combine(runDirectory, "end.txt"); var start = DateTimeOffset.ParseExact(File.ReadAllText(startFile).Trim(), "yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture); var end = DateTimeOffset.ParseExact(File.ReadAllText(endFile).Trim(), "yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture); var tfm = Path.GetFileName(runDirectory); var commit = Path.GetFileName(Path.GetDirectoryName(runDirectory)); var models = Directory.GetFiles(runDirectory, "*.json") .Select(file => JsonConvert.DeserializeObject <BriefJsonModel>(File.ReadAllText(file))) .Where(m => m.Benchmarks.Any()) .ToList(); if (!models.Any()) { // TODO: Throw? return; } var runKey = Guid.NewGuid().ToString(); File.WriteAllText(outputFile, runKey); var hostEnvironment = models.First().HostEnvironmentInfo; var environment = new BenchmarkEnvironment { Machine = Environment.MachineName.ToLowerInvariant(), OperatingSystem = hostEnvironment.OsVersion?.GetValue() ?? "", Processor = hostEnvironment.ProcessorName?.GetValue() ?? "", ProcessorCount = hostEnvironment.ProcessorCount, TargetFramework = tfm, JitModules = hostEnvironment.JitModules, HasRyuJit = hostEnvironment.HasRyuJit, Architecture = hostEnvironment.Architecture, RuntimeVersion = hostEnvironment.RuntimeVersion }; var environmentId = environmentRepository.GetOrAddEnvironmentId(environment); var run = new BenchmarkRun { BenchmarkRunId = Guid.NewGuid().ToString(), BenchmarkEnvironmentId = environmentId, Commit = commit, Start = start.ToTimestamp(), End = end.ToTimestamp(), BenchmarkDotNetVersion = hostEnvironment.BenchmarkDotNetVersion, }; var types = new List <BenchmarkType>(); foreach (var model in models) { var firstBenchmark = model.Benchmarks.First(); var typeId = Guid.NewGuid().ToString(); types.Add(new BenchmarkType { BenchmarkRunId = run.BenchmarkRunId, BenchmarkTypeId = typeId, FullTypeName = $"{firstBenchmark.Namespace}.{firstBenchmark.Type}", Namespace = firstBenchmark.Namespace, Type = firstBenchmark.Type, Benchmarks = { model.Benchmarks.Select(b => CreateBenchmark(b, typeId)).OrderBy(b => b.Method) } }); } run.Types_.AddRange(types.OrderBy(t => t.FullTypeName)); File.WriteAllBytes(outputFile, run.ToByteArray()); environmentRepository.AddRun(run); Console.WriteLine($"Created benchmark file for {commit} / {tfm}"); }
public CreateBenchmarkRun() { InitializeComponent(); Run = new BenchmarkRun(); }