public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, ImmutableList <SourceLogEntry> log)
{
Left = left;
Right = right;
var leftResults = left.AllResults.ToList();
var rightResults = right.AllResults.ToList();
LeftOnly = leftResults.ExceptBy(rightResults, result => result.FullMethod);
RightOnly = rightResults.ExceptBy(leftResults, result => result.FullMethod);
var pairs = (from l in leftResults
join r in rightResults on l.FullMethod equals r.FullMethod
select new BenchmarkPair(l, r)).ToList();
LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList();
RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList();
bool leftEarlier = BenchmarkRepository.BuildForLabel(left.Label) < BenchmarkRepository.BuildForLabel(right.Label);
var earlier = leftEarlier ? left : right;
var later = leftEarlier ? right : left;
var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label);
var laterHash = BenchmarkRepository.HashForLabel(later.Label);
LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList();
}
public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, ImmutableList<SourceLogEntry> log)
{
Left = left;
Right = right;
var leftResults = left.AllResults.ToList();
var rightResults = right.AllResults.ToList();
LeftOnly = leftResults.ExceptBy(rightResults, result => result.FullMethod);
RightOnly = rightResults.ExceptBy(leftResults, result => result.FullMethod);
var pairs = (from l in leftResults
join r in rightResults on l.FullMethod equals r.FullMethod
select new BenchmarkPair(l, r)).ToList();
LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList();
RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList();
bool leftEarlier = BenchmarkRepository.BuildForLabel(left.Label) < BenchmarkRepository.BuildForLabel(right.Label);
var earlier = leftEarlier ? left : right;
var later = leftEarlier ? right : left;
var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label);
var laterHash = BenchmarkRepository.HashForLabel(later.Label);
LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList();
}
public ActionResult BenchmarkRun(string machine, string label)
{
BenchmarkRun run = null;
string nextLabel = null; // In descending label order, so earlier...
foreach (var candidate in repository.RunsByMachine[machine])
{
if (run != null)
{
nextLabel = candidate.Label;
break;
}
if (candidate.Label == label)
{
run = candidate;
}
}
ImmutableList <SourceLogEntry> changes = null;
if (nextLabel != null)
{
string earlierHash = BenchmarkRepository.HashForLabel(nextLabel);
string thisHash = BenchmarkRepository.HashForLabel(label);
changes = log.EntriesBetween(earlierHash, thisHash).ToImmutableList();
}
return(View(new BenchmarkRunAndSourceLogs(run, changes)));
}
public void ShouldDisposeAllCounters()
{
var testCollector1 = new TestMetricCollector(new CounterMetricName("foo"), "bar");
var measureBucket1 = new MeasureBucket(testCollector1);
var testCollector2 = new TestMetricCollector(new CounterMetricName("foo"), "bar");
var measureBucket2 = new MeasureBucket(testCollector2);
var benchmarkRun = new BenchmarkRun(new List <MeasureBucket>(new[] { measureBucket1, measureBucket2 }),
new List <Counter>());
Assert.False(testCollector1.WasDisposed);
Assert.False(testCollector2.WasDisposed);
benchmarkRun.Dispose();
Assert.True(testCollector1.WasDisposed);
Assert.True(testCollector2.WasDisposed);
}
public BenchmarkDiff(BenchmarkRun left, BenchmarkRun right, MercurialLog log)
{
Left = left;
Right = right;
LeftOnly = left.Results.ExceptBy(right.Results, result => result.FullyQualifiedMethod);
RightOnly = right.Results.ExceptBy(left.Results, result => result.FullyQualifiedMethod);
var pairs = (from l in left.Results
join r in right.Results on l.FullyQualifiedMethod equals r.FullyQualifiedMethod
select new BenchmarkPair(l, r)).ToList();
LeftBetter = pairs.Where(pair => pair.Percent < ImprovementThreshold).ToList();
RightBetter = pairs.Where(pair => pair.Percent > RegressionThreshold).ToList();
var earlier = left.StartTime < right.StartTime ? left : right;
var later = left.StartTime < right.StartTime ? right : left;
var earlierHash = BenchmarkRepository.HashForLabel(earlier.Label);
var laterHash = BenchmarkRepository.HashForLabel(later.Label);
LogEntries = log.EntriesBetween(earlierHash, laterHash).ToList();
}
public void AddRun(BenchmarkRun run)
{
runs.Add(run);
}
/// <summary>
/// Pre-allocate all of the objects we're going to need for this benchmark
/// </summary>
private void Allocate()
{
_currentRun = Builder.NewRun(WarmupData);
}
public BenchmarkRunAndSourceLogs(BenchmarkRun benchmarkRun, ImmutableList<SourceLogEntry> changes)
{
this.Run = benchmarkRun;
this.Changes = changes;
}
public Entry(BenchmarkRun run, string description)
{
this.Run = run;
this.Description = description;
}
private BenchmarkRun?GetPreviousRun(BenchmarkRun run) => repository.GetEnvironment(run.BenchmarkEnvironmentId)?.Runs .SkipWhile(r => r.BenchmarkRunId != run.BenchmarkRunId) .Skip(1) .FirstOrDefault();
public BenchmarkRunAndSourceLogs(BenchmarkRun benchmarkRun, ImmutableList <SourceLogEntry> changes)
{
this.Run = benchmarkRun;
this.Changes = changes;
}
public Entry(BenchmarkRun run, string description)
{
this.Run = run;
this.Description = description;
}
public BenchmarkRunAndMercurialLogs(BenchmarkRun benchmarkRun, ImmutableList <MercurialLogEntry> changes)
{
this.Run = benchmarkRun;
this.Changes = changes;
}
private static void ProcessRun(BenchmarkRepository environmentRepository, string runDirectory)
{
var outputFile = Path.Combine(runDirectory, "benchmarks.pb");
// If the output file already exists, assume we're done.
if (File.Exists(outputFile))
{
return;
}
var startFile = Path.Combine(runDirectory, "start.txt");
var endFile = Path.Combine(runDirectory, "end.txt");
var start = DateTimeOffset.ParseExact(File.ReadAllText(startFile).Trim(), "yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture);
var end = DateTimeOffset.ParseExact(File.ReadAllText(endFile).Trim(), "yyyy-MM-ddTHH:mm:sszzz", CultureInfo.InvariantCulture);
var tfm = Path.GetFileName(runDirectory);
var commit = Path.GetFileName(Path.GetDirectoryName(runDirectory));
var models = Directory.GetFiles(runDirectory, "*.json")
.Select(file => JsonConvert.DeserializeObject <BriefJsonModel>(File.ReadAllText(file)))
.Where(m => m.Benchmarks.Any())
.ToList();
if (!models.Any())
{
// TODO: Throw?
return;
}
var runKey = Guid.NewGuid().ToString();
File.WriteAllText(outputFile, runKey);
var hostEnvironment = models.First().HostEnvironmentInfo;
var environment = new BenchmarkEnvironment
{
Machine = Environment.MachineName.ToLowerInvariant(),
OperatingSystem = hostEnvironment.OsVersion?.GetValue() ?? "",
Processor = hostEnvironment.ProcessorName?.GetValue() ?? "",
ProcessorCount = hostEnvironment.ProcessorCount,
TargetFramework = tfm,
JitModules = hostEnvironment.JitModules,
HasRyuJit = hostEnvironment.HasRyuJit,
Architecture = hostEnvironment.Architecture,
RuntimeVersion = hostEnvironment.RuntimeVersion
};
var environmentId = environmentRepository.GetOrAddEnvironmentId(environment);
var run = new BenchmarkRun
{
BenchmarkRunId = Guid.NewGuid().ToString(),
BenchmarkEnvironmentId = environmentId,
Commit = commit,
Start = start.ToTimestamp(),
End = end.ToTimestamp(),
BenchmarkDotNetVersion = hostEnvironment.BenchmarkDotNetVersion,
};
var types = new List <BenchmarkType>();
foreach (var model in models)
{
var firstBenchmark = model.Benchmarks.First();
var typeId = Guid.NewGuid().ToString();
types.Add(new BenchmarkType
{
BenchmarkRunId = run.BenchmarkRunId,
BenchmarkTypeId = typeId,
FullTypeName = $"{firstBenchmark.Namespace}.{firstBenchmark.Type}",
Namespace = firstBenchmark.Namespace,
Type = firstBenchmark.Type,
Benchmarks = { model.Benchmarks.Select(b => CreateBenchmark(b, typeId)).OrderBy(b => b.Method) }
});
}
run.Types_.AddRange(types.OrderBy(t => t.FullTypeName));
File.WriteAllBytes(outputFile, run.ToByteArray());
environmentRepository.AddRun(run);
Console.WriteLine($"Created benchmark file for {commit} / {tfm}");
}
public CreateBenchmarkRun()
{
InitializeComponent();
Run = new BenchmarkRun();
}
