static async Task FixRunSet(ParseObject runSet)
{
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.Include("benchmark")
.WhereEqualTo("runSet", runSet));
});
var benchmarkNames = runs.Select(r => (string)(((ParseObject)r ["benchmark"]) ["name"])).Distinct();
Console.WriteLine("run set {0} has {1} runs {2} benchmarks", runSet.ObjectId, runs.Count(), benchmarkNames.Count());
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
foreach (var name in benchmarkNames)
{
var numbers = runs.Where(r => (string)(((ParseObject)r ["benchmark"]) ["name"]) == name).Select(r => ParseInterface.NumberAsDouble(r ["elapsedMilliseconds"])).ToArray();
var avg = numbers.Average();
averages [name] = avg;
var sum = 0.0;
foreach (var v in numbers)
{
var diff = v - avg;
sum += diff * diff;
}
var variance = sum / numbers.Length;
variances [name] = variance;
Console.WriteLine("benchmark {0} average {1} variance {2}", name, avg, variance);
}
runSet ["elapsedTimeAverages"] = averages;
runSet ["elapsedTimeVariances"] = variances;
await runSet.SaveAsync();
}
static async Task <IEnumerable <ParseObject> > FetchRunsForRunSet(ParseObject runSet)
{
var id = runSet.ObjectId;
if (!runsForRunSetId.ContainsKey(id))
{
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.WhereEqualTo("runSet", runSet)
.Include("benchmark"));
});
runsForRunSetId [id] = runs.ToList();
}
return(runsForRunSetId [id]);
}
static async Task AddAverages()
{
var runSets = await ParseInterface.PageQueryWithRetry(() => ParseObject.GetQuery("RunSet"));
foreach (var runSet in runSets)
{
if (runSet.ContainsKey("elapsedTimeAverages") && runSet.ContainsKey("elapsedTimeVariances"))
{
var averages = runSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = runSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
var averagesKeys = new SortedSet <string> (averages.Keys);
var variancesKeys = new SortedSet <string> (variances.Keys);
if (averagesKeys.SetEquals(variancesKeys))
{
continue;
}
}
await FixRunSet(runSet);
}
Console.WriteLine("got {0} run sets", runSets.Count());
}
static async Task DeleteRunSet(string runSetId)
{
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("objectId", runSetId));
});
if (runSets.Count() != 1)
{
throw new Exception("Could not fetch run set");
}
var runSet = runSets.First();
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.WhereEqualTo("runSet", runSet));
});
Console.WriteLine("deleting " + runs.Count() + " runs");
foreach (var run in runs)
{
await run.DeleteAsync();
}
await runSet.DeleteAsync();
}
static async Task <ParseObject> GetPullRequestBaselineRunSet(string pullRequestURL, Benchmarker.Common.Git.Repository repository, Config config)
{
var gitHubClient = GitHubInterface.GitHubClient;
var match = Regex.Match(pullRequestURL, @"^https?://github\.com/mono/mono/pull/(\d+)/?$");
if (match == null)
{
Console.WriteLine("Error: Cannot parse pull request URL.");
Environment.Exit(1);
}
var pullRequestNumber = Int32.Parse(match.Groups [1].Value);
Console.WriteLine("pull request {0}", pullRequestNumber);
var pullRequest = await gitHubClient.PullRequest.Get("mono", "mono", pullRequestNumber);
var prRepo = pullRequest.Head.Repository.SshUrl;
var prBranch = pullRequest.Head.Ref;
var prSha = repository.Fetch(prRepo, prBranch);
if (prSha == null)
{
Console.Error.WriteLine("Error: Could not fetch pull request branch {0} from repo {1}", prBranch, prRepo);
Environment.Exit(1);
}
var masterSha = repository.Fetch("[email protected]:mono/mono.git", "master");
if (masterSha == null)
{
Console.Error.WriteLine("Error: Could not fetch master.");
Environment.Exit(1);
}
var baseSha = repository.MergeBase(prSha, masterSha);
if (baseSha == null)
{
Console.Error.WriteLine("Error: Could not determine merge base of pull request.");
Environment.Exit(1);
}
Console.WriteLine("Merge base sha is {0}", baseSha);
var revList = repository.RevList(baseSha);
if (revList == null)
{
Console.Error.WriteLine("Error: Could not get rev-list for merge base {0}.", baseSha);
Environment.Exit(1);
}
Console.WriteLine("{0} commits in rev-list", revList.Length);
var configObj = await config.GetFromParse();
if (configObj == null)
{
Console.Error.WriteLine("Error: The config does not exist.");
Environment.Exit(1);
}
var machineObj = await RunSet.GetMachineFromParse();
if (machineObj == null)
{
Console.Error.WriteLine("Error: The machine does not exist.");
Environment.Exit(1);
}
var runSets = await ParseInterface.PageQueryWithRetry(() => ParseObject.GetQuery ("RunSet")
.WhereEqualTo ("machine", machineObj)
.WhereEqualTo ("config", configObj)
.WhereDoesNotExist ("pullRequest")
.Include("commit"));
Console.WriteLine("{0} run sets", runSets.Count());
var runSetsByCommits = new Dictionary <string, ParseObject> ();
foreach (var runSet in runSets)
{
var sha = runSet.Get <ParseObject> ("commit").Get <string> ("hash");
if (runSetsByCommits.ContainsKey(sha))
{
// FIXME: select between them?
continue;
}
runSetsByCommits.Add(sha, runSet);
}
foreach (var sha in revList)
{
if (runSetsByCommits.ContainsKey(sha))
{
Console.WriteLine("tested base commit is {0}", sha);
return(runSetsByCommits [sha]);
}
}
return(null);
}
static async Task FindRegressions(string machineId, string configId, bool testRun)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId);
var config = await ParseObject.GetQuery("Config").GetAsync(configId);
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("config", config)
.WhereEqualTo("machine", machine)
.WhereNotEqualTo("failed", true)
.WhereDoesNotExist("pullRequest")
.Include("commit"));
});
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.Get <DateTime> ("startedAt");
var bStartedDate = b.Get <DateTime> ("startedAt");
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = ParseInterface.NumberAsDouble(kvp.Value);
var variance = ParseInterface.NumberAsDouble(variances [name]);
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testRuns = new List <ParseObject> ();
for (var j = 0; j < testWindowSize; ++j)
{
var runs = await FetchRunsForRunSet(sortedRunSets [i + j]);
testRuns.AddRange(runs);
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash");
Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash);
var fasterBenchmarks = new List <ParseObject> ();
var slowerBenchmarks = new List <ParseObject> ();
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [name];
var variance = windowVariances [name];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList();
if (runs.Count < 5)
{
continue;
}
var benchmark = runs [0].Get <ParseObject> ("benchmark");
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var run in runs)
{
var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]);
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > runs.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count);
lastWarningIndex [name] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > runs.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count);
lastWarningIndex [name] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <object> ();
var warnedSlowerBenchmarks = new List <object> ();
ParseObject warning = null;
if (!testRun)
{
var warnings = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RegressionWarnings")
.WhereEqualTo("runSet", testRunSet));
});
if (warnings.Count() > 1)
{
throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId);
}
if (warnings.Count() == 1)
{
warning = warnings.First();
}
if (warning != null)
{
warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks");
warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks");
}
}
var previousRunSet = sortedRunSets [i - 1];
var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config);
var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config);
var warnedAny = warnedAnyFaster || warnedAnySlower;
if (!testRun && warnedAny)
{
if (warning == null)
{
warning = ParseInterface.NewParseObject("RegressionWarnings");
warning ["runSet"] = testRunSet;
}
warning ["fasterBenchmarks"] = warnedFasterBenchmarks;
warning ["slowerBenchmarks"] = warnedSlowerBenchmarks;
await ParseInterface.RunWithRetry(() => warning.SaveAsync());
}
}
await Task.Delay(1000);
}
}
