static async Task FindRegressions(string machineId, string configId, bool testRun)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId);
var config = await ParseObject.GetQuery("Config").GetAsync(configId);
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("config", config)
.WhereEqualTo("machine", machine)
.WhereNotEqualTo("failed", true)
.WhereDoesNotExist("pullRequest")
.Include("commit"));
});
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.Get <DateTime> ("startedAt");
var bStartedDate = b.Get <DateTime> ("startedAt");
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = ParseInterface.NumberAsDouble(kvp.Value);
var variance = ParseInterface.NumberAsDouble(variances [name]);
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testRuns = new List <ParseObject> ();
for (var j = 0; j < testWindowSize; ++j)
{
var runs = await FetchRunsForRunSet(sortedRunSets [i + j]);
testRuns.AddRange(runs);
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash");
Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash);
var fasterBenchmarks = new List <ParseObject> ();
var slowerBenchmarks = new List <ParseObject> ();
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [name];
var variance = windowVariances [name];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList();
if (runs.Count < 5)
{
continue;
}
var benchmark = runs [0].Get <ParseObject> ("benchmark");
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var run in runs)
{
var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]);
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > runs.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count);
lastWarningIndex [name] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > runs.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count);
lastWarningIndex [name] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <object> ();
var warnedSlowerBenchmarks = new List <object> ();
ParseObject warning = null;
if (!testRun)
{
var warnings = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RegressionWarnings")
.WhereEqualTo("runSet", testRunSet));
});
if (warnings.Count() > 1)
{
throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId);
}
if (warnings.Count() == 1)
{
warning = warnings.First();
}
if (warning != null)
{
warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks");
warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks");
}
}
var previousRunSet = sortedRunSets [i - 1];
var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config);
var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config);
var warnedAny = warnedAnyFaster || warnedAnySlower;
if (!testRun && warnedAny)
{
if (warning == null)
{
warning = ParseInterface.NewParseObject("RegressionWarnings");
warning ["runSet"] = testRunSet;
}
warning ["fasterBenchmarks"] = warnedFasterBenchmarks;
warning ["slowerBenchmarks"] = warnedSlowerBenchmarks;
await ParseInterface.RunWithRetry(() => warning.SaveAsync());
}
}
await Task.Delay(1000);
}
}
public async Task <ParseObject> UploadToParse()
{
// FIXME: for amended run sets, delete existing runs of benchmarks we just ran
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
var logURLs = new Dictionary <string, string> ();
if (parseObject != null)
{
var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages");
foreach (var kvp in originalAverages)
{
averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in originalVariances)
{
variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs");
if (originalLogURLs != null)
{
foreach (var kvp in originalLogURLs)
{
logURLs [kvp.Key] = (string)kvp.Value;
}
}
}
foreach (var result in results)
{
var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds;
if (avgAndVariance == null)
{
continue;
}
averages [result.Benchmark.Name] = avgAndVariance.Item1;
variances [result.Benchmark.Name] = avgAndVariance.Item2;
}
if (LogURL != null)
{
string defaultURL;
logURLs.TryGetValue("*", out defaultURL);
if (defaultURL == null)
{
logURLs ["*"] = LogURL;
}
else if (defaultURL != LogURL)
{
foreach (var result in results)
{
logURLs [result.Benchmark.Name] = LogURL;
}
}
}
var saveList = new List <ParseObject> ();
var obj = parseObject ?? ParseInterface.NewParseObject("RunSet");
if (parseObject == null)
{
var m = await GetOrUploadMachineToParse(saveList);
var c = await Config.GetOrUploadToParse(saveList);
var commit = await Commit.GetOrUploadToParse(saveList);
obj ["machine"] = m;
obj ["config"] = c;
obj ["commit"] = commit;
obj ["buildURL"] = BuildURL;
obj ["startedAt"] = StartDateTime;
if (PullRequestURL != null)
{
var prObj = ParseInterface.NewParseObject("PullRequest");
prObj ["URL"] = PullRequestURL;
prObj ["baselineRunSet"] = PullRequestBaselineRunSet;
saveList.Add(prObj);
obj ["pullRequest"] = prObj;
}
}
obj ["finishedAt"] = FinishDateTime;
obj ["failed"] = averages.Count == 0;
obj ["elapsedTimeAverages"] = averages;
obj ["elapsedTimeVariances"] = variances;
obj ["logURLs"] = logURLs;
obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList);
obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList);
Console.WriteLine("uploading run set");
saveList.Add(obj);
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 1");
saveList.Clear();
parseObject = obj;
Console.WriteLine("uploading runs");
foreach (var result in results)
{
if (result.Config != Config)
{
throw new Exception("Results must have the same config as their RunSets");
}
await result.UploadRunsToParse(obj, saveList);
}
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 2");
Console.WriteLine("done uploading");
return(obj);
}
static async Task FixRunSet(ParseObject runSet)
{
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.Include("benchmark")
.WhereEqualTo("runSet", runSet));
});
var benchmarkNames = runs.Select(r => (string)(((ParseObject)r ["benchmark"]) ["name"])).Distinct();
Console.WriteLine("run set {0} has {1} runs {2} benchmarks", runSet.ObjectId, runs.Count(), benchmarkNames.Count());
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
foreach (var name in benchmarkNames)
{
var numbers = runs.Where(r => (string)(((ParseObject)r ["benchmark"]) ["name"]) == name).Select(r => ParseInterface.NumberAsDouble(r ["elapsedMilliseconds"])).ToArray();
var avg = numbers.Average();
averages [name] = avg;
var sum = 0.0;
foreach (var v in numbers)
{
var diff = v - avg;
sum += diff * diff;
}
var variance = sum / numbers.Length;
variances [name] = variance;
Console.WriteLine("benchmark {0} average {1} variance {2}", name, avg, variance);
}
runSet ["elapsedTimeAverages"] = averages;
runSet ["elapsedTimeVariances"] = variances;
await runSet.SaveAsync();
}