public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Commit").WhereEqualTo ("hash", Hash).FindAsync());
//Console.WriteLine ("FindAsync Commit");
if (results.Count() > 0)
{
return(results.First());
}
if (CommitDate == null)
{
throw new Exception("Cannot save a commit without a commit date");
}
var obj = ParseInterface.NewParseObject("Commit");
obj ["hash"] = Hash;
if (Branch != null)
{
obj ["branch"] = Branch;
}
if (MergeBaseHash != null)
{
obj ["mergeBaseHash"] = MergeBaseHash;
}
if (CommitDate != null)
{
obj ["commitDate"] = CommitDate;
}
saveList.Add(obj);
return(obj);
}
public async Task UploadRunsToParse(ParseObject runSet, List <ParseObject> saveList)
{
var b = await Benchmark.GetOrUploadToParse(saveList);
foreach (var run in Runs)
{
var obj = ParseInterface.NewParseObject("Run");
obj ["benchmark"] = b;
obj ["runSet"] = runSet;
obj ["elapsedMilliseconds"] = run.WallClockTime.TotalMilliseconds;
saveList.Add(obj);
}
}
public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
await FetchBenchmarks();
if (nameToParseObject.ContainsKey(Name))
{
return(nameToParseObject [Name]);
}
var obj = ParseInterface.NewParseObject("Benchmark");
obj ["name"] = Name;
saveList.Add(obj);
nameToParseObject [Name] = obj;
return(obj);
}
public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
var obj = await GetFromParse();
if (obj != null)
{
return(obj);
}
Console.WriteLine("creating new config");
obj = ParseInterface.NewParseObject("Config");
obj ["name"] = Name;
obj ["monoExecutable"] = MonoExecutable;
obj ["monoOptions"] = MonoOptions;
obj ["monoEnvironmentVariables"] = MonoEnvironmentVariables;
saveList.Add(obj);
return(obj);
}
async static Task <ParseObject> GetOrUploadMachineToParse(List <ParseObject> saveList)
{
var obj = await GetMachineFromParse();
if (obj != null)
{
return(obj);
}
var hostnameAndArch = LocalHostnameAndArch();
var hostname = hostnameAndArch.Item1;
var arch = hostnameAndArch.Item2;
obj = ParseInterface.NewParseObject("Machine");
obj ["name"] = hostname;
obj ["architecture"] = arch;
obj ["isDedicated"] = false;
saveList.Add(obj);
return(obj);
}
public async Task <ParseObject> UploadToParse()
{
// FIXME: for amended run sets, delete existing runs of benchmarks we just ran
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
var logURLs = new Dictionary <string, string> ();
if (parseObject != null)
{
var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages");
foreach (var kvp in originalAverages)
{
averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in originalVariances)
{
variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs");
if (originalLogURLs != null)
{
foreach (var kvp in originalLogURLs)
{
logURLs [kvp.Key] = (string)kvp.Value;
}
}
}
foreach (var result in results)
{
var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds;
if (avgAndVariance == null)
{
continue;
}
averages [result.Benchmark.Name] = avgAndVariance.Item1;
variances [result.Benchmark.Name] = avgAndVariance.Item2;
}
if (LogURL != null)
{
string defaultURL;
logURLs.TryGetValue("*", out defaultURL);
if (defaultURL == null)
{
logURLs ["*"] = LogURL;
}
else if (defaultURL != LogURL)
{
foreach (var result in results)
{
logURLs [result.Benchmark.Name] = LogURL;
}
}
}
var saveList = new List <ParseObject> ();
var obj = parseObject ?? ParseInterface.NewParseObject("RunSet");
if (parseObject == null)
{
var m = await GetOrUploadMachineToParse(saveList);
var c = await Config.GetOrUploadToParse(saveList);
var commit = await Commit.GetOrUploadToParse(saveList);
obj ["machine"] = m;
obj ["config"] = c;
obj ["commit"] = commit;
obj ["buildURL"] = BuildURL;
obj ["startedAt"] = StartDateTime;
if (PullRequestURL != null)
{
var prObj = ParseInterface.NewParseObject("PullRequest");
prObj ["URL"] = PullRequestURL;
prObj ["baselineRunSet"] = PullRequestBaselineRunSet;
saveList.Add(prObj);
obj ["pullRequest"] = prObj;
}
}
obj ["finishedAt"] = FinishDateTime;
obj ["failed"] = averages.Count == 0;
obj ["elapsedTimeAverages"] = averages;
obj ["elapsedTimeVariances"] = variances;
obj ["logURLs"] = logURLs;
obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList);
obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList);
Console.WriteLine("uploading run set");
saveList.Add(obj);
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 1");
saveList.Clear();
parseObject = obj;
Console.WriteLine("uploading runs");
foreach (var result in results)
{
if (result.Config != Config)
{
throw new Exception("Results must have the same config as their RunSets");
}
await result.UploadRunsToParse(obj, saveList);
}
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 2");
Console.WriteLine("done uploading");
return(obj);
}
static async Task FindRegressions(string machineId, string configId, bool testRun)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId);
var config = await ParseObject.GetQuery("Config").GetAsync(configId);
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("config", config)
.WhereEqualTo("machine", machine)
.WhereNotEqualTo("failed", true)
.WhereDoesNotExist("pullRequest")
.Include("commit"));
});
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.Get <DateTime> ("startedAt");
var bStartedDate = b.Get <DateTime> ("startedAt");
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = ParseInterface.NumberAsDouble(kvp.Value);
var variance = ParseInterface.NumberAsDouble(variances [name]);
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testRuns = new List <ParseObject> ();
for (var j = 0; j < testWindowSize; ++j)
{
var runs = await FetchRunsForRunSet(sortedRunSets [i + j]);
testRuns.AddRange(runs);
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash");
Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash);
var fasterBenchmarks = new List <ParseObject> ();
var slowerBenchmarks = new List <ParseObject> ();
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [name];
var variance = windowVariances [name];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList();
if (runs.Count < 5)
{
continue;
}
var benchmark = runs [0].Get <ParseObject> ("benchmark");
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var run in runs)
{
var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]);
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > runs.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count);
lastWarningIndex [name] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > runs.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count);
lastWarningIndex [name] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <object> ();
var warnedSlowerBenchmarks = new List <object> ();
ParseObject warning = null;
if (!testRun)
{
var warnings = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RegressionWarnings")
.WhereEqualTo("runSet", testRunSet));
});
if (warnings.Count() > 1)
{
throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId);
}
if (warnings.Count() == 1)
{
warning = warnings.First();
}
if (warning != null)
{
warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks");
warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks");
}
}
var previousRunSet = sortedRunSets [i - 1];
var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config);
var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config);
var warnedAny = warnedAnyFaster || warnedAnySlower;
if (!testRun && warnedAny)
{
if (warning == null)
{
warning = ParseInterface.NewParseObject("RegressionWarnings");
warning ["runSet"] = testRunSet;
}
warning ["fasterBenchmarks"] = warnedFasterBenchmarks;
warning ["slowerBenchmarks"] = warnedSlowerBenchmarks;
await ParseInterface.RunWithRetry(() => warning.SaveAsync());
}
}
await Task.Delay(1000);
}
}