static async Task FindRegressions(string machineId, string configId, bool testRun) { const int baselineWindowSize = 5; const int testWindowSize = 3; const double controlLimitSize = 6; var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId); var config = await ParseObject.GetQuery("Config").GetAsync(configId); var runSets = await ParseInterface.PageQueryWithRetry(() => { return(ParseObject.GetQuery("RunSet") .WhereEqualTo("config", config) .WhereEqualTo("machine", machine) .WhereNotEqualTo("failed", true) .WhereDoesNotExist("pullRequest") .Include("commit")); }); var sortedRunSets = runSets.ToList(); sortedRunSets.Sort((a, b) => { var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate"); var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate"); var result = aCommitDate.CompareTo(bCommitDate); if (result != 0) { return(result); } var aStartedDate = a.Get <DateTime> ("startedAt"); var bStartedDate = b.Get <DateTime> ("startedAt"); return(aStartedDate.CompareTo(bStartedDate)); }); var lastWarningIndex = new Dictionary <string, int> (); for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i) { var windowAverages = new Dictionary <string, double> (); var windowVariances = new Dictionary <string, double> (); var benchmarkCounts = new Dictionary <string, int> (); for (var j = 1; j <= baselineWindowSize; ++j) { var baselineRunSet = sortedRunSets [i - j]; var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages"); var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances"); foreach (var kvp in averages) { var name = kvp.Key; var average = ParseInterface.NumberAsDouble(kvp.Value); var variance = ParseInterface.NumberAsDouble(variances [name]); if (!windowAverages.ContainsKey(name)) { windowAverages [name] = 0.0; windowVariances [name] = 0.0; benchmarkCounts [name] = 0; } windowAverages [name] += average; windowVariances [name] += variance; benchmarkCounts [name] += 1; } } foreach (var kvp in benchmarkCounts) { var name = kvp.Key; var count = kvp.Value; windowAverages [name] /= count; windowVariances [name] /= count; } var testRuns = new List <ParseObject> (); for (var j = 0; j < testWindowSize; ++j) { var runs = await FetchRunsForRunSet(sortedRunSets [i + j]); testRuns.AddRange(runs); } var testRunSet = sortedRunSets [i]; var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash"); Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash); var fasterBenchmarks = new List <ParseObject> (); var slowerBenchmarks = new List <ParseObject> (); foreach (var kvp in benchmarkCounts) { var name = kvp.Key; if (kvp.Value < baselineWindowSize) { continue; } if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize) { continue; } var average = windowAverages [name]; var variance = windowVariances [name]; var stdDev = Math.Sqrt(variance); var lowerControlLimit = average - controlLimitSize * stdDev; var upperControlLimit = average + controlLimitSize * stdDev; var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList(); if (runs.Count < 5) { continue; } var benchmark = runs [0].Get <ParseObject> ("benchmark"); var numOutliersFaster = 0; var numOutliersSlower = 0; foreach (var run in runs) { var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]); if (elapsed < lowerControlLimit) { ++numOutliersFaster; } if (elapsed > upperControlLimit) { ++numOutliersSlower; } } if (numOutliersFaster > runs.Count * 3 / 4) { Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count); lastWarningIndex [name] = i; fasterBenchmarks.Add(benchmark); } else if (numOutliersSlower > runs.Count * 3 / 4) { Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count); lastWarningIndex [name] = i; slowerBenchmarks.Add(benchmark); } /* * else if (numOutliersFaster == 0 && numOutliersSlower == 0) { * Console.WriteLine (" nothing in {0}", name); * } else { * Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count); * } */ } if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0) { var warnedFasterBenchmarks = new List <object> (); var warnedSlowerBenchmarks = new List <object> (); ParseObject warning = null; if (!testRun) { var warnings = await ParseInterface.PageQueryWithRetry(() => { return(ParseObject.GetQuery("RegressionWarnings") .WhereEqualTo("runSet", testRunSet)); }); if (warnings.Count() > 1) { throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId); } if (warnings.Count() == 1) { warning = warnings.First(); } if (warning != null) { warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks"); warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks"); } } var previousRunSet = sortedRunSets [i - 1]; var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config); var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config); var warnedAny = warnedAnyFaster || warnedAnySlower; if (!testRun && warnedAny) { if (warning == null) { warning = ParseInterface.NewParseObject("RegressionWarnings"); warning ["runSet"] = testRunSet; } warning ["fasterBenchmarks"] = warnedFasterBenchmarks; warning ["slowerBenchmarks"] = warnedSlowerBenchmarks; await ParseInterface.RunWithRetry(() => warning.SaveAsync()); } } await Task.Delay(1000); } }
public async Task <ParseObject> UploadToParse() { // FIXME: for amended run sets, delete existing runs of benchmarks we just ran var averages = new Dictionary <string, double> (); var variances = new Dictionary <string, double> (); var logURLs = new Dictionary <string, string> (); if (parseObject != null) { var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages"); foreach (var kvp in originalAverages) { averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value); } var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances"); foreach (var kvp in originalVariances) { variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value); } var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs"); if (originalLogURLs != null) { foreach (var kvp in originalLogURLs) { logURLs [kvp.Key] = (string)kvp.Value; } } } foreach (var result in results) { var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds; if (avgAndVariance == null) { continue; } averages [result.Benchmark.Name] = avgAndVariance.Item1; variances [result.Benchmark.Name] = avgAndVariance.Item2; } if (LogURL != null) { string defaultURL; logURLs.TryGetValue("*", out defaultURL); if (defaultURL == null) { logURLs ["*"] = LogURL; } else if (defaultURL != LogURL) { foreach (var result in results) { logURLs [result.Benchmark.Name] = LogURL; } } } var saveList = new List <ParseObject> (); var obj = parseObject ?? ParseInterface.NewParseObject("RunSet"); if (parseObject == null) { var m = await GetOrUploadMachineToParse(saveList); var c = await Config.GetOrUploadToParse(saveList); var commit = await Commit.GetOrUploadToParse(saveList); obj ["machine"] = m; obj ["config"] = c; obj ["commit"] = commit; obj ["buildURL"] = BuildURL; obj ["startedAt"] = StartDateTime; if (PullRequestURL != null) { var prObj = ParseInterface.NewParseObject("PullRequest"); prObj ["URL"] = PullRequestURL; prObj ["baselineRunSet"] = PullRequestBaselineRunSet; saveList.Add(prObj); obj ["pullRequest"] = prObj; } } obj ["finishedAt"] = FinishDateTime; obj ["failed"] = averages.Count == 0; obj ["elapsedTimeAverages"] = averages; obj ["elapsedTimeVariances"] = variances; obj ["logURLs"] = logURLs; obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList); obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList); Console.WriteLine("uploading run set"); saveList.Add(obj); await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList)); //Console.WriteLine ("SaveAllAsync saveList 1"); saveList.Clear(); parseObject = obj; Console.WriteLine("uploading runs"); foreach (var result in results) { if (result.Config != Config) { throw new Exception("Results must have the same config as their RunSets"); } await result.UploadRunsToParse(obj, saveList); } await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList)); //Console.WriteLine ("SaveAllAsync saveList 2"); Console.WriteLine("done uploading"); return(obj); }
static async Task FixRunSet(ParseObject runSet) { var runs = await ParseInterface.PageQueryWithRetry(() => { return(ParseObject.GetQuery("Run") .Include("benchmark") .WhereEqualTo("runSet", runSet)); }); var benchmarkNames = runs.Select(r => (string)(((ParseObject)r ["benchmark"]) ["name"])).Distinct(); Console.WriteLine("run set {0} has {1} runs {2} benchmarks", runSet.ObjectId, runs.Count(), benchmarkNames.Count()); var averages = new Dictionary <string, double> (); var variances = new Dictionary <string, double> (); foreach (var name in benchmarkNames) { var numbers = runs.Where(r => (string)(((ParseObject)r ["benchmark"]) ["name"]) == name).Select(r => ParseInterface.NumberAsDouble(r ["elapsedMilliseconds"])).ToArray(); var avg = numbers.Average(); averages [name] = avg; var sum = 0.0; foreach (var v in numbers) { var diff = v - avg; sum += diff * diff; } var variance = sum / numbers.Length; variances [name] = variance; Console.WriteLine("benchmark {0} average {1} variance {2}", name, avg, variance); } runSet ["elapsedTimeAverages"] = averages; runSet ["elapsedTimeVariances"] = variances; await runSet.SaveAsync(); }