public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList) { var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Commit").WhereEqualTo ("hash", Hash).FindAsync()); //Console.WriteLine ("FindAsync Commit"); if (results.Count() > 0) { return(results.First()); } if (CommitDate == null) { throw new Exception("Cannot save a commit without a commit date"); } var obj = ParseInterface.NewParseObject("Commit"); obj ["hash"] = Hash; if (Branch != null) { obj ["branch"] = Branch; } if (MergeBaseHash != null) { obj ["mergeBaseHash"] = MergeBaseHash; } if (CommitDate != null) { obj ["commitDate"] = CommitDate; } saveList.Add(obj); return(obj); }
public static async Task <RunSet> FromId(string id, Config config, Commit commit, string buildURL, string logURL) { var obj = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("RunSet").GetAsync(id)); //Console.WriteLine ("GetAsync RunSet"); if (obj == null) { throw new Exception("Could not fetch run set."); } var runSet = new RunSet { parseObject = obj, StartDateTime = obj.Get <DateTime> ("startedAt"), FinishDateTime = obj.Get <DateTime> ("finishedAt"), BuildURL = obj.Get <string> ("buildURL"), LogURL = logURL }; var configObj = obj.Get <ParseObject> ("config"); var commitObj = obj.Get <ParseObject> ("commit"); var machineObj = obj.Get <ParseObject> ("machine"); await ParseInterface.RunWithRetry(() => ParseObject.FetchAllAsync(new ParseObject[] { configObj, commitObj, machineObj })); //Console.WriteLine ("FindAllAsync config, commit, machine"); if (!config.EqualToParseObject(configObj)) { throw new Exception("Config does not match the one in the database."); } if (commit.Hash != commitObj.Get <string> ("hash")) { throw new Exception("Commit does not match the one in the database."); } if (buildURL != runSet.BuildURL) { throw new Exception("Build URL does not match the one in the database."); } if (!AreWeOnParseMachine(machineObj)) { throw new Exception("Machine does not match the one in the database."); } runSet.Config = config; runSet.Commit = commit; foreach (var o in obj.Get <List <object> > ("timedOutBenchmarks")) { runSet.timedOutBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId)); } foreach (var o in obj.Get <List <object> > ("crashedBenchmarks")) { runSet.crashedBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId)); } return(runSet); }
public static async Task <ParseObject> GetMachineFromParse() { var hostnameAndArch = LocalHostnameAndArch(); var hostname = hostnameAndArch.Item1; var arch = hostnameAndArch.Item2; var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Machine").WhereEqualTo ("name", hostname).WhereEqualTo ("architecture", arch).FindAsync()); //Console.WriteLine ("FindAsync Machine"); if (results.Count() > 0) { return(results.First()); } return(null); }
static async Task FetchBenchmarks() { if (nameToParseObject != null) { return; } nameToParseObject = new Dictionary <string, ParseObject> (); var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Benchmark").FindAsync()); //Console.WriteLine ("FindAsync Benchmark"); foreach (var o in results) { nameToParseObject [o.Get <string> ("name")] = o; } }
public async Task <ParseObject> GetFromParse() { var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Config") .WhereEqualTo ("name", Name) .WhereEqualTo ("monoExecutable", MonoExecutable) .FindAsync()); //Console.WriteLine ("FindAsync Config"); foreach (var o in results) { if (EqualToParseObject(o)) { Console.WriteLine("found config " + o.ObjectId); return(o); } } return(null); }
public async Task <ParseObject> UploadToParse() { // FIXME: for amended run sets, delete existing runs of benchmarks we just ran var averages = new Dictionary <string, double> (); var variances = new Dictionary <string, double> (); var logURLs = new Dictionary <string, string> (); if (parseObject != null) { var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages"); foreach (var kvp in originalAverages) { averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value); } var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances"); foreach (var kvp in originalVariances) { variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value); } var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs"); if (originalLogURLs != null) { foreach (var kvp in originalLogURLs) { logURLs [kvp.Key] = (string)kvp.Value; } } } foreach (var result in results) { var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds; if (avgAndVariance == null) { continue; } averages [result.Benchmark.Name] = avgAndVariance.Item1; variances [result.Benchmark.Name] = avgAndVariance.Item2; } if (LogURL != null) { string defaultURL; logURLs.TryGetValue("*", out defaultURL); if (defaultURL == null) { logURLs ["*"] = LogURL; } else if (defaultURL != LogURL) { foreach (var result in results) { logURLs [result.Benchmark.Name] = LogURL; } } } var saveList = new List <ParseObject> (); var obj = parseObject ?? ParseInterface.NewParseObject("RunSet"); if (parseObject == null) { var m = await GetOrUploadMachineToParse(saveList); var c = await Config.GetOrUploadToParse(saveList); var commit = await Commit.GetOrUploadToParse(saveList); obj ["machine"] = m; obj ["config"] = c; obj ["commit"] = commit; obj ["buildURL"] = BuildURL; obj ["startedAt"] = StartDateTime; if (PullRequestURL != null) { var prObj = ParseInterface.NewParseObject("PullRequest"); prObj ["URL"] = PullRequestURL; prObj ["baselineRunSet"] = PullRequestBaselineRunSet; saveList.Add(prObj); obj ["pullRequest"] = prObj; } } obj ["finishedAt"] = FinishDateTime; obj ["failed"] = averages.Count == 0; obj ["elapsedTimeAverages"] = averages; obj ["elapsedTimeVariances"] = variances; obj ["logURLs"] = logURLs; obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList); obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList); Console.WriteLine("uploading run set"); saveList.Add(obj); await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList)); //Console.WriteLine ("SaveAllAsync saveList 1"); saveList.Clear(); parseObject = obj; Console.WriteLine("uploading runs"); foreach (var result in results) { if (result.Config != Config) { throw new Exception("Results must have the same config as their RunSets"); } await result.UploadRunsToParse(obj, saveList); } await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList)); //Console.WriteLine ("SaveAllAsync saveList 2"); Console.WriteLine("done uploading"); return(obj); }
public async Task <Commit> GetCommit(string optionalCommitHash, string optionalGitRepoDir) { if (NoMono) { // FIXME: return a dummy commit return(null); } var info = NewProcessStartInfo(); /* Run without timing with --version */ info.Arguments = "--version"; Console.Out.WriteLine("\t$> {0} {1} {2}", PrintableEnvironmentVariables(info), info.FileName, info.Arguments); var process = Process.Start(info); var version = Task.Run(() => new StreamReader(process.StandardOutput.BaseStream).ReadToEnd()).Result; var versionError = Task.Run(() => new StreamReader(process.StandardError.BaseStream).ReadToEnd()).Result; process.WaitForExit(); process.Close(); var line = version.Split(new char[] { '\n' }, 2) [0]; var regex = new Regex("^Mono JIT.*\\((.*)/([0-9a-f]+) (.*)\\)"); var match = regex.Match(line); var commit = new Commit(); if (match.Success) { commit.Branch = match.Groups [1].Value; commit.Hash = match.Groups [2].Value; var date = match.Groups [3].Value; Console.WriteLine("branch: " + commit.Branch + " hash: " + commit.Hash + " date: " + date); } else { if (optionalCommitHash == null) { Console.Error.WriteLine("Error: cannot parse mono version and no commit given."); return(null); } } if (commit.Branch == "(detached") { commit.Branch = null; } if (optionalCommitHash != null) { if (commit.Hash != null && !optionalCommitHash.StartsWith(commit.Hash)) { Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one reported with --version."); return(null); } commit.Hash = optionalCommitHash; } try { var gitRepoDir = optionalGitRepoDir ?? Path.GetDirectoryName(Mono); var repo = new Repository(gitRepoDir); var gitHash = repo.RevParse(commit.Hash); if (gitHash == null) { Console.WriteLine("Could not get commit " + commit.Hash + " from repository"); } else { Console.WriteLine("Got commit " + gitHash + " from repository"); if (optionalCommitHash != null && optionalCommitHash != gitHash) { Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from the git repository."); return(null); } commit.Hash = gitHash; commit.MergeBaseHash = repo.MergeBase(commit.Hash, "master"); commit.CommitDate = repo.CommitDate(commit.Hash); if (commit.CommitDate == null) { Console.Error.WriteLine("Error: Could not get commit date from the git repository."); return(null); } Console.WriteLine("Commit {0} merge base {1} date {2}", commit.Hash, commit.MergeBaseHash, commit.CommitDate); } } catch (Exception) { Console.WriteLine("Could not get git repository"); } Octokit.Commit gitHubCommit = null; try { var gitHubClient = GitHubInterface.GitHubClient; gitHubCommit = await ParseInterface.RunWithRetry(() => gitHubClient.GitDatabase.Commit.Get("mono", "mono", commit.Hash), typeof(Octokit.NotFoundException)); } catch (Octokit.NotFoundException) { Console.WriteLine("Commit " + commit.Hash + " not found on GitHub"); } if (gitHubCommit == null) { Console.WriteLine("Could not get commit " + commit.Hash + " from GitHub"); } else { if (optionalCommitHash != null && optionalCommitHash != gitHubCommit.Sha) { Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from GitHub."); return(null); } commit.Hash = gitHubCommit.Sha; if (commit.CommitDate == null) { commit.CommitDate = gitHubCommit.Committer.Date.DateTime; } Console.WriteLine("Got commit " + commit.Hash + " from GitHub"); } return(commit); }
static async Task FindRegressions(string machineId, string configId, bool testRun) { const int baselineWindowSize = 5; const int testWindowSize = 3; const double controlLimitSize = 6; var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId); var config = await ParseObject.GetQuery("Config").GetAsync(configId); var runSets = await ParseInterface.PageQueryWithRetry(() => { return(ParseObject.GetQuery("RunSet") .WhereEqualTo("config", config) .WhereEqualTo("machine", machine) .WhereNotEqualTo("failed", true) .WhereDoesNotExist("pullRequest") .Include("commit")); }); var sortedRunSets = runSets.ToList(); sortedRunSets.Sort((a, b) => { var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate"); var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate"); var result = aCommitDate.CompareTo(bCommitDate); if (result != 0) { return(result); } var aStartedDate = a.Get <DateTime> ("startedAt"); var bStartedDate = b.Get <DateTime> ("startedAt"); return(aStartedDate.CompareTo(bStartedDate)); }); var lastWarningIndex = new Dictionary <string, int> (); for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i) { var windowAverages = new Dictionary <string, double> (); var windowVariances = new Dictionary <string, double> (); var benchmarkCounts = new Dictionary <string, int> (); for (var j = 1; j <= baselineWindowSize; ++j) { var baselineRunSet = sortedRunSets [i - j]; var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages"); var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances"); foreach (var kvp in averages) { var name = kvp.Key; var average = ParseInterface.NumberAsDouble(kvp.Value); var variance = ParseInterface.NumberAsDouble(variances [name]); if (!windowAverages.ContainsKey(name)) { windowAverages [name] = 0.0; windowVariances [name] = 0.0; benchmarkCounts [name] = 0; } windowAverages [name] += average; windowVariances [name] += variance; benchmarkCounts [name] += 1; } } foreach (var kvp in benchmarkCounts) { var name = kvp.Key; var count = kvp.Value; windowAverages [name] /= count; windowVariances [name] /= count; } var testRuns = new List <ParseObject> (); for (var j = 0; j < testWindowSize; ++j) { var runs = await FetchRunsForRunSet(sortedRunSets [i + j]); testRuns.AddRange(runs); } var testRunSet = sortedRunSets [i]; var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash"); Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash); var fasterBenchmarks = new List <ParseObject> (); var slowerBenchmarks = new List <ParseObject> (); foreach (var kvp in benchmarkCounts) { var name = kvp.Key; if (kvp.Value < baselineWindowSize) { continue; } if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize) { continue; } var average = windowAverages [name]; var variance = windowVariances [name]; var stdDev = Math.Sqrt(variance); var lowerControlLimit = average - controlLimitSize * stdDev; var upperControlLimit = average + controlLimitSize * stdDev; var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList(); if (runs.Count < 5) { continue; } var benchmark = runs [0].Get <ParseObject> ("benchmark"); var numOutliersFaster = 0; var numOutliersSlower = 0; foreach (var run in runs) { var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]); if (elapsed < lowerControlLimit) { ++numOutliersFaster; } if (elapsed > upperControlLimit) { ++numOutliersSlower; } } if (numOutliersFaster > runs.Count * 3 / 4) { Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count); lastWarningIndex [name] = i; fasterBenchmarks.Add(benchmark); } else if (numOutliersSlower > runs.Count * 3 / 4) { Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count); lastWarningIndex [name] = i; slowerBenchmarks.Add(benchmark); } /* * else if (numOutliersFaster == 0 && numOutliersSlower == 0) { * Console.WriteLine (" nothing in {0}", name); * } else { * Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count); * } */ } if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0) { var warnedFasterBenchmarks = new List <object> (); var warnedSlowerBenchmarks = new List <object> (); ParseObject warning = null; if (!testRun) { var warnings = await ParseInterface.PageQueryWithRetry(() => { return(ParseObject.GetQuery("RegressionWarnings") .WhereEqualTo("runSet", testRunSet)); }); if (warnings.Count() > 1) { throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId); } if (warnings.Count() == 1) { warning = warnings.First(); } if (warning != null) { warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks"); warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks"); } } var previousRunSet = sortedRunSets [i - 1]; var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config); var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config); var warnedAny = warnedAnyFaster || warnedAnySlower; if (!testRun && warnedAny) { if (warning == null) { warning = ParseInterface.NewParseObject("RegressionWarnings"); warning ["runSet"] = testRunSet; } warning ["fasterBenchmarks"] = warnedFasterBenchmarks; warning ["slowerBenchmarks"] = warnedSlowerBenchmarks; await ParseInterface.RunWithRetry(() => warning.SaveAsync()); } } await Task.Delay(1000); } }