public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Commit").WhereEqualTo ("hash", Hash).FindAsync());
//Console.WriteLine ("FindAsync Commit");
if (results.Count() > 0)
{
return(results.First());
}
if (CommitDate == null)
{
throw new Exception("Cannot save a commit without a commit date");
}
var obj = ParseInterface.NewParseObject("Commit");
obj ["hash"] = Hash;
if (Branch != null)
{
obj ["branch"] = Branch;
}
if (MergeBaseHash != null)
{
obj ["mergeBaseHash"] = MergeBaseHash;
}
if (CommitDate != null)
{
obj ["commitDate"] = CommitDate;
}
saveList.Add(obj);
return(obj);
}
public static async Task <RunSet> FromId(string id, Config config, Commit commit, string buildURL, string logURL)
{
var obj = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("RunSet").GetAsync(id));
//Console.WriteLine ("GetAsync RunSet");
if (obj == null)
{
throw new Exception("Could not fetch run set.");
}
var runSet = new RunSet {
parseObject = obj,
StartDateTime = obj.Get <DateTime> ("startedAt"),
FinishDateTime = obj.Get <DateTime> ("finishedAt"),
BuildURL = obj.Get <string> ("buildURL"),
LogURL = logURL
};
var configObj = obj.Get <ParseObject> ("config");
var commitObj = obj.Get <ParseObject> ("commit");
var machineObj = obj.Get <ParseObject> ("machine");
await ParseInterface.RunWithRetry(() => ParseObject.FetchAllAsync(new ParseObject[] { configObj, commitObj, machineObj }));
//Console.WriteLine ("FindAllAsync config, commit, machine");
if (!config.EqualToParseObject(configObj))
{
throw new Exception("Config does not match the one in the database.");
}
if (commit.Hash != commitObj.Get <string> ("hash"))
{
throw new Exception("Commit does not match the one in the database.");
}
if (buildURL != runSet.BuildURL)
{
throw new Exception("Build URL does not match the one in the database.");
}
if (!AreWeOnParseMachine(machineObj))
{
throw new Exception("Machine does not match the one in the database.");
}
runSet.Config = config;
runSet.Commit = commit;
foreach (var o in obj.Get <List <object> > ("timedOutBenchmarks"))
{
runSet.timedOutBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId));
}
foreach (var o in obj.Get <List <object> > ("crashedBenchmarks"))
{
runSet.crashedBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId));
}
return(runSet);
}
public static async Task <ParseObject> GetMachineFromParse()
{
var hostnameAndArch = LocalHostnameAndArch();
var hostname = hostnameAndArch.Item1;
var arch = hostnameAndArch.Item2;
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Machine").WhereEqualTo ("name", hostname).WhereEqualTo ("architecture", arch).FindAsync());
//Console.WriteLine ("FindAsync Machine");
if (results.Count() > 0)
{
return(results.First());
}
return(null);
}
static async Task FetchBenchmarks()
{
if (nameToParseObject != null)
{
return;
}
nameToParseObject = new Dictionary <string, ParseObject> ();
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Benchmark").FindAsync());
//Console.WriteLine ("FindAsync Benchmark");
foreach (var o in results)
{
nameToParseObject [o.Get <string> ("name")] = o;
}
}
public async Task <ParseObject> GetFromParse()
{
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Config")
.WhereEqualTo ("name", Name)
.WhereEqualTo ("monoExecutable", MonoExecutable)
.FindAsync());
//Console.WriteLine ("FindAsync Config");
foreach (var o in results)
{
if (EqualToParseObject(o))
{
Console.WriteLine("found config " + o.ObjectId);
return(o);
}
}
return(null);
}
public async Task <ParseObject> UploadToParse()
{
// FIXME: for amended run sets, delete existing runs of benchmarks we just ran
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
var logURLs = new Dictionary <string, string> ();
if (parseObject != null)
{
var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages");
foreach (var kvp in originalAverages)
{
averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in originalVariances)
{
variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs");
if (originalLogURLs != null)
{
foreach (var kvp in originalLogURLs)
{
logURLs [kvp.Key] = (string)kvp.Value;
}
}
}
foreach (var result in results)
{
var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds;
if (avgAndVariance == null)
{
continue;
}
averages [result.Benchmark.Name] = avgAndVariance.Item1;
variances [result.Benchmark.Name] = avgAndVariance.Item2;
}
if (LogURL != null)
{
string defaultURL;
logURLs.TryGetValue("*", out defaultURL);
if (defaultURL == null)
{
logURLs ["*"] = LogURL;
}
else if (defaultURL != LogURL)
{
foreach (var result in results)
{
logURLs [result.Benchmark.Name] = LogURL;
}
}
}
var saveList = new List <ParseObject> ();
var obj = parseObject ?? ParseInterface.NewParseObject("RunSet");
if (parseObject == null)
{
var m = await GetOrUploadMachineToParse(saveList);
var c = await Config.GetOrUploadToParse(saveList);
var commit = await Commit.GetOrUploadToParse(saveList);
obj ["machine"] = m;
obj ["config"] = c;
obj ["commit"] = commit;
obj ["buildURL"] = BuildURL;
obj ["startedAt"] = StartDateTime;
if (PullRequestURL != null)
{
var prObj = ParseInterface.NewParseObject("PullRequest");
prObj ["URL"] = PullRequestURL;
prObj ["baselineRunSet"] = PullRequestBaselineRunSet;
saveList.Add(prObj);
obj ["pullRequest"] = prObj;
}
}
obj ["finishedAt"] = FinishDateTime;
obj ["failed"] = averages.Count == 0;
obj ["elapsedTimeAverages"] = averages;
obj ["elapsedTimeVariances"] = variances;
obj ["logURLs"] = logURLs;
obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList);
obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList);
Console.WriteLine("uploading run set");
saveList.Add(obj);
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 1");
saveList.Clear();
parseObject = obj;
Console.WriteLine("uploading runs");
foreach (var result in results)
{
if (result.Config != Config)
{
throw new Exception("Results must have the same config as their RunSets");
}
await result.UploadRunsToParse(obj, saveList);
}
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 2");
Console.WriteLine("done uploading");
return(obj);
}
public async Task <Commit> GetCommit(string optionalCommitHash, string optionalGitRepoDir)
{
if (NoMono)
{
// FIXME: return a dummy commit
return(null);
}
var info = NewProcessStartInfo();
/* Run without timing with --version */
info.Arguments = "--version";
Console.Out.WriteLine("\t$> {0} {1} {2}", PrintableEnvironmentVariables(info), info.FileName, info.Arguments);
var process = Process.Start(info);
var version = Task.Run(() => new StreamReader(process.StandardOutput.BaseStream).ReadToEnd()).Result;
var versionError = Task.Run(() => new StreamReader(process.StandardError.BaseStream).ReadToEnd()).Result;
process.WaitForExit();
process.Close();
var line = version.Split(new char[] { '\n' }, 2) [0];
var regex = new Regex("^Mono JIT.*\\((.*)/([0-9a-f]+) (.*)\\)");
var match = regex.Match(line);
var commit = new Commit();
if (match.Success)
{
commit.Branch = match.Groups [1].Value;
commit.Hash = match.Groups [2].Value;
var date = match.Groups [3].Value;
Console.WriteLine("branch: " + commit.Branch + " hash: " + commit.Hash + " date: " + date);
}
else
{
if (optionalCommitHash == null)
{
Console.Error.WriteLine("Error: cannot parse mono version and no commit given.");
return(null);
}
}
if (commit.Branch == "(detached")
{
commit.Branch = null;
}
if (optionalCommitHash != null)
{
if (commit.Hash != null && !optionalCommitHash.StartsWith(commit.Hash))
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one reported with --version.");
return(null);
}
commit.Hash = optionalCommitHash;
}
try {
var gitRepoDir = optionalGitRepoDir ?? Path.GetDirectoryName(Mono);
var repo = new Repository(gitRepoDir);
var gitHash = repo.RevParse(commit.Hash);
if (gitHash == null)
{
Console.WriteLine("Could not get commit " + commit.Hash + " from repository");
}
else
{
Console.WriteLine("Got commit " + gitHash + " from repository");
if (optionalCommitHash != null && optionalCommitHash != gitHash)
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from the git repository.");
return(null);
}
commit.Hash = gitHash;
commit.MergeBaseHash = repo.MergeBase(commit.Hash, "master");
commit.CommitDate = repo.CommitDate(commit.Hash);
if (commit.CommitDate == null)
{
Console.Error.WriteLine("Error: Could not get commit date from the git repository.");
return(null);
}
Console.WriteLine("Commit {0} merge base {1} date {2}", commit.Hash, commit.MergeBaseHash, commit.CommitDate);
}
} catch (Exception) {
Console.WriteLine("Could not get git repository");
}
Octokit.Commit gitHubCommit = null;
try {
var gitHubClient = GitHubInterface.GitHubClient;
gitHubCommit = await ParseInterface.RunWithRetry(() => gitHubClient.GitDatabase.Commit.Get("mono", "mono", commit.Hash), typeof(Octokit.NotFoundException));
} catch (Octokit.NotFoundException) {
Console.WriteLine("Commit " + commit.Hash + " not found on GitHub");
}
if (gitHubCommit == null)
{
Console.WriteLine("Could not get commit " + commit.Hash + " from GitHub");
}
else
{
if (optionalCommitHash != null && optionalCommitHash != gitHubCommit.Sha)
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from GitHub.");
return(null);
}
commit.Hash = gitHubCommit.Sha;
if (commit.CommitDate == null)
{
commit.CommitDate = gitHubCommit.Committer.Date.DateTime;
}
Console.WriteLine("Got commit " + commit.Hash + " from GitHub");
}
return(commit);
}
static async Task FindRegressions(string machineId, string configId, bool testRun)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId);
var config = await ParseObject.GetQuery("Config").GetAsync(configId);
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("config", config)
.WhereEqualTo("machine", machine)
.WhereNotEqualTo("failed", true)
.WhereDoesNotExist("pullRequest")
.Include("commit"));
});
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.Get <DateTime> ("startedAt");
var bStartedDate = b.Get <DateTime> ("startedAt");
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = ParseInterface.NumberAsDouble(kvp.Value);
var variance = ParseInterface.NumberAsDouble(variances [name]);
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testRuns = new List <ParseObject> ();
for (var j = 0; j < testWindowSize; ++j)
{
var runs = await FetchRunsForRunSet(sortedRunSets [i + j]);
testRuns.AddRange(runs);
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash");
Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash);
var fasterBenchmarks = new List <ParseObject> ();
var slowerBenchmarks = new List <ParseObject> ();
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [name];
var variance = windowVariances [name];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList();
if (runs.Count < 5)
{
continue;
}
var benchmark = runs [0].Get <ParseObject> ("benchmark");
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var run in runs)
{
var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]);
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > runs.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count);
lastWarningIndex [name] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > runs.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count);
lastWarningIndex [name] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <object> ();
var warnedSlowerBenchmarks = new List <object> ();
ParseObject warning = null;
if (!testRun)
{
var warnings = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RegressionWarnings")
.WhereEqualTo("runSet", testRunSet));
});
if (warnings.Count() > 1)
{
throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId);
}
if (warnings.Count() == 1)
{
warning = warnings.First();
}
if (warning != null)
{
warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks");
warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks");
}
}
var previousRunSet = sortedRunSets [i - 1];
var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config);
var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config);
var warnedAny = warnedAnyFaster || warnedAnySlower;
if (!testRun && warnedAny)
{
if (warning == null)
{
warning = ParseInterface.NewParseObject("RegressionWarnings");
warning ["runSet"] = testRunSet;
}
warning ["fasterBenchmarks"] = warnedFasterBenchmarks;
warning ["slowerBenchmarks"] = warnedSlowerBenchmarks;
await ParseInterface.RunWithRetry(() => warning.SaveAsync());
}
}
await Task.Delay(1000);
}
}