public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Commit").WhereEqualTo ("hash", Hash).FindAsync());
//Console.WriteLine ("FindAsync Commit");
if (results.Count() > 0)
{
return(results.First());
}
if (CommitDate == null)
{
throw new Exception("Cannot save a commit without a commit date");
}
var obj = ParseInterface.NewParseObject("Commit");
obj ["hash"] = Hash;
if (Branch != null)
{
obj ["branch"] = Branch;
}
if (MergeBaseHash != null)
{
obj ["mergeBaseHash"] = MergeBaseHash;
}
if (CommitDate != null)
{
obj ["commitDate"] = CommitDate;
}
saveList.Add(obj);
return(obj);
}
static async Task FixRunSet(ParseObject runSet)
{
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.Include("benchmark")
.WhereEqualTo("runSet", runSet));
});
var benchmarkNames = runs.Select(r => (string)(((ParseObject)r ["benchmark"]) ["name"])).Distinct();
Console.WriteLine("run set {0} has {1} runs {2} benchmarks", runSet.ObjectId, runs.Count(), benchmarkNames.Count());
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
foreach (var name in benchmarkNames)
{
var numbers = runs.Where(r => (string)(((ParseObject)r ["benchmark"]) ["name"]) == name).Select(r => ParseInterface.NumberAsDouble(r ["elapsedMilliseconds"])).ToArray();
var avg = numbers.Average();
averages [name] = avg;
var sum = 0.0;
foreach (var v in numbers)
{
var diff = v - avg;
sum += diff * diff;
}
var variance = sum / numbers.Length;
variances [name] = variance;
Console.WriteLine("benchmark {0} average {1} variance {2}", name, avg, variance);
}
runSet ["elapsedTimeAverages"] = averages;
runSet ["elapsedTimeVariances"] = variances;
await runSet.SaveAsync();
}
static void Main(string[] args)
{
var sampleBnf = @"
<syntax> ::= <rule> | <rule> <syntax>
<rule> ::= <identifier> ""::="" <expression> <line-end>
<expression> ::= <list> | <list> ""|"" <expression>
<line-end> ::= <EOL> | <line-end> <line-end>
<list> ::= <term > | <term> <list>
<term> ::= <literal > | <identifier>
<identifier> ::= ""<"" <rule-name> "">""
<literal> ::= '""' <text> '""' | ""'"" <text> ""'""";
var grammar = new BnfGrammar();
for (long i = 0; i < 10000; i++)
{
var parseEngine = new ParseEngine(grammar);
var parseInterface = new ParseInterface(parseEngine, sampleBnf);
var stringReader = new StringReader(sampleBnf);
while (!parseInterface.EndOfStream() && parseInterface.Read()) { }
var result = parseInterface.ParseEngine.IsAccepted();
}
}
static void InitializeParseInterface()
{
if (!ParseInterface.Initialize())
{
Console.Error.WriteLine("Error: Could not initialize Parse interface.");
Environment.Exit(1);
}
}
public static async Task <RunSet> FromId(string id, Config config, Commit commit, string buildURL, string logURL)
{
var obj = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("RunSet").GetAsync(id));
//Console.WriteLine ("GetAsync RunSet");
if (obj == null)
{
throw new Exception("Could not fetch run set.");
}
var runSet = new RunSet {
parseObject = obj,
StartDateTime = obj.Get <DateTime> ("startedAt"),
FinishDateTime = obj.Get <DateTime> ("finishedAt"),
BuildURL = obj.Get <string> ("buildURL"),
LogURL = logURL
};
var configObj = obj.Get <ParseObject> ("config");
var commitObj = obj.Get <ParseObject> ("commit");
var machineObj = obj.Get <ParseObject> ("machine");
await ParseInterface.RunWithRetry(() => ParseObject.FetchAllAsync(new ParseObject[] { configObj, commitObj, machineObj }));
//Console.WriteLine ("FindAllAsync config, commit, machine");
if (!config.EqualToParseObject(configObj))
{
throw new Exception("Config does not match the one in the database.");
}
if (commit.Hash != commitObj.Get <string> ("hash"))
{
throw new Exception("Commit does not match the one in the database.");
}
if (buildURL != runSet.BuildURL)
{
throw new Exception("Build URL does not match the one in the database.");
}
if (!AreWeOnParseMachine(machineObj))
{
throw new Exception("Machine does not match the one in the database.");
}
runSet.Config = config;
runSet.Commit = commit;
foreach (var o in obj.Get <List <object> > ("timedOutBenchmarks"))
{
runSet.timedOutBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId));
}
foreach (var o in obj.Get <List <object> > ("crashedBenchmarks"))
{
runSet.crashedBenchmarks.Add(await Benchmark.FromId(((ParseObject)o).ObjectId));
}
return(runSet);
}
public async Task UploadRunsToParse(ParseObject runSet, List <ParseObject> saveList)
{
var b = await Benchmark.GetOrUploadToParse(saveList);
foreach (var run in Runs)
{
var obj = ParseInterface.NewParseObject("Run");
obj ["benchmark"] = b;
obj ["runSet"] = runSet;
obj ["elapsedMilliseconds"] = run.WallClockTime.TotalMilliseconds;
saveList.Add(obj);
}
}
public static async Task <ParseObject> GetMachineFromParse()
{
var hostnameAndArch = LocalHostnameAndArch();
var hostname = hostnameAndArch.Item1;
var arch = hostnameAndArch.Item2;
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Machine").WhereEqualTo ("name", hostname).WhereEqualTo ("architecture", arch).FindAsync());
//Console.WriteLine ("FindAsync Machine");
if (results.Count() > 0)
{
return(results.First());
}
return(null);
}
static async Task <IEnumerable <ParseObject> > FetchRunsForRunSet(ParseObject runSet)
{
var id = runSet.ObjectId;
if (!runsForRunSetId.ContainsKey(id))
{
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.WhereEqualTo("runSet", runSet)
.Include("benchmark"));
});
runsForRunSetId [id] = runs.ToList();
}
return(runsForRunSetId [id]);
}
public void Test_Bnf_That_Parses_Large_Grammar_In_File()
{
var bnf = File.ReadAllText(Path.Combine(TestContext.TestDeploymentDir, "Bnf", "AnsiC.bnf"));
Assert.IsFalse(string.IsNullOrEmpty(bnf));
var grammar = new BnfGrammar();
var parseEngine = new ParseEngine(grammar);
var parseInterface = new ParseInterface(parseEngine, bnf);
while (!parseInterface.EndOfStream())
{
if (!parseInterface.Read())
Assert.Fail("Error Parsing At Position {0}", parseInterface.Position);
}
Assert.IsTrue(parseEngine.IsAccepted());
}
static async Task FetchBenchmarks()
{
if (nameToParseObject != null)
{
return;
}
nameToParseObject = new Dictionary <string, ParseObject> ();
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Benchmark").FindAsync());
//Console.WriteLine ("FindAsync Benchmark");
foreach (var o in results)
{
nameToParseObject [o.Get <string> ("name")] = o;
}
}
public void Test_NodeWalker_That_Walks_Simple_Regex()
{
var regexGrammar = new RegexGrammar();
var regexParseEngine = new ParseEngine(regexGrammar);
var regexParseInterface = new ParseInterface(regexParseEngine, @"[(]\d[)]");
while (!regexParseInterface.EndOfStream())
{
if (!regexParseInterface.Read())
Assert.Fail("error parsing input at position {0}", regexParseInterface.Position);
}
Assert.IsTrue(regexParseEngine.IsAccepted());
var nodeVisitor = new NodeVisitor();
var root = regexParseEngine.GetRoot();
root.Accept(nodeVisitor);
Assert.AreEqual(34, nodeVisitor.VisitLog.Count);
}
public async Task <ParseObject> GetFromParse()
{
var results = await ParseInterface.RunWithRetry(() => ParseObject.GetQuery ("Config")
.WhereEqualTo ("name", Name)
.WhereEqualTo ("monoExecutable", MonoExecutable)
.FindAsync());
//Console.WriteLine ("FindAsync Config");
foreach (var o in results)
{
if (EqualToParseObject(o))
{
Console.WriteLine("found config " + o.ObjectId);
return(o);
}
}
return(null);
}
public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
await FetchBenchmarks();
if (nameToParseObject.ContainsKey(Name))
{
return(nameToParseObject [Name]);
}
var obj = ParseInterface.NewParseObject("Benchmark");
obj ["name"] = Name;
saveList.Add(obj);
nameToParseObject [Name] = obj;
return(obj);
}
public async Task <ParseObject> GetOrUploadToParse(List <ParseObject> saveList)
{
var obj = await GetFromParse();
if (obj != null)
{
return(obj);
}
Console.WriteLine("creating new config");
obj = ParseInterface.NewParseObject("Config");
obj ["name"] = Name;
obj ["monoExecutable"] = MonoExecutable;
obj ["monoOptions"] = MonoOptions;
obj ["monoEnvironmentVariables"] = MonoEnvironmentVariables;
saveList.Add(obj);
return(obj);
}
async static Task <ParseObject> GetOrUploadMachineToParse(List <ParseObject> saveList)
{
var obj = await GetMachineFromParse();
if (obj != null)
{
return(obj);
}
var hostnameAndArch = LocalHostnameAndArch();
var hostname = hostnameAndArch.Item1;
var arch = hostnameAndArch.Item2;
obj = ParseInterface.NewParseObject("Machine");
obj ["name"] = hostname;
obj ["architecture"] = arch;
obj ["isDedicated"] = false;
saveList.Add(obj);
return(obj);
}
static async Task AddAverages()
{
var runSets = await ParseInterface.PageQueryWithRetry(() => ParseObject.GetQuery("RunSet"));
foreach (var runSet in runSets)
{
if (runSet.ContainsKey("elapsedTimeAverages") && runSet.ContainsKey("elapsedTimeVariances"))
{
var averages = runSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = runSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
var averagesKeys = new SortedSet <string> (averages.Keys);
var variancesKeys = new SortedSet <string> (variances.Keys);
if (averagesKeys.SetEquals(variancesKeys))
{
continue;
}
}
await FixRunSet(runSet);
}
Console.WriteLine("got {0} run sets", runSets.Count());
}
public void Test_Bnf_That_Parse_Produces_Bnf_Chart()
{
var grammar = new BnfGrammar();
var parseEngine = new ParseEngine(grammar);
var parseInterface = new ParseInterface(parseEngine, _bnfText);
while (!parseInterface.EndOfStream())
{
if (!parseInterface.Read())
{
var position = parseInterface.Position;
var startIndex = 0;
for (int i = position; i >= 0; i--)
{
if (_bnfText[i] == '\n' && i > 0)
if (_bnfText[i - 1] == '\r')
{
startIndex = i;
break;
}
}
var endIndex = _bnfText.IndexOf("\r\n", position);
endIndex = endIndex < 0 ? _bnfText.Length : endIndex;
var length = endIndex - startIndex;
var stringBuilder = new StringBuilder();
stringBuilder
.AppendFormat("Error parsing input string at position {0}.", parseInterface.Position)
.AppendLine()
.AppendFormat("start: {0}", startIndex)
.AppendLine()
.AppendLine(_bnfText.Substring(startIndex, length));
Assert.Fail(stringBuilder.ToString());
}
}
Assert.IsTrue(
parseInterface.ParseEngine.IsAccepted(),
"error at position {0}", parseInterface.Position);
}
static async Task DeleteRunSet(string runSetId)
{
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("objectId", runSetId));
});
if (runSets.Count() != 1)
{
throw new Exception("Could not fetch run set");
}
var runSet = runSets.First();
var runs = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("Run")
.WhereEqualTo("runSet", runSet));
});
Console.WriteLine("deleting " + runs.Count() + " runs");
foreach (var run in runs)
{
await run.DeleteAsync();
}
await runSet.DeleteAsync();
}
public async Task <Commit> GetCommit(string optionalCommitHash, string optionalGitRepoDir)
{
if (NoMono)
{
// FIXME: return a dummy commit
return(null);
}
var info = NewProcessStartInfo();
/* Run without timing with --version */
info.Arguments = "--version";
Console.Out.WriteLine("\t$> {0} {1} {2}", PrintableEnvironmentVariables(info), info.FileName, info.Arguments);
var process = Process.Start(info);
var version = Task.Run(() => new StreamReader(process.StandardOutput.BaseStream).ReadToEnd()).Result;
var versionError = Task.Run(() => new StreamReader(process.StandardError.BaseStream).ReadToEnd()).Result;
process.WaitForExit();
process.Close();
var line = version.Split(new char[] { '\n' }, 2) [0];
var regex = new Regex("^Mono JIT.*\\((.*)/([0-9a-f]+) (.*)\\)");
var match = regex.Match(line);
var commit = new Commit();
if (match.Success)
{
commit.Branch = match.Groups [1].Value;
commit.Hash = match.Groups [2].Value;
var date = match.Groups [3].Value;
Console.WriteLine("branch: " + commit.Branch + " hash: " + commit.Hash + " date: " + date);
}
else
{
if (optionalCommitHash == null)
{
Console.Error.WriteLine("Error: cannot parse mono version and no commit given.");
return(null);
}
}
if (commit.Branch == "(detached")
{
commit.Branch = null;
}
if (optionalCommitHash != null)
{
if (commit.Hash != null && !optionalCommitHash.StartsWith(commit.Hash))
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one reported with --version.");
return(null);
}
commit.Hash = optionalCommitHash;
}
try {
var gitRepoDir = optionalGitRepoDir ?? Path.GetDirectoryName(Mono);
var repo = new Repository(gitRepoDir);
var gitHash = repo.RevParse(commit.Hash);
if (gitHash == null)
{
Console.WriteLine("Could not get commit " + commit.Hash + " from repository");
}
else
{
Console.WriteLine("Got commit " + gitHash + " from repository");
if (optionalCommitHash != null && optionalCommitHash != gitHash)
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from the git repository.");
return(null);
}
commit.Hash = gitHash;
commit.MergeBaseHash = repo.MergeBase(commit.Hash, "master");
commit.CommitDate = repo.CommitDate(commit.Hash);
if (commit.CommitDate == null)
{
Console.Error.WriteLine("Error: Could not get commit date from the git repository.");
return(null);
}
Console.WriteLine("Commit {0} merge base {1} date {2}", commit.Hash, commit.MergeBaseHash, commit.CommitDate);
}
} catch (Exception) {
Console.WriteLine("Could not get git repository");
}
Octokit.Commit gitHubCommit = null;
try {
var gitHubClient = GitHubInterface.GitHubClient;
gitHubCommit = await ParseInterface.RunWithRetry(() => gitHubClient.GitDatabase.Commit.Get("mono", "mono", commit.Hash), typeof(Octokit.NotFoundException));
} catch (Octokit.NotFoundException) {
Console.WriteLine("Commit " + commit.Hash + " not found on GitHub");
}
if (gitHubCommit == null)
{
Console.WriteLine("Could not get commit " + commit.Hash + " from GitHub");
}
else
{
if (optionalCommitHash != null && optionalCommitHash != gitHubCommit.Sha)
{
Console.Error.WriteLine("Error: Commit hash specified on command line does not match the one from GitHub.");
return(null);
}
commit.Hash = gitHubCommit.Sha;
if (commit.CommitDate == null)
{
commit.CommitDate = gitHubCommit.Committer.Date.DateTime;
}
Console.WriteLine("Got commit " + commit.Hash + " from GitHub");
}
return(commit);
}
public async Task <ParseObject> UploadToParse()
{
// FIXME: for amended run sets, delete existing runs of benchmarks we just ran
var averages = new Dictionary <string, double> ();
var variances = new Dictionary <string, double> ();
var logURLs = new Dictionary <string, string> ();
if (parseObject != null)
{
var originalAverages = parseObject.Get <Dictionary <string, object> > ("elapsedTimeAverages");
foreach (var kvp in originalAverages)
{
averages [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalVariances = parseObject.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in originalVariances)
{
variances [kvp.Key] = ParseInterface.NumberAsDouble(kvp.Value);
}
var originalLogURLs = parseObject.Get <Dictionary <string, object> > ("logURLs");
if (originalLogURLs != null)
{
foreach (var kvp in originalLogURLs)
{
logURLs [kvp.Key] = (string)kvp.Value;
}
}
}
foreach (var result in results)
{
var avgAndVariance = result.AverageAndVarianceWallClockTimeMilliseconds;
if (avgAndVariance == null)
{
continue;
}
averages [result.Benchmark.Name] = avgAndVariance.Item1;
variances [result.Benchmark.Name] = avgAndVariance.Item2;
}
if (LogURL != null)
{
string defaultURL;
logURLs.TryGetValue("*", out defaultURL);
if (defaultURL == null)
{
logURLs ["*"] = LogURL;
}
else if (defaultURL != LogURL)
{
foreach (var result in results)
{
logURLs [result.Benchmark.Name] = LogURL;
}
}
}
var saveList = new List <ParseObject> ();
var obj = parseObject ?? ParseInterface.NewParseObject("RunSet");
if (parseObject == null)
{
var m = await GetOrUploadMachineToParse(saveList);
var c = await Config.GetOrUploadToParse(saveList);
var commit = await Commit.GetOrUploadToParse(saveList);
obj ["machine"] = m;
obj ["config"] = c;
obj ["commit"] = commit;
obj ["buildURL"] = BuildURL;
obj ["startedAt"] = StartDateTime;
if (PullRequestURL != null)
{
var prObj = ParseInterface.NewParseObject("PullRequest");
prObj ["URL"] = PullRequestURL;
prObj ["baselineRunSet"] = PullRequestBaselineRunSet;
saveList.Add(prObj);
obj ["pullRequest"] = prObj;
}
}
obj ["finishedAt"] = FinishDateTime;
obj ["failed"] = averages.Count == 0;
obj ["elapsedTimeAverages"] = averages;
obj ["elapsedTimeVariances"] = variances;
obj ["logURLs"] = logURLs;
obj ["timedOutBenchmarks"] = await BenchmarkListToParseObjectArray(timedOutBenchmarks, saveList);
obj ["crashedBenchmarks"] = await BenchmarkListToParseObjectArray(crashedBenchmarks, saveList);
Console.WriteLine("uploading run set");
saveList.Add(obj);
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 1");
saveList.Clear();
parseObject = obj;
Console.WriteLine("uploading runs");
foreach (var result in results)
{
if (result.Config != Config)
{
throw new Exception("Results must have the same config as their RunSets");
}
await result.UploadRunsToParse(obj, saveList);
}
await ParseInterface.RunWithRetry(() => ParseObject.SaveAllAsync(saveList));
//Console.WriteLine ("SaveAllAsync saveList 2");
Console.WriteLine("done uploading");
return(obj);
}
static async Task FindRegressions(string machineId, string configId, bool testRun)
{
const int baselineWindowSize = 5;
const int testWindowSize = 3;
const double controlLimitSize = 6;
var machine = await ParseObject.GetQuery("Machine").GetAsync(machineId);
var config = await ParseObject.GetQuery("Config").GetAsync(configId);
var runSets = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RunSet")
.WhereEqualTo("config", config)
.WhereEqualTo("machine", machine)
.WhereNotEqualTo("failed", true)
.WhereDoesNotExist("pullRequest")
.Include("commit"));
});
var sortedRunSets = runSets.ToList();
sortedRunSets.Sort((a, b) => {
var aCommitDate = a.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var bCommitDate = b.Get <ParseObject> ("commit").Get <DateTime> ("commitDate");
var result = aCommitDate.CompareTo(bCommitDate);
if (result != 0)
{
return(result);
}
var aStartedDate = a.Get <DateTime> ("startedAt");
var bStartedDate = b.Get <DateTime> ("startedAt");
return(aStartedDate.CompareTo(bStartedDate));
});
var lastWarningIndex = new Dictionary <string, int> ();
for (var i = baselineWindowSize; i <= sortedRunSets.Count - testWindowSize; ++i)
{
var windowAverages = new Dictionary <string, double> ();
var windowVariances = new Dictionary <string, double> ();
var benchmarkCounts = new Dictionary <string, int> ();
for (var j = 1; j <= baselineWindowSize; ++j)
{
var baselineRunSet = sortedRunSets [i - j];
var averages = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeAverages");
var variances = baselineRunSet.Get <Dictionary <string, object> > ("elapsedTimeVariances");
foreach (var kvp in averages)
{
var name = kvp.Key;
var average = ParseInterface.NumberAsDouble(kvp.Value);
var variance = ParseInterface.NumberAsDouble(variances [name]);
if (!windowAverages.ContainsKey(name))
{
windowAverages [name] = 0.0;
windowVariances [name] = 0.0;
benchmarkCounts [name] = 0;
}
windowAverages [name] += average;
windowVariances [name] += variance;
benchmarkCounts [name] += 1;
}
}
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
var count = kvp.Value;
windowAverages [name] /= count;
windowVariances [name] /= count;
}
var testRuns = new List <ParseObject> ();
for (var j = 0; j < testWindowSize; ++j)
{
var runs = await FetchRunsForRunSet(sortedRunSets [i + j]);
testRuns.AddRange(runs);
}
var testRunSet = sortedRunSets [i];
var commitHash = testRunSet.Get <ParseObject> ("commit").Get <string> ("hash");
Console.WriteLine("{0} {1}", testRunSet.ObjectId, commitHash);
var fasterBenchmarks = new List <ParseObject> ();
var slowerBenchmarks = new List <ParseObject> ();
foreach (var kvp in benchmarkCounts)
{
var name = kvp.Key;
if (kvp.Value < baselineWindowSize)
{
continue;
}
if (lastWarningIndex.ContainsKey(name) && lastWarningIndex [name] >= i - baselineWindowSize)
{
continue;
}
var average = windowAverages [name];
var variance = windowVariances [name];
var stdDev = Math.Sqrt(variance);
var lowerControlLimit = average - controlLimitSize * stdDev;
var upperControlLimit = average + controlLimitSize * stdDev;
var runs = testRuns.Where(o => o.Get <ParseObject> ("benchmark").Get <string> ("name") == name).ToList();
if (runs.Count < 5)
{
continue;
}
var benchmark = runs [0].Get <ParseObject> ("benchmark");
var numOutliersFaster = 0;
var numOutliersSlower = 0;
foreach (var run in runs)
{
var elapsed = ParseInterface.NumberAsDouble(run ["elapsedMilliseconds"]);
if (elapsed < lowerControlLimit)
{
++numOutliersFaster;
}
if (elapsed > upperControlLimit)
{
++numOutliersSlower;
}
}
if (numOutliersFaster > runs.Count * 3 / 4)
{
Console.WriteLine("+ regression in {0}: {1}/{2}", name, numOutliersFaster, runs.Count);
lastWarningIndex [name] = i;
fasterBenchmarks.Add(benchmark);
}
else if (numOutliersSlower > runs.Count * 3 / 4)
{
Console.WriteLine("- regression in {0}: {1}/{2}", name, numOutliersSlower, runs.Count);
lastWarningIndex [name] = i;
slowerBenchmarks.Add(benchmark);
}
/*
* else if (numOutliersFaster == 0 && numOutliersSlower == 0) {
* Console.WriteLine (" nothing in {0}", name);
* } else {
* Console.WriteLine ("? suspected in {0}: {1}-/{2}+/{3}", name, numOutliersSlower, numOutliersFaster, runs.Count);
* }
*/
}
if (fasterBenchmarks.Count != 0 || slowerBenchmarks.Count != 0)
{
var warnedFasterBenchmarks = new List <object> ();
var warnedSlowerBenchmarks = new List <object> ();
ParseObject warning = null;
if (!testRun)
{
var warnings = await ParseInterface.PageQueryWithRetry(() => {
return(ParseObject.GetQuery("RegressionWarnings")
.WhereEqualTo("runSet", testRunSet));
});
if (warnings.Count() > 1)
{
throw new Exception("There is more than one RegressionWarning for run set " + testRunSet.ObjectId);
}
if (warnings.Count() == 1)
{
warning = warnings.First();
}
if (warning != null)
{
warnedFasterBenchmarks = warning.Get <List <object> > ("fasterBenchmarks");
warnedSlowerBenchmarks = warning.Get <List <object> > ("slowerBenchmarks");
}
}
var previousRunSet = sortedRunSets [i - 1];
var warnedAnyFaster = await WarnIfNecessary(testRun, fasterBenchmarks, warnedFasterBenchmarks, true, testRunSet, previousRunSet, machine, config);
var warnedAnySlower = await WarnIfNecessary(testRun, slowerBenchmarks, warnedSlowerBenchmarks, false, testRunSet, previousRunSet, machine, config);
var warnedAny = warnedAnyFaster || warnedAnySlower;
if (!testRun && warnedAny)
{
if (warning == null)
{
warning = ParseInterface.NewParseObject("RegressionWarnings");
warning ["runSet"] = testRunSet;
}
warning ["fasterBenchmarks"] = warnedFasterBenchmarks;
warning ["slowerBenchmarks"] = warnedSlowerBenchmarks;
await ParseInterface.RunWithRetry(() => warning.SaveAsync());
}
}
await Task.Delay(1000);
}
}
private void ParseInput(string input)
{
var parseInterface = new ParseInterface(_parseEngine, input);
for (int i = 0; i < input.Length; i++)
Assert.IsTrue(parseInterface.Read(),
string.Format("Line 0, Column {1} : Invalid Character '{0}'",
input[i],
_parseEngine.Location));
}
static async Task <ParseObject> GetPullRequestBaselineRunSet(string pullRequestURL, Benchmarker.Common.Git.Repository repository, Config config)
{
var gitHubClient = GitHubInterface.GitHubClient;
var match = Regex.Match(pullRequestURL, @"^https?://github\.com/mono/mono/pull/(\d+)/?$");
if (match == null)
{
Console.WriteLine("Error: Cannot parse pull request URL.");
Environment.Exit(1);
}
var pullRequestNumber = Int32.Parse(match.Groups [1].Value);
Console.WriteLine("pull request {0}", pullRequestNumber);
var pullRequest = await gitHubClient.PullRequest.Get("mono", "mono", pullRequestNumber);
var prRepo = pullRequest.Head.Repository.SshUrl;
var prBranch = pullRequest.Head.Ref;
var prSha = repository.Fetch(prRepo, prBranch);
if (prSha == null)
{
Console.Error.WriteLine("Error: Could not fetch pull request branch {0} from repo {1}", prBranch, prRepo);
Environment.Exit(1);
}
var masterSha = repository.Fetch("[email protected]:mono/mono.git", "master");
if (masterSha == null)
{
Console.Error.WriteLine("Error: Could not fetch master.");
Environment.Exit(1);
}
var baseSha = repository.MergeBase(prSha, masterSha);
if (baseSha == null)
{
Console.Error.WriteLine("Error: Could not determine merge base of pull request.");
Environment.Exit(1);
}
Console.WriteLine("Merge base sha is {0}", baseSha);
var revList = repository.RevList(baseSha);
if (revList == null)
{
Console.Error.WriteLine("Error: Could not get rev-list for merge base {0}.", baseSha);
Environment.Exit(1);
}
Console.WriteLine("{0} commits in rev-list", revList.Length);
var configObj = await config.GetFromParse();
if (configObj == null)
{
Console.Error.WriteLine("Error: The config does not exist.");
Environment.Exit(1);
}
var machineObj = await RunSet.GetMachineFromParse();
if (machineObj == null)
{
Console.Error.WriteLine("Error: The machine does not exist.");
Environment.Exit(1);
}
var runSets = await ParseInterface.PageQueryWithRetry(() => ParseObject.GetQuery ("RunSet")
.WhereEqualTo ("machine", machineObj)
.WhereEqualTo ("config", configObj)
.WhereDoesNotExist ("pullRequest")
.Include("commit"));
Console.WriteLine("{0} run sets", runSets.Count());
var runSetsByCommits = new Dictionary <string, ParseObject> ();
foreach (var runSet in runSets)
{
var sha = runSet.Get <ParseObject> ("commit").Get <string> ("hash");
if (runSetsByCommits.ContainsKey(sha))
{
// FIXME: select between them?
continue;
}
runSetsByCommits.Add(sha, runSet);
}
foreach (var sha in revList)
{
if (runSetsByCommits.ContainsKey(sha))
{
Console.WriteLine("tested base commit is {0}", sha);
return(runSetsByCommits [sha]);
}
}
return(null);
}
public static void Main(string[] args)
{
string[] benchmarkNames = null;
//var pausetime = false;
var timeout = -1;
string commitFromCmdline = null;
string gitRepoDir = null;
string rootFromCmdline = null;
string buildURL = null;
string logURL = null;
string pullRequestURL = null;
string monoRepositoryPath = null;
string runSetId = null;
string configFile = null;
bool justCreateRunSet = false;
bool justListBenchmarks = false;
var optindex = 0;
for (; optindex < args.Length; ++optindex)
{
if (args [optindex] == "-b" || args [optindex] == "--benchmarks")
{
var newNames = args [++optindex].Split(',').Select(s => s.Trim());
if (benchmarkNames == null)
{
benchmarkNames = newNames.ToArray();
}
else
{
benchmarkNames = newNames.Union(benchmarkNames).ToArray();
}
}
else if (args [optindex] == "-c" || args [optindex] == "--config-file")
{
configFile = args [++optindex];
}
else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks")
{
justListBenchmarks = true;
}
else if (args [optindex] == "--commit")
{
commitFromCmdline = args [++optindex];
}
else if (args [optindex] == "--git-repo")
{
gitRepoDir = args [++optindex];
}
else if (args [optindex] == "--build-url")
{
buildURL = args [++optindex];
}
else if (args [optindex] == "--log-url")
{
logURL = args [++optindex];
}
else if (args [optindex] == "--pull-request-url")
{
pullRequestURL = args [++optindex];
}
else if (args [optindex] == "--mono-repository")
{
monoRepositoryPath = args [++optindex];
}
else if (args [optindex] == "--create-run-set")
{
justCreateRunSet = true;
}
else if (args [optindex] == "--run-set-id")
{
runSetId = args [++optindex];
}
else if (args [optindex] == "--root")
{
rootFromCmdline = args [++optindex];
}
else if (args [optindex] == "-t" || args [optindex] == "--timeout")
{
timeout = Int32.Parse(args [++optindex]);
timeout = timeout <= 0 ? -1 : timeout;
// } else if (args [optindex] == "-p" || args [optindex] == "--pause-time") {
// pausetime = Boolean.Parse (args [++optindex]);
}
else if (args [optindex].StartsWith("--help"))
{
UsageAndExit();
}
else if (args [optindex] == "--")
{
optindex += 1;
break;
}
else if (args [optindex].StartsWith("-"))
{
Console.Error.WriteLine("unknown parameter {0}", args [optindex]);
UsageAndExit();
}
else
{
break;
}
}
if (justCreateRunSet && runSetId != null)
{
Console.Error.WriteLine("Error: --create-run-set and --run-set-id are incompatible.");
Environment.Exit(1);
}
if (justListBenchmarks && benchmarkNames != null)
{
Console.Error.WriteLine("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible.");
Environment.Exit(1);
}
string testsDir, benchmarksDir, machinesDir;
if (args.Length - optindex == 4)
{
if (configFile != null)
{
Console.Error.WriteLine("Error: You must not specify the config file twice.");
Environment.Exit(1);
}
testsDir = args [optindex++];
benchmarksDir = args [optindex++];
machinesDir = args [optindex++];
configFile = args [optindex++];
}
else if (args.Length - optindex == 0)
{
var exeLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
var exeName = Path.GetFileName(exeLocation);
var exeDir = Path.GetDirectoryName(exeLocation);
if (exeName != "compare.exe")
{
Console.Error.WriteLine("Error: Executable is not `compare.exe`. Please specify all paths manually.");
Environment.Exit(1);
}
if (Path.GetFileName(exeDir) != "tools")
{
Console.Error.WriteLine("Error: Executable is not in the `tools` directory. Please specify all paths manually.");
Environment.Exit(1);
}
var root = Path.GetDirectoryName(exeDir);
testsDir = Path.Combine(root, "tests");
benchmarksDir = Path.Combine(root, "benchmarks");
machinesDir = Path.Combine(root, "machines");
if (configFile == null)
{
configFile = Path.Combine(root, "configs", "default-sgen.conf");
}
}
else
{
UsageAndExit(null, 1);
return;
}
var benchmarks = Benchmark.LoadAllFrom(benchmarksDir, benchmarkNames);
if (benchmarks == null)
{
Console.WriteLine("Error: Could not load all benchmarks.");
Environment.Exit(1);
}
if (justListBenchmarks)
{
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
Console.Out.WriteLine(benchmark.Name);
}
Environment.Exit(0);
}
var gitHubClient = GitHubInterface.GitHubClient;
if (!ParseInterface.Initialize())
{
Console.Error.WriteLine("Error: Could not initialize Parse interface.");
Environment.Exit(1);
}
var config = Config.LoadFrom(configFile, rootFromCmdline);
var machine = Machine.LoadCurrentFrom(machinesDir);
var commit = AsyncContext.Run(() => config.GetCommit(commitFromCmdline, gitRepoDir));
if (commit == null)
{
Console.WriteLine("Error: Could not get commit");
Environment.Exit(1);
}
if (commit.CommitDate == null)
{
Console.WriteLine("Error: Could not get a commit date.");
Environment.Exit(1);
}
RunSet runSet;
if (runSetId != null)
{
if (pullRequestURL != null)
{
Console.WriteLine("Error: Pull request URL cannot be specified for an existing run set.");
Environment.Exit(1);
}
runSet = AsyncContext.Run(() => RunSet.FromId(runSetId, config, commit, buildURL, logURL));
if (runSet == null)
{
Console.WriteLine("Error: Could not get run set.");
Environment.Exit(1);
}
}
else
{
ParseObject pullRequestBaselineRunSet = null;
if (pullRequestURL != null)
{
if (monoRepositoryPath == null)
{
Console.Error.WriteLine("Error: Must specify a mono repository path to test a pull request.");
Environment.Exit(1);
}
var repo = new Benchmarker.Common.Git.Repository(monoRepositoryPath);
pullRequestBaselineRunSet = AsyncContext.Run(() => GetPullRequestBaselineRunSet(pullRequestURL, repo, config));
}
runSet = new RunSet {
StartDateTime = DateTime.Now,
Config = config,
Commit = commit,
BuildURL = buildURL,
LogURL = logURL,
PullRequestURL = pullRequestURL,
PullRequestBaselineRunSet = pullRequestBaselineRunSet
};
}
if (!justCreateRunSet)
{
var someSuccess = false;
foreach (var benchmark in benchmarks.OrderBy(b => b.Name))
{
/* Run the benchmarks */
if (config.Count <= 0)
{
throw new ArgumentOutOfRangeException(String.Format("configs [\"{0}\"].Count <= 0", config.Name));
}
Console.Out.WriteLine("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name);
var runner = new Runner(testsDir, config, benchmark, machine, timeout);
var result = new Result {
DateTime = DateTime.Now,
Benchmark = benchmark,
Config = config,
};
var haveTimedOut = false;
var haveCrashed = false;
for (var i = 0; i < config.Count + 1; ++i)
{
bool timedOut;
Console.Out.Write("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format("({0}/{1})", i, config.Count));
var run = runner.Run(out timedOut);
// skip first one
if (i == 0)
{
continue;
}
if (run != null)
{
result.Runs.Add(run);
someSuccess = true;
}
else
{
if (timedOut)
{
haveTimedOut = true;
}
else
{
haveCrashed = true;
}
}
}
if (haveTimedOut)
{
runSet.TimedOutBenchmarks.Add(benchmark);
}
if (haveCrashed)
{
runSet.CrashedBenchmarks.Add(benchmark);
}
// FIXME: implement pausetime
//if (pausetime)
// throw new NotImplementedException ();
runSet.Results.Add(result);
}
if (!someSuccess)
{
Console.WriteLine("all runs failed.");
}
}
runSet.FinishDateTime = DateTime.Now;
Console.WriteLine("uploading");
try {
var parseObject = AsyncContext.Run(() => runSet.UploadToParse());
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/runset.html#{0}", parseObject.ObjectId);
ParseObject pullRequestObject = null;
if (pullRequestURL != null)
{
pullRequestObject = parseObject.Get <ParseObject> ("pullRequest");
Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#{0}", pullRequestObject.ObjectId);
}
Console.Write("{{ \"runSetId\": \"{0}\"", parseObject.ObjectId);
if (pullRequestURL != null)
{
Console.Write(", \"pullRequestId\": \"{0}\"", pullRequestObject.ObjectId);
}
Console.WriteLine(" }");
} catch (Exception exc) {
Console.WriteLine("Error: Failure uploading data: " + exc);
Environment.Exit(1);
}
}
public static string CreateServerServiceCode(ParseInterface parse)
{
return("");
}
public static string CreateClientInterfaceCode(ParseInterface parse)
{
return("");
}
