static async Task <Tuple <long, string> > GetPullRequestBaselineRunSetId(Product product, string pullRequestURL, compare.Repository repository, Config config) { var gitHubClient = GitHubInterface.GitHubClient; var match = Regex.Match(pullRequestURL, product.PullRequestRegexp); if (match == null) { Console.Error.WriteLine("Error: Cannot parse pull request URL."); Environment.Exit(1); } var pullRequestNumber = Int32.Parse(match.Groups [1].Value); Console.WriteLine("pull request {0}", pullRequestNumber); var pullRequest = await gitHubClient.PullRequest.Get("mono", "mono", pullRequestNumber); var prRepo = pullRequest.Head.Repository.CloneUrl; var prBranch = pullRequest.Head.Ref; var prSha = repository.Fetch(prRepo, prBranch); if (prSha == null) { Console.Error.WriteLine("Error: Could not fetch pull request branch {0} from repo {1}", prBranch, prRepo); Environment.Exit(1); } var masterSha = repository.Fetch(product.GitRepositoryUrl, "master"); if (masterSha == null) { Console.Error.WriteLine("Error: Could not fetch master."); Environment.Exit(1); } var baseSha = repository.MergeBase(prSha, masterSha); if (baseSha == null) { Console.Error.WriteLine("Error: Could not determine merge base of pull request."); Environment.Exit(1); } Console.WriteLine("Merge base sha is {0}", baseSha); var revList = repository.RevList(baseSha); if (revList == null) { Console.Error.WriteLine("Error: Could not get rev-list for merge base {0}.", baseSha); Environment.Exit(1); } Console.WriteLine("{0} commits in rev-list", revList.Length); // FIXME: also support `--machine` var hostarch = compare.Utils.LocalHostnameAndArch(); var machine = new Machine { Name = hostarch.Item1, Architecture = hostarch.Item2 }; JArray runsets = await HttpApi.GetRunsets(machine.Name, config.Name); if (runsets == null) { Console.Error.WriteLine("Error: Could not get run sets."); Environment.Exit(1); } var runSetIdsByCommits = new Dictionary <string, long> (); foreach (var rs in runsets) { var id = rs ["ID"].ToObject <long> (); var commit = rs ["MainProduct"] ["Commit"].ToObject <string> (); runSetIdsByCommits [commit] = id; } Console.WriteLine("{0} run sets", runSetIdsByCommits.Count); foreach (var sha in revList) { if (runSetIdsByCommits.ContainsKey(sha)) { Console.WriteLine("tested base commit is {0}", sha); return(Tuple.Create(runSetIdsByCommits [sha], baseSha)); } } return(null); }
public async static Task<bool> CompleteCommit (Config cfg, Commit commit) { if (commit.Product.Name == "mono" && !cfg.NoMono) { var binaryProtocolFile = cfg.ProducesBinaryProtocol ? "/tmp/binprot.dummy" : null; var info = NewProcessStartInfo (cfg, binaryProtocolFile); if (!String.IsNullOrWhiteSpace (info.FileName)) { /* Run without timing with --version */ info.Arguments = "--version"; Console.Out.WriteLine ("\t$> {0} {1} {2}", PrintableEnvironmentVariables (info), info.FileName, info.Arguments); var process = Process.Start (info); var version = Task.Run (() => new StreamReader (process.StandardOutput.BaseStream).ReadToEnd ()).Result; var versionError = Task.Run (() => new StreamReader (process.StandardError.BaseStream).ReadToEnd ()).Result; process.WaitForExit (); process.Close (); var line = version.Split (new char[] { '\n' }, 2) [0]; var regex = new Regex ("^Mono JIT.*\\((.*)/([0-9a-f]+) (.*)\\)"); var match = regex.Match (line); if (match.Success) { commit.Branch = match.Groups [1].Value; var hash = match.Groups [2].Value; if (commit.Hash != null) { if (!commit.Hash.StartsWith (hash)) { Console.Error.WriteLine ("Error: Commit hash for mono specified on command line does not match the one reported with --version."); return false; } } else { commit.Hash = hash; } var date = match.Groups [3].Value; Console.WriteLine ("branch: " + commit.Branch + " hash: " + commit.Hash + " date: " + date); } } if (commit.Branch == "(detached") commit.Branch = null; try { var gitRepoDir = Path.GetDirectoryName (cfg.Mono); var repo = new Repository (gitRepoDir); var gitHash = repo.RevParse (commit.Hash); if (gitHash == null) { Console.WriteLine ("Could not get commit " + commit.Hash + " from repository"); } else { Console.WriteLine ("Got commit " + gitHash + " from repository"); if (commit.Hash != null && commit.Hash != gitHash) { Console.Error.WriteLine ("Error: Commit hash specified on command line does not match the one from the git repository."); return false; } commit.Hash = gitHash; commit.MergeBaseHash = repo.MergeBase (commit.Hash, "master"); commit.CommitDate = repo.CommitDate (commit.Hash); if (commit.CommitDate == null) { Console.Error.WriteLine ("Error: Could not get commit date from the git repository."); return false; } Console.WriteLine ("Commit {0} merge base {1} date {2}", commit.Hash, commit.MergeBaseHash, commit.CommitDate); } } catch (Exception) { Console.WriteLine ("Could not get git repository"); } } if (commit.Hash == null) { Console.Error.WriteLine ("Error: cannot parse mono version and no commit given."); return false; } Octokit.Commit gitHubCommit = await ResolveFullHashViaGithub (commit); if (gitHubCommit == null) { Console.WriteLine ("Could not get commit " + commit.Hash + " from GitHub"); } else { commit.Hash = gitHubCommit.Sha; if (commit.CommitDate == null) commit.CommitDate = gitHubCommit.Committer.Date.DateTime.ToLocalTime (); Console.WriteLine ("Got commit " + commit.Hash + " from GitHub"); } if (commit.CommitDate == null) { Console.Error.WriteLine ("Error: Could not get a commit date."); return false; } return true; }
public static int Main(string[] args) { IEnumerable <string> benchmarkNames = null; //var pausetime = false; var timeout = -1; string rootFromCmdline = null; string buildURL = null; string logURL = null; string pullRequestURL = null; string monoRepositoryPath = null; long? runSetId = null; long? runId = null; string configFile = null; string machineName = null; bool justCreateRunSet = false; bool justListBenchmarks = false; string valgrindBinary = null; ValgrindTool? valgrindTool = null; string valgrindOutputFilename = null; string grepBinprotPath = null; string binprotFilePath = null; bool jitStats = false; Commit mainCommit = null; List <Commit> secondaryCommits = new List <Commit> (); var exeLocation = System.Reflection.Assembly.GetEntryAssembly().Location; var exeName = Path.GetFileName(exeLocation); var exeDir = Path.GetDirectoryName(exeLocation); if (exeName != "compare.exe") { Console.Error.WriteLine("Error: Executable is not `compare.exe`. Please specify all paths manually."); Environment.Exit(1); } if (Path.GetFileName(exeDir) != "tools") { Console.Error.WriteLine("Error: Executable is not in the `tools` directory. Please specify all paths manually."); Environment.Exit(1); } var root = Path.GetDirectoryName(exeDir); var testsDir = Path.Combine(root, "tests"); var benchmarksDir = Path.Combine(root, "benchmarks"); var machinesDir = Path.Combine(root, "machines"); var productsDir = Path.Combine(root, "products"); var optindex = 0; for (; optindex < args.Length; ++optindex) { if (args [optindex] == "-b" || args [optindex] == "--benchmarks") { var newNames = args [++optindex].Split(',').Select(s => s.Trim()); if (benchmarkNames == null) { benchmarkNames = newNames.ToArray(); } else { benchmarkNames = newNames.Union(benchmarkNames).ToArray(); } } else if (args [optindex] == "-c" || args [optindex] == "--config-file") { configFile = args [++optindex]; } else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks") { justListBenchmarks = true; } else if (args [optindex] == "--machine") { machineName = args [++optindex]; } else if (args [optindex] == "--build-url") { buildURL = args [++optindex]; } else if (args [optindex] == "--log-url") { logURL = args [++optindex]; } else if (args [optindex] == "--pull-request-url") { pullRequestURL = args [++optindex]; } else if (args [optindex] == "--mono-repository") { monoRepositoryPath = args [++optindex]; } else if (args [optindex] == "--create-run-set") { justCreateRunSet = true; } else if (args [optindex] == "--run-set-id") { runSetId = Int64.Parse(args [++optindex]); } else if (args [optindex] == "--run-id") { runId = Int64.Parse(args [++optindex]); } else if (args [optindex] == "--root") { rootFromCmdline = args [++optindex]; } else if (args [optindex] == "--main-product") { var name = args [++optindex]; var hash = args [++optindex]; if (mainCommit != null) { Console.Error.WriteLine("Error: Only one --main-product is supported."); UsageAndExit(); } var product = compare.Utils.LoadProductFromFile(name, productsDir); mainCommit = new Commit { Product = product, Hash = hash }; } else if (args [optindex] == "--secondary-product") { var name = args [++optindex]; var hash = args [++optindex]; var product = compare.Utils.LoadProductFromFile(name, productsDir); secondaryCommits.Add(new Commit { Product = product, Hash = hash }); } else if (args [optindex] == "--valgrind-massif") { if (valgrindBinary != null) { Console.Error.WriteLine("Error: More than one Valgrind option given."); UsageAndExit(); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Massif; } else if (args [optindex] == "--valgrind-cachegrind") { if (valgrindBinary != null) { Console.Error.WriteLine("Error: More than one Valgrind option given."); UsageAndExit(); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Cachegrind; } else if (args [optindex] == "-t" || args [optindex] == "--timeout") { timeout = Int32.Parse(args [++optindex]); timeout = timeout <= 0 ? -1 : timeout; } else if (args [optindex] == "--sgen-grep-binprot") { grepBinprotPath = args [++optindex]; } else if (args [optindex] == "--upload-pause-times") { binprotFilePath = args [++optindex]; } else if (args [optindex] == "--jit-stats") { jitStats = true; } else if (args [optindex].StartsWith("--help")) { UsageAndExit(); } else if (args [optindex] == "--") { optindex += 1; break; } else if (args [optindex].StartsWith("-")) { Console.Error.WriteLine("unknown parameter {0}", args [optindex]); UsageAndExit(); } else { break; } } var configFileFromCommandLine = configFile != null; if (!configFileFromCommandLine) { configFile = Path.Combine(root, "configs", "default-sgen.conf"); } var config = compare.Utils.LoadConfigFromFile(configFile, rootFromCmdline, !(justListBenchmarks || binprotFilePath != null)); if (justCreateRunSet && runSetId != null) { Console.Error.WriteLine("Error: --create-run-set and --run-set-id are incompatible."); Environment.Exit(1); } if (justListBenchmarks && benchmarkNames != null) { Console.Error.WriteLine("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible."); Environment.Exit(1); } if (justListBenchmarks && !configFileFromCommandLine) { Console.Error.WriteLine("Error: -l/--list-benchmarks requires --config-file."); Environment.Exit(1); } if (args.Length - optindex != 0) { return(UsageAndExit(null, 1)); } if (binprotFilePath != null && (runId == null || grepBinprotPath == null)) { Console.Error.WriteLine("Error: --upload-pause-times also requires --run-id and --sgen-grep-binprot."); Environment.Exit(1); } if (benchmarkNames == null) { benchmarkNames = config.Benchmarks; } var benchmarks = compare.Utils.LoadAllBenchmarksFrom(benchmarksDir, benchmarkNames); if (benchmarks == null) { Console.Error.WriteLine("Error: Could not load all benchmarks."); Environment.Exit(1); } if (justListBenchmarks) { if (machineName != null) { var listMachine = compare.Utils.LoadMachineFromFile(machineName, machinesDir); if (listMachine == null) { Console.Error.WriteLine("Error: Could not load machine `{0}`.", machineName); Environment.Exit(1); } if (listMachine.ExcludeBenchmarks != null) { benchmarks = benchmarks.Where(b => !listMachine.ExcludeBenchmarks.Contains(b.Name)).ToList(); } } foreach (var benchmark in benchmarks.OrderBy(b => b.Name)) { Console.Out.WriteLine(benchmark.Name); } Environment.Exit(0); } InitCommons(); if (binprotFilePath != null) { var success = AsyncContext.Run(() => UploadPauseTimes(binprotFilePath, grepBinprotPath, runId.Value)); Environment.Exit(success ? 0 : 1); } if (mainCommit == null) { mainCommit = new Commit { Product = compare.Utils.LoadProductFromFile("mono", productsDir) } } ; var gitHubClient = GitHubInterface.GitHubClient; Machine machine = null; if (machineName == null) { machine = compare.Utils.LoadMachineCurrentFrom(machinesDir); } else { machine = compare.Utils.LoadMachineFromFile(machineName, machinesDir); } if (machine != null && machine.ExcludeBenchmarks != null) { benchmarks = benchmarks.Where(b => !machine.ExcludeBenchmarks.Contains(b.Name)).ToList(); } if (machine == null) // couldn't find machine file { var hostarch = compare.Utils.LocalHostnameAndArch(); machine = new Machine(); machine.Name = hostarch.Item1; machine.Architecture = hostarch.Item2; } foreach (var commit in new Commit[] { mainCommit }.Concat(secondaryCommits)) { if (!AsyncContext.Run(() => compare.Utils.CompleteCommit(config, commit))) { Console.Error.WriteLine("Error: Could not get commit for product {0}.", commit.Product.Name); Environment.Exit(1); } } RunSet runSet; if (runSetId != null) { if (pullRequestURL != null) { Console.Error.WriteLine("Error: Pull request URL cannot be specified for an existing run set."); Environment.Exit(1); } runSet = AsyncContext.Run(() => RunSet.FromId(machine, runSetId.Value, config, mainCommit, secondaryCommits, buildURL, logURL)); if (runSet == null) { Console.Error.WriteLine("Error: Could not get run set."); Environment.Exit(1); } } else { long?pullRequestBaselineRunSetId = null; if (pullRequestURL != null) { if (monoRepositoryPath == null) { Console.Error.WriteLine("Error: Must specify a mono repository path to test a pull request."); Environment.Exit(1); } var repo = new compare.Repository(monoRepositoryPath); var baselineResult = AsyncContext.Run(() => GetPullRequestBaselineRunSetId(mainCommit.Product, pullRequestURL, repo, config)); if (baselineResult == null) { Console.Error.WriteLine("Error: No appropriate baseline run set found."); Environment.Exit(1); } pullRequestBaselineRunSetId = baselineResult.Item1; mainCommit.MergeBaseHash = baselineResult.Item2; } runSet = new RunSet { StartDateTime = DateTime.Now, Machine = machine, Config = config, Commit = mainCommit, SecondaryCommits = secondaryCommits, BuildURL = buildURL, LogURL = logURL, PullRequestURL = pullRequestURL, PullRequestBaselineRunSetId = pullRequestBaselineRunSetId }; Console.Error.WriteLine("Set start time to {0}", runSet.StartDateTime.ToString(RunSet.DATETIME_PRETTY)); } var reportFailure = false; if (!justCreateRunSet) { var someSuccess = false; var runTool = valgrindBinary; string runToolArguments = null; if (runTool != null) { switch (valgrindTool) { case ValgrindTool.Massif: runToolArguments = string.Format("--tool=massif --massif-out-file={0} --max-snapshots=1000 --detailed-freq=100 --pages-as-heap=yes", valgrindOutputFilename); break; case ValgrindTool.Cachegrind: runToolArguments = string.Format("--tool=cachegrind --cachegrind-out-file={0} --cache-sim=yes --branch-sim=yes", valgrindOutputFilename); break; default: Console.Error.WriteLine("Error: Unsupported Valgrind tool."); Environment.Exit(1); break; } } int binaryProtocolIndex = 0; foreach (var benchmark in benchmarks.OrderBy(b => b.Name)) { // Run the benchmarks if (config.Count <= 0) { throw new ArgumentOutOfRangeException(String.Format("configs [\"{0}\"].Count <= 0", config.Name)); } Console.Out.WriteLine("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name); var runner = new compare.UnixRunner(testsDir, config, benchmark, machine, timeout, runTool, runToolArguments); var haveTimedOut = false; var haveCrashed = false; var count = valgrindBinary == null ? config.Count + 1 : 1; var successCount = 0; for (var i = 0; i < count; ++i) { bool timedOut; string stdoutOutput; if (valgrindBinary == null) { Console.Out.Write("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format("({0}/{1})", i, config.Count)); } string binaryProtocolFile = null; string workingDirectory = Path.Combine(testsDir, benchmark.TestDirectory); if (config.ProducesBinaryProtocol) { do { ++binaryProtocolIndex; binaryProtocolFile = Path.Combine(workingDirectory, string.Format("binprot.{0}", binaryProtocolIndex)); } while (File.Exists(binaryProtocolFile)); } var elapsedMilliseconds = runner.Run(binaryProtocolFile, out timedOut, out stdoutOutput); // if running for time, the first one is the dry run if (valgrindBinary == null && i == 0) { continue; } if (elapsedMilliseconds != null) { var run = new Run { Benchmark = benchmark, BinaryProtocolFilename = binaryProtocolFile == null ? null : Path.Combine(workingDirectory, binaryProtocolFile) }; if (valgrindBinary == null) { run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.Time, Value = TimeSpan.FromMilliseconds(elapsedMilliseconds.Value) }); if (jitStats) { foreach (var phase in ParseJITPhases(stdoutOutput)) { run.RunMetrics.Add(phase); } } } else { switch (valgrindTool) { case ValgrindTool.Massif: { var results = MemoryIntegral(valgrindOutputFilename); run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.MemoryIntegral, Value = results.Item1 }); run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.Instructions, Value = results.Item2 }); } break; case ValgrindTool.Cachegrind: { var results = CacheAndBranches(valgrindOutputFilename); run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.CachegrindResults, Value = results.Item1 }); run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.CacheMissRate, Value = results.Item2 }); run.RunMetrics.Add(new RunMetric { Metric = RunMetric.MetricType.BranchMispredictionRate, Value = results.Item3 }); } break; } } runSet.Runs.Add(run); successCount++; someSuccess = true; } else { if (timedOut) { haveTimedOut = true; } else { haveCrashed = true; } } } if (haveTimedOut) { runSet.TimedOutBenchmarks.Add(benchmark.Name); } if (haveCrashed) { runSet.CrashedBenchmarks.Add(benchmark.Name); } if (haveTimedOut || successCount == 0) { reportFailure = true; } } if (!someSuccess) { Console.WriteLine("all runs failed."); } } runSet.FinishDateTime = DateTime.Now; Console.Error.WriteLine("Start time is {0} - finish time is {1}", runSet.StartDateTime.ToString(RunSet.DATETIME_PRETTY), runSet.FinishDateTime.ToString(RunSet.DATETIME_PRETTY)); Console.WriteLine(JsonConvert.SerializeObject(runSet.AsDict())); var uploadResult = AsyncContext.Run(() => runSet.Upload()); if (uploadResult == null) { Console.Error.WriteLine("Error: Could not upload run set."); Environment.Exit(1); } Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/runset.html#id={0}", uploadResult.RunSetId); if (pullRequestURL != null) { Console.WriteLine("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#id={0}", uploadResult.PullRequestId.Value); } Console.Write("{{ \"runSetId\": \"{0}\"", uploadResult.RunSetId); if (pullRequestURL != null) { Console.Write(", \"pullRequestId\": \"{0}\"", uploadResult.PullRequestId.Value); } Console.Write(", \"runs\": [ "); var runStrings = new List <string> (); var allRuns = runSet.Runs.ToList(); for (var i = 0; i < allRuns.Count; i++) { var run = allRuns [i]; var id = uploadResult.RunIds [i]; var str = string.Format("\"id\": {0}", id); if (run.BinaryProtocolFilename != null) { str = string.Format("{0}, \"binaryProtocolFile\": \"{1}\"", str, run.BinaryProtocolFilename); } runStrings.Add("{ " + str + " }"); } Console.Write(string.Join(", ", runStrings)); Console.Write(" ]"); Console.WriteLine(" }"); if (reportFailure) { Console.Error.WriteLine("Error: Some benchmarks timed out or failed completely."); return(1); } return(0); } }
public static int Main (string[] args) { IEnumerable<string> benchmarkNames = null; //var pausetime = false; var timeout = -1; string rootFromCmdline = null; string buildURL = null; string logURL = null; string pullRequestURL = null; string monoRepositoryPath = null; long? runSetId = null; long? runId = null; string configFile = null; string machineName = null; bool justCreateRunSet = false; bool justListBenchmarks = false; string valgrindBinary = null; ValgrindTool? valgrindTool = null; string valgrindOutputFilename = null; string grepBinprotPath = null; string binprotFilePath = null; bool jitStats = false; Commit mainCommit = null; List<Commit> secondaryCommits = new List<Commit> (); var exeLocation = System.Reflection.Assembly.GetEntryAssembly ().Location; var exeName = Path.GetFileName (exeLocation); var exeDir = Path.GetDirectoryName (exeLocation); if (exeName != "compare.exe") { Console.Error.WriteLine ("Error: Executable is not `compare.exe`. Please specify all paths manually."); Environment.Exit (1); } if (Path.GetFileName (exeDir) != "tools") { Console.Error.WriteLine ("Error: Executable is not in the `tools` directory. Please specify all paths manually."); Environment.Exit (1); } var root = Path.GetDirectoryName (exeDir); var testsDir = Path.Combine (root, "tests"); var benchmarksDir = Path.Combine (root, "benchmarks"); var machinesDir = Path.Combine (root, "machines"); var productsDir = Path.Combine (root, "products"); var optindex = 0; for (; optindex < args.Length; ++optindex) { if (args [optindex] == "-b" || args [optindex] == "--benchmarks") { var newNames = args [++optindex].Split (',').Select (s => s.Trim ()); if (benchmarkNames == null) benchmarkNames = newNames.ToArray (); else benchmarkNames = newNames.Union (benchmarkNames).ToArray (); } else if (args [optindex] == "-c" || args [optindex] == "--config-file") { configFile = args [++optindex]; } else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks") { justListBenchmarks = true; } else if (args [optindex] == "--machine") { machineName = args [++optindex]; } else if (args [optindex] == "--build-url") { buildURL = args [++optindex]; } else if (args [optindex] == "--log-url") { logURL = args [++optindex]; } else if (args [optindex] == "--pull-request-url") { pullRequestURL = args [++optindex]; } else if (args [optindex] == "--mono-repository") { monoRepositoryPath = args [++optindex]; } else if (args [optindex] == "--create-run-set") { justCreateRunSet = true; } else if (args [optindex] == "--run-set-id") { runSetId = Int64.Parse (args [++optindex]); } else if (args [optindex] == "--run-id") { runId = Int64.Parse (args [++optindex]); } else if (args [optindex] == "--root") { rootFromCmdline = args [++optindex]; } else if (args [optindex] == "--main-product") { var name = args [++optindex]; var hash = args [++optindex]; if (mainCommit != null) { Console.Error.WriteLine ("Error: Only one --main-product is supported."); UsageAndExit (); } var product = compare.Utils.LoadProductFromFile (name, productsDir); mainCommit = new Commit { Product = product, Hash = hash }; } else if (args [optindex] == "--secondary-product") { var name = args [++optindex]; var hash = args [++optindex]; var product = compare.Utils.LoadProductFromFile (name, productsDir); secondaryCommits.Add (new Commit { Product = product, Hash = hash }); } else if (args [optindex] == "--valgrind-massif") { if (valgrindBinary != null) { Console.Error.WriteLine ("Error: More than one Valgrind option given."); UsageAndExit (); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Massif; } else if (args [optindex] == "--valgrind-cachegrind") { if (valgrindBinary != null) { Console.Error.WriteLine ("Error: More than one Valgrind option given."); UsageAndExit (); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Cachegrind; } else if (args [optindex] == "-t" || args [optindex] == "--timeout") { timeout = Int32.Parse (args [++optindex]); timeout = timeout <= 0 ? -1 : timeout; } else if (args [optindex] == "--sgen-grep-binprot") { grepBinprotPath = args [++optindex]; } else if (args [optindex] == "--upload-pause-times") { binprotFilePath = args [++optindex]; } else if (args [optindex] == "--jit-stats") { jitStats = true; } else if (args [optindex].StartsWith ("--help")) { UsageAndExit (); } else if (args [optindex] == "--") { optindex += 1; break; } else if (args [optindex].StartsWith ("-")) { Console.Error.WriteLine ("unknown parameter {0}", args [optindex]); UsageAndExit (); } else { break; } } var configFileFromCommandLine = configFile != null; if (!configFileFromCommandLine) configFile = Path.Combine (root, "configs", "default-sgen.conf"); var config = compare.Utils.LoadConfigFromFile (configFile, rootFromCmdline, !(justListBenchmarks || binprotFilePath != null)); if (justCreateRunSet && runSetId != null) { Console.Error.WriteLine ("Error: --create-run-set and --run-set-id are incompatible."); Environment.Exit (1); } if (justListBenchmarks && benchmarkNames != null) { Console.Error.WriteLine ("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible."); Environment.Exit (1); } if (justListBenchmarks && !configFileFromCommandLine) { Console.Error.WriteLine ("Error: -l/--list-benchmarks requires --config-file."); Environment.Exit (1); } if (args.Length - optindex != 0) return UsageAndExit (null, 1); if (binprotFilePath != null && (runId == null || grepBinprotPath == null)) { Console.Error.WriteLine ("Error: --upload-pause-times also requires --run-id and --sgen-grep-binprot."); Environment.Exit (1); } if (benchmarkNames == null) benchmarkNames = config.Benchmarks; var benchmarks = compare.Utils.LoadAllBenchmarksFrom (benchmarksDir, benchmarkNames); if (benchmarks == null) { Console.Error.WriteLine ("Error: Could not load all benchmarks."); Environment.Exit (1); } if (justListBenchmarks) { if (machineName != null) { var listMachine = compare.Utils.LoadMachineFromFile (machineName, machinesDir); if (listMachine == null) { Console.Error.WriteLine ("Error: Could not load machine `{0}`.", machineName); Environment.Exit (1); } if (listMachine.ExcludeBenchmarks != null) benchmarks = benchmarks.Where (b => !listMachine.ExcludeBenchmarks.Contains (b.Name)).ToList (); } foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) { Console.Out.WriteLine (benchmark.Name); } Environment.Exit (0); } InitCommons (); if (binprotFilePath != null) { var success = AsyncContext.Run (() => UploadPauseTimes (binprotFilePath, grepBinprotPath, runId.Value)); Environment.Exit (success ? 0 : 1); } if (mainCommit == null) mainCommit = new Commit { Product = compare.Utils.LoadProductFromFile ("mono", productsDir) }; var gitHubClient = GitHubInterface.GitHubClient; Machine machine = null; if (machineName == null) { machine = compare.Utils.LoadMachineCurrentFrom (machinesDir); } else { machine = compare.Utils.LoadMachineFromFile (machineName, machinesDir); } if (machine != null && machine.ExcludeBenchmarks != null) benchmarks = benchmarks.Where (b => !machine.ExcludeBenchmarks.Contains (b.Name)).ToList (); if (machine == null) { // couldn't find machine file var hostarch = compare.Utils.LocalHostnameAndArch (); machine = new Machine (); machine.Name = hostarch.Item1; machine.Architecture = hostarch.Item2; } foreach (var commit in new Commit[] { mainCommit }.Concat (secondaryCommits)) { if (!AsyncContext.Run (() => compare.Utils.CompleteCommit (config, commit))) { Console.Error.WriteLine ("Error: Could not get commit for product {0}.", commit.Product.Name); Environment.Exit (1); } } RunSet runSet; if (runSetId != null) { if (pullRequestURL != null) { Console.Error.WriteLine ("Error: Pull request URL cannot be specified for an existing run set."); Environment.Exit (1); } runSet = AsyncContext.Run (() => RunSet.FromId (machine, runSetId.Value, config, mainCommit, secondaryCommits, buildURL, logURL)); if (runSet == null) { Console.Error.WriteLine ("Error: Could not get run set."); Environment.Exit (1); } } else { long? pullRequestBaselineRunSetId = null; if (pullRequestURL != null) { if (monoRepositoryPath == null) { Console.Error.WriteLine ("Error: Must specify a mono repository path to test a pull request."); Environment.Exit (1); } var repo = new compare.Repository (monoRepositoryPath); var baselineResult = AsyncContext.Run (() => GetPullRequestBaselineRunSetId (mainCommit.Product, pullRequestURL, repo, config)); if (baselineResult == null) { Console.Error.WriteLine ("Error: No appropriate baseline run set found."); Environment.Exit (1); } pullRequestBaselineRunSetId = baselineResult.Item1; mainCommit.MergeBaseHash = baselineResult.Item2; } runSet = new RunSet { StartDateTime = DateTime.Now, Machine = machine, Config = config, Commit = mainCommit, SecondaryCommits = secondaryCommits, BuildURL = buildURL, LogURL = logURL, PullRequestURL = pullRequestURL, PullRequestBaselineRunSetId = pullRequestBaselineRunSetId }; Console.Error.WriteLine ("Set start time to {0}", runSet.StartDateTime); } var reportFailure = false; if (!justCreateRunSet) { var someSuccess = false; var runTool = valgrindBinary; string runToolArguments = null; if (runTool != null) { switch (valgrindTool) { case ValgrindTool.Massif: runToolArguments = string.Format ("--tool=massif --massif-out-file={0} --max-snapshots=1000 --detailed-freq=100 --pages-as-heap=yes", valgrindOutputFilename); break; case ValgrindTool.Cachegrind: runToolArguments = string.Format ("--tool=cachegrind --cachegrind-out-file={0} --cache-sim=yes --branch-sim=yes", valgrindOutputFilename); break; default: Console.Error.WriteLine ("Error: Unsupported Valgrind tool."); Environment.Exit (1); break; } } int binaryProtocolIndex = 0; foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) { // Run the benchmarks if (config.Count <= 0) throw new ArgumentOutOfRangeException (String.Format ("configs [\"{0}\"].Count <= 0", config.Name)); Console.Out.WriteLine ("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name); var runner = new compare.UnixRunner (testsDir, config, benchmark, machine, timeout, runTool, runToolArguments); var haveTimedOut = false; var haveCrashed = false; var count = valgrindBinary == null ? config.Count + 1 : 1; var successCount = 0; for (var i = 0; i < count; ++i) { bool timedOut; string stdoutOutput; if (valgrindBinary == null) Console.Out.Write ("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format ("({0}/{1})", i, config.Count)); string binaryProtocolFile = null; string workingDirectory = Path.Combine (testsDir, benchmark.TestDirectory); if (config.ProducesBinaryProtocol) { do { ++binaryProtocolIndex; binaryProtocolFile = Path.Combine(workingDirectory, string.Format ("binprot.{0}", binaryProtocolIndex)); } while (File.Exists (binaryProtocolFile)); } var elapsedMilliseconds = runner.Run (binaryProtocolFile, out timedOut, out stdoutOutput); // if running for time, the first one is the dry run if (valgrindBinary == null && i == 0) continue; if (elapsedMilliseconds != null) { var run = new Run { Benchmark = benchmark, BinaryProtocolFilename = binaryProtocolFile == null ? null : Path.Combine(workingDirectory, binaryProtocolFile) }; if (valgrindBinary == null) { run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.Time, Value = TimeSpan.FromMilliseconds (elapsedMilliseconds.Value) }); if (jitStats) { foreach (var phase in ParseJITPhases (stdoutOutput)) { run.RunMetrics.Add (phase); } } } else { switch (valgrindTool) { case ValgrindTool.Massif: { var results = MemoryIntegral (valgrindOutputFilename); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.MemoryIntegral, Value = results.Item1 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.Instructions, Value = results.Item2 }); } break; case ValgrindTool.Cachegrind: { var results = CacheAndBranches (valgrindOutputFilename); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.CachegrindResults, Value = results.Item1 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.CacheMissRate, Value = results.Item2 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.BranchMispredictionRate, Value = results.Item3 }); } break; } } runSet.Runs.Add (run); successCount++; someSuccess = true; } else { if (timedOut) haveTimedOut = true; else haveCrashed = true; } } if (haveTimedOut) runSet.TimedOutBenchmarks.Add (benchmark.Name); if (haveCrashed) runSet.CrashedBenchmarks.Add (benchmark.Name); if (haveTimedOut || successCount == 0) reportFailure = true; } if (!someSuccess) Console.WriteLine ("all runs failed."); } runSet.FinishDateTime = DateTime.Now; Console.Error.WriteLine ("Start time is {0} - finish time is {1}", runSet.StartDateTime, runSet.FinishDateTime); Console.WriteLine (JsonConvert.SerializeObject (runSet.AsDict ())); var uploadResult = AsyncContext.Run (() => Utils.RunWithRetry (() => runSet.Upload ())); if (uploadResult == null) { Console.Error.WriteLine ("Error: Could not upload run set."); Environment.Exit (1); } Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/runset.html#id={0}", uploadResult.RunSetId); if (pullRequestURL != null) Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#id={0}", uploadResult.PullRequestId.Value); Console.Write ("{{ \"runSetId\": \"{0}\"", uploadResult.RunSetId); if (pullRequestURL != null) Console.Write (", \"pullRequestId\": \"{0}\"", uploadResult.PullRequestId.Value); Console.Write (", \"runs\": [ "); var runStrings = new List<string> (); var allRuns = runSet.Runs.ToList (); for (var i = 0; i < allRuns.Count; i++) { var run = allRuns [i]; var id = uploadResult.RunIds [i]; var str = string.Format ("\"id\": {0}", id); if (run.BinaryProtocolFilename != null) str = string.Format ("{0}, \"binaryProtocolFile\": \"{1}\"", str, run.BinaryProtocolFilename); runStrings.Add ("{ " + str + " }"); } Console.Write (string.Join (", ", runStrings)); Console.Write (" ]"); Console.WriteLine (" }"); if (reportFailure) { Console.Error.WriteLine ("Error: Some benchmarks timed out or failed completely."); return 1; } return 0; }