public static async Task<RunSet> FromId (Machine local_machine, long local_runsetid, Config local_config, Commit local_mainCommit, List<Commit> local_secondaryCommits, string local_buildURL, string local_logURL) { using (var client = new HttpClient ()) { JObject db_result = await HttpApi.GetRunset (local_runsetid); if (db_result == null) { return null; } var runSet = new RunSet { Id = local_runsetid, StartDateTime = db_result ["StartedAt"].ToObject<DateTime> (), FinishDateTime = db_result ["FinishedAt"].ToObject<DateTime> (), BuildURL = db_result ["BuildURL"].ToObject<string> (), Machine = local_machine, LogURL = local_logURL, Config = local_config, Commit = local_mainCommit, TimedOutBenchmarks = db_result ["TimedOutBenchmarks"].ToObject<List<string>> (), CrashedBenchmarks = db_result ["CrashedBenchmarks"].ToObject<List<string>> () }; var db_mainProductCommit = db_result ["MainProduct"] ["Commit"].ToObject<string> (); if (local_mainCommit.Hash != db_mainProductCommit) throw new Exception (String.Format ("Commit ({0}) does not match the one in the database ({1}).", local_mainCommit.Hash, db_mainProductCommit)); var db_secondaryCommits = new List<Commit> (); foreach (var sc in db_result ["SecondaryProducts"]) { db_secondaryCommits.Add (new Commit { Hash = sc ["Commit"].ToObject<string> (), Product = new Product { Name = sc ["Name"].ToObject<string> () } }); } if (local_secondaryCommits != null) { if (local_secondaryCommits.Count != db_secondaryCommits.Count) throw new Exception ("Secondary commits don't match the database."); foreach (var sc in db_secondaryCommits) { if (!local_secondaryCommits.Any (c => c.Hash == sc.Hash && c.Product.Name == sc.Product.Name)) throw new Exception ("Secondary commits don't match the database."); } // local commits have more information (e.g. datetime) runSet.SecondaryCommits = local_secondaryCommits; } else { runSet.SecondaryCommits = db_secondaryCommits; } if (local_buildURL != null && local_buildURL != runSet.BuildURL) throw new Exception ("Build URL does not match the one in the database."); var db_machineName = db_result ["Machine"] ["Name"].ToObject<string> (); var db_machineArchitecture = db_result ["Machine"] ["Architecture"].ToObject<string> (); if (local_machine.Name != db_machineName || local_machine.Architecture != db_machineArchitecture) throw new Exception ("Machine does not match the one in the database. \"" + db_machineName + "\" vs. \"" + local_machine.Name + "\""); if (!local_config.EqualsApiObject (db_result ["Config"])) throw new Exception ("Config does not match the one in the database."); return runSet; } }
void RunBenchmark(long runSetId, string benchmarkName, string machineName, string architecture) { const int DRY_RUNS = 3; const int ITERATIONS = 10; Logging.GetLogging().InfoFormat("Benchmarker | hostname \"{0}\" architecture \"{1}\"", machineName, architecture); Logging.GetLogging().InfoFormat("Benchmarker | configname \"{0}\"", "default"); models.Commit mainCommit = DetermineCommit(); models.Machine machine = new models.Machine { Name = machineName, Architecture = architecture }; models.Config config = new models.Config { Name = "default", Mono = String.Empty, MonoOptions = new string[0], MonoEnvironmentVariables = new Dictionary <string, string> (), Count = ITERATIONS }; models.RunSet runSet = AsyncContext.Run(() => models.RunSet.FromId(machine, runSetId, config, mainCommit, null, null, null /* TODO: logURL? */)); if (runSet == null) { Logging.GetLogging().Warn("RunSetID " + runSetId + " not found"); return; } new Task(() => { try { for (var i = 0; i < (ITERATIONS + DRY_RUNS); i++) { var run = Iteration(benchmarkName, i, i < DRY_RUNS); if (i >= DRY_RUNS) { runSet.Runs.Add(run); } } var result = AsyncContext.Run(() => runSet.Upload()); if (result == null) { RunOnUiThread(() => SetStartButtonText("failed")); } else { RunOnUiThread(() => SetStartButtonText("start")); } } catch (Exception e) { RunOnUiThread(() => SetStartButtonText("failed")); Logging.GetLogging().Error(e); } finally { if (AndroidCPUManagment.IsRooted()) { CpuManager.RestoreCPUStates(); } } }).Start(); }
private static void CreateBenchViewSubmission(string submissionType, RunSet runSet) { Console.WriteLine("Creating BenchView submission json"); var arguments = new string[] { $"\"{s_submissionPy}\"", $"\"{s_measurementJson}\"", $"--metadata=\"{s_submissionMetadataJson}\"", $"--build=\"{s_buildJson}\"", $"--machine-data=\"{s_machinedataJson}\"", $"--group=\"{s_group}\"", $"--type=\"{submissionType}\"", $"--config-name=\"{runSet.Config.Name}\"", $"--config MonoOptions \\\"{string.Join(" ", runSet.Config.MonoOptions)}\\\"", $"--architecture=\"{runSet.Machine.Architecture}\"", $"--machinepool=\"{runSet.Machine.Name}\"", $"-o=\"{s_submissionJson}\"" }; ShellOutVital(s_pythonProcessName, string.Join(" ", arguments)); }
internal static void CreateBenchviewReport(string submissionType, RunSet runSet) { Console.WriteLine("Creating benchview results..."); // Serialize the xamarin/benchmarker object to a file. var jsonConvertedSerializedRunSet = JsonConvert.SerializeObject(runSet); using (var sw = new StreamWriter(s_runSetJsonFileName)) sw.Write(jsonConvertedSerializedRunSet); var result = ConvertToMeasurement(s_runSetJsonFileName); CreateBenchViewSubmission(submissionType, runSet); }
public static int Main (string[] args) { IEnumerable<string> benchmarkNames = null; //var pausetime = false; var timeout = -1; string rootFromCmdline = null; string buildURL = null; string logURL = null; string pullRequestURL = null; string monoRepositoryPath = null; long? runSetId = null; long? runId = null; string configFile = null; string machineName = null; bool justCreateRunSet = false; bool justListBenchmarks = false; string valgrindBinary = null; ValgrindTool? valgrindTool = null; string valgrindOutputFilename = null; string grepBinprotPath = null; string binprotFilePath = null; bool jitStats = false; Commit mainCommit = null; List<Commit> secondaryCommits = new List<Commit> (); var exeLocation = System.Reflection.Assembly.GetEntryAssembly ().Location; var exeName = Path.GetFileName (exeLocation); var exeDir = Path.GetDirectoryName (exeLocation); if (exeName != "compare.exe") { Console.Error.WriteLine ("Error: Executable is not `compare.exe`. Please specify all paths manually."); Environment.Exit (1); } if (Path.GetFileName (exeDir) != "tools") { Console.Error.WriteLine ("Error: Executable is not in the `tools` directory. Please specify all paths manually."); Environment.Exit (1); } var root = Path.GetDirectoryName (exeDir); var testsDir = Path.Combine (root, "tests"); var benchmarksDir = Path.Combine (root, "benchmarks"); var machinesDir = Path.Combine (root, "machines"); var productsDir = Path.Combine (root, "products"); var optindex = 0; for (; optindex < args.Length; ++optindex) { if (args [optindex] == "-b" || args [optindex] == "--benchmarks") { var newNames = args [++optindex].Split (',').Select (s => s.Trim ()); if (benchmarkNames == null) benchmarkNames = newNames.ToArray (); else benchmarkNames = newNames.Union (benchmarkNames).ToArray (); } else if (args [optindex] == "-c" || args [optindex] == "--config-file") { configFile = args [++optindex]; } else if (args [optindex] == "-l" || args [optindex] == "--list-benchmarks") { justListBenchmarks = true; } else if (args [optindex] == "--machine") { machineName = args [++optindex]; } else if (args [optindex] == "--build-url") { buildURL = args [++optindex]; } else if (args [optindex] == "--log-url") { logURL = args [++optindex]; } else if (args [optindex] == "--pull-request-url") { pullRequestURL = args [++optindex]; } else if (args [optindex] == "--mono-repository") { monoRepositoryPath = args [++optindex]; } else if (args [optindex] == "--create-run-set") { justCreateRunSet = true; } else if (args [optindex] == "--run-set-id") { runSetId = Int64.Parse (args [++optindex]); } else if (args [optindex] == "--run-id") { runId = Int64.Parse (args [++optindex]); } else if (args [optindex] == "--root") { rootFromCmdline = args [++optindex]; } else if (args [optindex] == "--main-product") { var name = args [++optindex]; var hash = args [++optindex]; if (mainCommit != null) { Console.Error.WriteLine ("Error: Only one --main-product is supported."); UsageAndExit (); } var product = compare.Utils.LoadProductFromFile (name, productsDir); mainCommit = new Commit { Product = product, Hash = hash }; } else if (args [optindex] == "--secondary-product") { var name = args [++optindex]; var hash = args [++optindex]; var product = compare.Utils.LoadProductFromFile (name, productsDir); secondaryCommits.Add (new Commit { Product = product, Hash = hash }); } else if (args [optindex] == "--valgrind-massif") { if (valgrindBinary != null) { Console.Error.WriteLine ("Error: More than one Valgrind option given."); UsageAndExit (); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Massif; } else if (args [optindex] == "--valgrind-cachegrind") { if (valgrindBinary != null) { Console.Error.WriteLine ("Error: More than one Valgrind option given."); UsageAndExit (); } valgrindBinary = args [++optindex]; valgrindOutputFilename = args [++optindex]; valgrindTool = ValgrindTool.Cachegrind; } else if (args [optindex] == "-t" || args [optindex] == "--timeout") { timeout = Int32.Parse (args [++optindex]); timeout = timeout <= 0 ? -1 : timeout; } else if (args [optindex] == "--sgen-grep-binprot") { grepBinprotPath = args [++optindex]; } else if (args [optindex] == "--upload-pause-times") { binprotFilePath = args [++optindex]; } else if (args [optindex] == "--jit-stats") { jitStats = true; } else if (args [optindex].StartsWith ("--help")) { UsageAndExit (); } else if (args [optindex] == "--") { optindex += 1; break; } else if (args [optindex].StartsWith ("-")) { Console.Error.WriteLine ("unknown parameter {0}", args [optindex]); UsageAndExit (); } else { break; } } var configFileFromCommandLine = configFile != null; if (!configFileFromCommandLine) configFile = Path.Combine (root, "configs", "default-sgen.conf"); var config = compare.Utils.LoadConfigFromFile (configFile, rootFromCmdline, !(justListBenchmarks || binprotFilePath != null)); if (justCreateRunSet && runSetId != null) { Console.Error.WriteLine ("Error: --create-run-set and --run-set-id are incompatible."); Environment.Exit (1); } if (justListBenchmarks && benchmarkNames != null) { Console.Error.WriteLine ("Error: -b/--benchmarks and -l/--list-benchmarks are incompatible."); Environment.Exit (1); } if (justListBenchmarks && !configFileFromCommandLine) { Console.Error.WriteLine ("Error: -l/--list-benchmarks requires --config-file."); Environment.Exit (1); } if (args.Length - optindex != 0) return UsageAndExit (null, 1); if (binprotFilePath != null && (runId == null || grepBinprotPath == null)) { Console.Error.WriteLine ("Error: --upload-pause-times also requires --run-id and --sgen-grep-binprot."); Environment.Exit (1); } if (benchmarkNames == null) benchmarkNames = config.Benchmarks; var benchmarks = compare.Utils.LoadAllBenchmarksFrom (benchmarksDir, benchmarkNames); if (benchmarks == null) { Console.Error.WriteLine ("Error: Could not load all benchmarks."); Environment.Exit (1); } if (justListBenchmarks) { if (machineName != null) { var listMachine = compare.Utils.LoadMachineFromFile (machineName, machinesDir); if (listMachine == null) { Console.Error.WriteLine ("Error: Could not load machine `{0}`.", machineName); Environment.Exit (1); } if (listMachine.ExcludeBenchmarks != null) benchmarks = benchmarks.Where (b => !listMachine.ExcludeBenchmarks.Contains (b.Name)).ToList (); } foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) { Console.Out.WriteLine (benchmark.Name); } Environment.Exit (0); } InitCommons (); if (binprotFilePath != null) { var success = AsyncContext.Run (() => UploadPauseTimes (binprotFilePath, grepBinprotPath, runId.Value)); Environment.Exit (success ? 0 : 1); } if (mainCommit == null) mainCommit = new Commit { Product = compare.Utils.LoadProductFromFile ("mono", productsDir) }; var gitHubClient = GitHubInterface.GitHubClient; Machine machine = null; if (machineName == null) { machine = compare.Utils.LoadMachineCurrentFrom (machinesDir); } else { machine = compare.Utils.LoadMachineFromFile (machineName, machinesDir); } if (machine != null && machine.ExcludeBenchmarks != null) benchmarks = benchmarks.Where (b => !machine.ExcludeBenchmarks.Contains (b.Name)).ToList (); if (machine == null) { // couldn't find machine file var hostarch = compare.Utils.LocalHostnameAndArch (); machine = new Machine (); machine.Name = hostarch.Item1; machine.Architecture = hostarch.Item2; } foreach (var commit in new Commit[] { mainCommit }.Concat (secondaryCommits)) { if (!AsyncContext.Run (() => compare.Utils.CompleteCommit (config, commit))) { Console.Error.WriteLine ("Error: Could not get commit for product {0}.", commit.Product.Name); Environment.Exit (1); } } RunSet runSet; if (runSetId != null) { if (pullRequestURL != null) { Console.Error.WriteLine ("Error: Pull request URL cannot be specified for an existing run set."); Environment.Exit (1); } runSet = AsyncContext.Run (() => RunSet.FromId (machine, runSetId.Value, config, mainCommit, secondaryCommits, buildURL, logURL)); if (runSet == null) { Console.Error.WriteLine ("Error: Could not get run set."); Environment.Exit (1); } } else { long? pullRequestBaselineRunSetId = null; if (pullRequestURL != null) { if (monoRepositoryPath == null) { Console.Error.WriteLine ("Error: Must specify a mono repository path to test a pull request."); Environment.Exit (1); } var repo = new compare.Repository (monoRepositoryPath); var baselineResult = AsyncContext.Run (() => GetPullRequestBaselineRunSetId (mainCommit.Product, pullRequestURL, repo, config)); if (baselineResult == null) { Console.Error.WriteLine ("Error: No appropriate baseline run set found."); Environment.Exit (1); } pullRequestBaselineRunSetId = baselineResult.Item1; mainCommit.MergeBaseHash = baselineResult.Item2; } runSet = new RunSet { StartDateTime = DateTime.Now, Machine = machine, Config = config, Commit = mainCommit, SecondaryCommits = secondaryCommits, BuildURL = buildURL, LogURL = logURL, PullRequestURL = pullRequestURL, PullRequestBaselineRunSetId = pullRequestBaselineRunSetId }; Console.Error.WriteLine ("Set start time to {0}", runSet.StartDateTime); } var reportFailure = false; if (!justCreateRunSet) { var someSuccess = false; var runTool = valgrindBinary; string runToolArguments = null; if (runTool != null) { switch (valgrindTool) { case ValgrindTool.Massif: runToolArguments = string.Format ("--tool=massif --massif-out-file={0} --max-snapshots=1000 --detailed-freq=100 --pages-as-heap=yes", valgrindOutputFilename); break; case ValgrindTool.Cachegrind: runToolArguments = string.Format ("--tool=cachegrind --cachegrind-out-file={0} --cache-sim=yes --branch-sim=yes", valgrindOutputFilename); break; default: Console.Error.WriteLine ("Error: Unsupported Valgrind tool."); Environment.Exit (1); break; } } int binaryProtocolIndex = 0; foreach (var benchmark in benchmarks.OrderBy (b => b.Name)) { // Run the benchmarks if (config.Count <= 0) throw new ArgumentOutOfRangeException (String.Format ("configs [\"{0}\"].Count <= 0", config.Name)); Console.Out.WriteLine ("Running benchmark \"{0}\" with config \"{1}\"", benchmark.Name, config.Name); var runner = new compare.UnixRunner (testsDir, config, benchmark, machine, timeout, runTool, runToolArguments); var haveTimedOut = false; var haveCrashed = false; var count = valgrindBinary == null ? config.Count + 1 : 1; var successCount = 0; for (var i = 0; i < count; ++i) { bool timedOut; string stdoutOutput; if (valgrindBinary == null) Console.Out.Write ("\t\t-> {0} ", i == 0 ? "[dry run]" : String.Format ("({0}/{1})", i, config.Count)); string binaryProtocolFile = null; string workingDirectory = Path.Combine (testsDir, benchmark.TestDirectory); if (config.ProducesBinaryProtocol) { do { ++binaryProtocolIndex; binaryProtocolFile = Path.Combine(workingDirectory, string.Format ("binprot.{0}", binaryProtocolIndex)); } while (File.Exists (binaryProtocolFile)); } var elapsedMilliseconds = runner.Run (binaryProtocolFile, out timedOut, out stdoutOutput); // if running for time, the first one is the dry run if (valgrindBinary == null && i == 0) continue; if (elapsedMilliseconds != null) { var run = new Run { Benchmark = benchmark, BinaryProtocolFilename = binaryProtocolFile == null ? null : Path.Combine(workingDirectory, binaryProtocolFile) }; if (valgrindBinary == null) { run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.Time, Value = TimeSpan.FromMilliseconds (elapsedMilliseconds.Value) }); if (jitStats) { foreach (var phase in ParseJITPhases (stdoutOutput)) { run.RunMetrics.Add (phase); } } } else { switch (valgrindTool) { case ValgrindTool.Massif: { var results = MemoryIntegral (valgrindOutputFilename); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.MemoryIntegral, Value = results.Item1 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.Instructions, Value = results.Item2 }); } break; case ValgrindTool.Cachegrind: { var results = CacheAndBranches (valgrindOutputFilename); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.CachegrindResults, Value = results.Item1 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.CacheMissRate, Value = results.Item2 }); run.RunMetrics.Add (new RunMetric { Metric = RunMetric.MetricType.BranchMispredictionRate, Value = results.Item3 }); } break; } } runSet.Runs.Add (run); successCount++; someSuccess = true; } else { if (timedOut) haveTimedOut = true; else haveCrashed = true; } } if (haveTimedOut) runSet.TimedOutBenchmarks.Add (benchmark.Name); if (haveCrashed) runSet.CrashedBenchmarks.Add (benchmark.Name); if (haveTimedOut || successCount == 0) reportFailure = true; } if (!someSuccess) Console.WriteLine ("all runs failed."); } runSet.FinishDateTime = DateTime.Now; Console.Error.WriteLine ("Start time is {0} - finish time is {1}", runSet.StartDateTime, runSet.FinishDateTime); Console.WriteLine (JsonConvert.SerializeObject (runSet.AsDict ())); var uploadResult = AsyncContext.Run (() => Utils.RunWithRetry (() => runSet.Upload ())); if (uploadResult == null) { Console.Error.WriteLine ("Error: Could not upload run set."); Environment.Exit (1); } Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/runset.html#id={0}", uploadResult.RunSetId); if (pullRequestURL != null) Console.WriteLine ("http://xamarin.github.io/benchmarker/front-end/pullrequest.html#id={0}", uploadResult.PullRequestId.Value); Console.Write ("{{ \"runSetId\": \"{0}\"", uploadResult.RunSetId); if (pullRequestURL != null) Console.Write (", \"pullRequestId\": \"{0}\"", uploadResult.PullRequestId.Value); Console.Write (", \"runs\": [ "); var runStrings = new List<string> (); var allRuns = runSet.Runs.ToList (); for (var i = 0; i < allRuns.Count; i++) { var run = allRuns [i]; var id = uploadResult.RunIds [i]; var str = string.Format ("\"id\": {0}", id); if (run.BinaryProtocolFilename != null) str = string.Format ("{0}, \"binaryProtocolFile\": \"{1}\"", str, run.BinaryProtocolFilename); runStrings.Add ("{ " + str + " }"); } Console.Write (string.Join (", ", runStrings)); Console.Write (" ]"); Console.WriteLine (" }"); if (reportFailure) { Console.Error.WriteLine ("Error: Some benchmarks timed out or failed completely."); return 1; } return 0; }
public static async Task <RunSet> FromId(Machine local_machine, long local_runsetid, Config local_config, Commit local_mainCommit, List <Commit> local_secondaryCommits, string local_buildURL, string local_logURL) { using (var client = new HttpClient()) { JObject db_result = await HttpApi.GetRunset(local_runsetid); if (db_result == null) { return(null); } Console.Error.WriteLine("trying to parse Start date: \"" + db_result ["StartedAt"].ToObject <DateTime> ().ToLocalTime().ToString(DATETIME_PRETTY) + "\""); var runSet = new RunSet { Id = local_runsetid, StartDateTime = db_result ["StartedAt"].ToObject <DateTime> ().ToLocalTime(), FinishDateTime = db_result ["FinishedAt"].ToObject <DateTime> ().ToLocalTime(), BuildURL = db_result ["BuildURL"].ToObject <string> (), Machine = local_machine, LogURL = local_logURL, Config = local_config, Commit = local_mainCommit, TimedOutBenchmarks = db_result ["TimedOutBenchmarks"].ToObject <List <string> > (), CrashedBenchmarks = db_result ["CrashedBenchmarks"].ToObject <List <string> > () }; var db_mainProductCommit = db_result ["MainProduct"] ["Commit"].ToObject <string> (); if (local_mainCommit.Hash != db_mainProductCommit) { throw new Exception(String.Format("Commit ({0}) does not match the one in the database ({1}).", local_mainCommit.Hash, db_mainProductCommit)); } var db_secondaryCommits = new List <Commit> (); foreach (var sc in db_result["SecondaryProducts"]) { db_secondaryCommits.Add(new Commit { Hash = sc ["Commit"].ToObject <string> (), Product = new Product { Name = sc ["Name"].ToObject <string> () } }); } if (local_secondaryCommits != null) { if (local_secondaryCommits.Count != db_secondaryCommits.Count) { throw new Exception("Secondary commits don't match the database."); } foreach (var sc in db_secondaryCommits) { if (!local_secondaryCommits.Any(c => c.Hash == sc.Hash && c.Product.Name == sc.Product.Name)) { throw new Exception("Secondary commits don't match the database."); } } // local commits have more information (e.g. datetime) runSet.SecondaryCommits = local_secondaryCommits; } else { runSet.SecondaryCommits = db_secondaryCommits; } if (local_buildURL != null && local_buildURL != runSet.BuildURL) { throw new Exception("Build URL does not match the one in the database."); } var db_machineName = db_result ["Machine"] ["Name"].ToObject <string> (); var db_machineArchitecture = db_result ["Machine"] ["Architecture"].ToObject <string> (); if (local_machine.Name != db_machineName || local_machine.Architecture != db_machineArchitecture) { throw new Exception("Machine does not match the one in the database. \"" + db_machineName + "\" vs. \"" + local_machine.Name + "\""); } if (!local_config.EqualsApiObject(db_result ["Config"])) { throw new Exception("Config does not match the one in the database."); } return(runSet); } }