public async Task FindExperiments() { ExperimentDefinition def1 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare"); ExperimentDefinition def2 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 2", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare"); ExperimentManager manager = NewManager(); var e1 = await manager.StartExperiment(def1); var e2 = await manager.StartExperiment(def1); var e4 = await manager.StartExperiment(def2); var r1 = await Task.WhenAll(manager.GetResults(e1)); var r2 = await Task.WhenAll(manager.GetResults(e2)); var r4 = await Task.WhenAll(manager.GetResults(e4)); var loaded = (await manager.FindExperiments()).ToArray(); Assert.AreEqual(3, loaded.Length, "Number of found experiments (same manager)"); var manager2 = OpenManager(manager); var loaded2 = (await manager2.FindExperiments()).ToArray(); Assert.AreEqual(3, loaded2.Length, "Number of found experiments (reloaded)"); var loaded3 = (await manager2.FindExperiments(new ExperimentManager.ExperimentFilter { ParametersEquals = "{0} 2" })).ToArray(); Assert.AreEqual(1, loaded3.Length, "Number of found experiments (reloaded, filtered)"); }
private async void DownloadResultsAsync() { if (IsFiltering) { return; } IsFiltering = true; var handle = uiService.StartIndicateLongOperation("Comparing the experiments..."); try { allResults = CompareItems = null; var t1 = Task.Run(() => manager.GetResults(id1)); var t2 = Task.Run(() => manager.GetResults(id2)); allResults1 = (await t1).Benchmarks; allResults2 = (await t2).Benchmarks; JoinResults(); IsFiltering = false; } catch (Exception ex) { uiService.ShowError(ex, "Failed to get results for the experiments"); } finally { IsFiltering = false; uiService.StopIndicateLongOperation(handle); } }
private async void RefreshResultsAsync() { var handle = uiService.StartIndicateLongOperation("Loading experiment results..."); try { experimentResults = null; allResults = Results = null; var res = await Task.Run(() => manager.GetResults(id)); experimentResults = res; allResults = Results = res.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray(); RefreshJobStatus(); } catch (Exception ex) { experimentResults = null; allResults = Results = null; uiService.ShowError(ex.Message, "Failed to load experiment results"); } finally { uiService.StopIndicateLongOperation(handle); } }
static async Task Run(ExperimentManager manager, ExperimentDefinition definition) { int id = await manager.StartExperiment(definition); var results = manager.GetResults(id); var filter = new ExperimentManager.ExperimentFilter { BenchmarkContainerEquals = definition.BenchmarkDirectory, CategoryEquals = definition.Category, ExecutableEquals = definition.Executable, ParametersEquals = definition.Parameters }; var history = (await manager.FindExperiments(filter)).Where(q => q.ID != id).ToArray(); Dictionary <string, BenchmarkResult> lastBenchmarks = new Dictionary <string, BenchmarkResult>(); if (history.Length != 0) { var lastResults = await manager.GetResults(history.Max(e => e.ID)); foreach (var b in lastResults.Benchmarks) { lastBenchmarks[b.BenchmarkFileName] = b; } } var print = results.ContinueWith(task => { if (task.IsFaulted) { PrintError(String.Format("Failed to complete benchmarks {0}", task.Exception.Message)); return; } BenchmarkResult[] benchmarks = task.Result.Benchmarks; foreach (var benchmark in benchmarks) { BenchmarkResult lastBenchmark = null; lastBenchmarks.TryGetValue(benchmark.BenchmarkFileName, out lastBenchmark); if (lastBenchmark != null && lastBenchmark.Status != Measurement.ResultStatus.Success) { lastBenchmark = null; } PrintBenchmark(benchmark, lastBenchmark); } }); await print; }
private static async Task <ExperimentStatistics> GetStatistics(ExperimentManager manager, int id, Measurement.Domain domain) { var results = await manager.GetResults(id); var aggr = domain.Aggregate(results.Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds))); return(new ExperimentStatistics(aggr)); }
public async Task RunExperiment() { ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_1", "csv", "{0}", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default"); ExperimentManager manager = NewManager(); var expId = await manager.StartExperiment(def); var results = (await manager.GetResults(expId)).Benchmarks; Assert.AreEqual(1, results.Length, "Number of completed benchmarks"); var res = results[0]; Assert.AreEqual(0, res.ExitCode, "exit code"); Assert.AreEqual(ResultStatus.Success, res.Status, "status"); Assert.IsTrue(res.CPUTime.TotalSeconds < 1, "Total runtime"); }
public async Task RunExperimentsWithCategory() { ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1000", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare"); ExperimentManager manager = NewManager(); var expId = await manager.StartExperiment(def); var results = (await manager.GetResults(expId)).Benchmarks; Assert.AreEqual(3, results.Length, "Number of completed benchmarks"); foreach (var res in results) { Assert.AreEqual(0, res.ExitCode, "exit code"); Assert.AreEqual(ResultStatus.Success, res.Status, "status"); Assert.IsTrue(res.TotalProcessorTime.TotalSeconds < 10, "Total runtime"); } }
private static async Task <ExperimentResults[]> DownloadResultsAsync(ExperimentStatusViewModel[] experiments, ExperimentManager manager) { var count = experiments.Length; var t = new Task <ExperimentResults> [count]; for (int i = 0; i < count; i++) { int index = i; t[index] = Task.Run(() => manager.GetResults(experiments[index].ID)); } var b = new ExperimentResults[count]; for (int j = 0; j < count; j++) { int index = j; b[index] = await t[index]; } return(b); }
public static async void SaveOutput(string selectedPath, ExperimentStatusViewModel experiment, ExperimentManager manager, IUIService uiService) { if (experiment == null) { throw new ArgumentNullException("experiment"); } if (manager == null) { throw new ArgumentNullException("manager"); } if (uiService == null) { throw new ArgumentNullException("uiService"); } var handle = uiService.StartIndicateLongOperation("Saving output..."); try { string drctry = string.Format(@"{0}\{1}", selectedPath, experiment.ID.ToString()); await Task.Run(async() => { double total = 0.0; Directory.CreateDirectory(drctry); var benchs = await manager.GetResults(experiment.ID); var benchsVm = benchs.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray(); total = benchsVm.Length; for (int i = 0; i < total; i++) { UTF8Encoding enc = new UTF8Encoding(); string stdout = await benchsVm[i].GetStdOutAsync(false); string stderr = await benchsVm[i].GetStdErrAsync(false); string path = drctry + @"\" + experiment.Category + @"\" + benchsVm[i].Filename; path = path.Replace("/", @"\"); Directory.CreateDirectory(path.Substring(0, path.LastIndexOf(@"\"))); if (stdout != null && stdout.Length > 0) { FileStream stdoutf = File.Open(path + ".out.txt", FileMode.OpenOrCreate); stdoutf.Write(enc.GetBytes(stdout), 0, enc.GetByteCount(stdout)); stdoutf.Close(); } if (stderr != null && stderr.Length > 0) { FileStream stderrf = File.Open(path + ".err.txt", FileMode.OpenOrCreate); stderrf.Write(enc.GetBytes(stderr), 0, enc.GetByteCount(stderr)); stderrf.Close(); } } }); uiService.ShowInfo("Output saved to " + drctry); } catch (Exception ex) { uiService.ShowError(ex, "Failed to save output"); } finally { uiService.StopIndicateLongOperation(handle); } }