private Task <BenchmarkResult>[] RunExperiment(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0) { if (experiment == null) { throw new ArgumentNullException("experiment"); } if (factory == null) { throw new ArgumentNullException("factory"); } Domain domain = domainResolver.GetDomain(experiment.DomainName); string executable; if (Path.IsPathRooted(experiment.Executable)) { executable = experiment.Executable; } else { executable = Path.Combine(rootFolder, experiment.Executable); } if (!File.Exists(executable)) { throw new ArgumentException("Executable not found"); } string benchmarkFolder = string.IsNullOrEmpty(experiment.Category) ? experiment.BenchmarkDirectory : Path.Combine(experiment.BenchmarkDirectory, experiment.Category); if (!Path.IsPathRooted(benchmarkFolder)) { benchmarkFolder = Path.Combine(rootFolder, benchmarkFolder); } var benchmarks = Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories); var results = new List <Task <BenchmarkResult> >(256); foreach (string benchmarkFile in Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories)) { var task = factory.StartNew(_benchmark => { string inputFullPath = (string)_benchmark; string inputRelativePath = Utils.MakeRelativePath(benchmarkFolder, inputFullPath); Trace.WriteLine("Running benchmark " + Path.GetFileName(inputRelativePath)); string args = experiment.Parameters; return(RunBenchmark(id, executable, experiment.Parameters, inputRelativePath, inputFullPath, repetitions, experiment.BenchmarkTimeout, experiment.MemoryLimitMB, null, null, domain, normal)); }, benchmarkFile, TaskCreationOptions.LongRunning); results.Add(task); } return(results.ToArray()); }
public async Task <ExperimentSummary[]> Update(string timelineName, params int[] experiments) { if (experiments == null) { throw new ArgumentNullException(nameof(experiments)); } Trace.WriteLine("Downloading experiment results..."); var all_summaries = await DownloadSummary(timelineName); Table timeline = all_summaries.Item1; RecordsTable records = all_summaries.Item2; string etag = all_summaries.Item3; foreach (var experimentId in experiments) { var exp = await storage.GetExperiment(experimentId); // fails if not found var domain = resolveDomain.GetDomain(exp.DomainName); var results = (await storage.GetResults(experimentId)).Benchmarks; Trace.WriteLine("Building summary for the experiment " + experimentId); var catSummary = ExperimentSummary.Build(results, domain, ExperimentSummary.DuplicateResolution.Ignore); var expSummary = new ExperimentSummary(experimentId, DateTimeOffset.Now, catSummary); timeline = ExperimentSummaryStorage.AppendOrReplace(timeline, expSummary); Trace.WriteLine("Updating records..."); records.UpdateWith(results, domain); } await UploadSummary(timelineName, timeline, records, all_summaries.Item3); var resultfromTable = ExperimentSummaryStorage.LoadFromTable(timeline); Array.Sort(resultfromTable, (el1, el2) => DateTimeOffset.Compare(el2.Date, el1.Date)); return(resultfromTable); }
public static async Task <ExperimentPropertiesViewModel> CreateAsync(ExperimentManager manager, int id, IDomainResolver domainResolver, IUIService ui) { if (manager == null) { throw new ArgumentNullException("manager"); } Experiment exp = await manager.TryFindExperiment(id); if (exp == null) { throw new KeyNotFoundException(string.Format("There is no experiment with id {0}.", id)); } return(new ExperimentPropertiesViewModel(exp.Definition, exp.Status, domainResolver.GetDomain(exp.Definition.DomainName ?? "Z3"), manager, ui)); }
public static async void SaveMetaCSV(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IDomainResolver domainResolver, IUIService uiService) { if (filename == null) { throw new ArgumentNullException("filename"); } if (experiments == null) { throw new ArgumentNullException("experiments"); } if (manager == null) { throw new ArgumentNullException("manager"); } if (domainResolver == null) { throw new ArgumentNullException("domain"); } if (uiService == null) { throw new ArgumentNullException("uiService"); } var handle = uiService.StartIndicateLongOperation("Save meta csv..."); try { StreamWriter f = new StreamWriter(filename, false); f.WriteLine("\"ID\",\"# Total\",\"# SAT\",\"# UNSAT\",\"# UNKNOWN\",\"# Timeout\",\"# Memout\",\"# Bug\",\"# Error\",\"# Unique\",\"Parameters\",\"Note\""); var count = experiments.Length; var b = new ExperimentResults[count]; b = await DownloadResultsAsync(experiments, manager); var unique = computeUnique(experiments, b); for (var i = 0; i < count; i++) { var domain = domainResolver.GetDomain(experiments[i].Definition.DomainName ?? "Z3"); var aggr = domain.Aggregate(b[i].Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds))); var statistics = new ExperimentStatistics(aggr); var def = experiments[i].Definition; string ps = def.Parameters.Trim(' '); string note = experiments[i].Note.Trim(' '); int? sat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeySat], CultureInfo.InvariantCulture); int? unsat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnsat], CultureInfo.InvariantCulture); int? unknown = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnknown], CultureInfo.InvariantCulture); int? bugs = statistics == null ? null : (int?)statistics.AggregatedResults.Bugs; int? errors = statistics == null ? null : (int?)statistics.AggregatedResults.Errors; int? timeouts = statistics == null ? null : (int?)statistics.AggregatedResults.Timeouts; int? memouts = statistics == null ? null : (int?)statistics.AggregatedResults.MemoryOuts; f.WriteLine(experiments[i].ID + "," + experiments[i].BenchmarksTotal + "," + sat + "," + unsat + "," + unknown + "," + timeouts + "," + memouts + "," + bugs + "," + errors + "," + unique[i] + "," + "\"" + ps + "\"," + "\"" + note + "\""); } f.WriteLine(); f.Close(); } catch (Exception ex) { uiService.ShowError(ex, "Failed to save meta CSV"); } finally { uiService.StopIndicateLongOperation(handle); } }
private async void SubmitNewJob(NewExperimentViewModel vm) { NewJobDialog dlg = new NewJobDialog(); dlg.DataContext = vm; dlg.Owner = this; if (dlg.ShowDialog() == true) { var handle = uiService.StartIndicateLongOperation("Submitting new experiment..."); try { vm.SaveRecentSettings(); } catch (Exception ex) { uiService.ShowWarning(ex.Message, "Failed to save recent settings"); } Tuple <string, int?, Exception>[] result; try { if (!vm.Parameters.Contains("{0}")) { var domain = domainResolver.GetDomain(vm.Domain); vm.Parameters = domain.AddFileNameArgument(vm.Parameters, "{0}"); } // Submitting the new experiment result = await managerVm.SubmitExperiments(vm); } catch (Exception ex) { uiService.ShowError(ex, "Failed to submit an experiment"); return; } finally { uiService.StopIndicateLongOperation(handle); } experimentsVm.Refresh(); StringBuilder sb = new StringBuilder(); for (int i = 0; i < result.Length; i++) { if (i > 0) { sb.AppendLine(); } var r = result[i]; if (r.Item2.HasValue) { sb.AppendFormat("Experiment for category {0} successfully submitted with id {1}.", r.Item1, r.Item2.Value); } if (r.Item3 != null) { sb.AppendFormat("Experiment for category {0} could not be submitted: {1}.", r.Item1, r.Item3.Message); } } uiService.ShowInfo(sb.ToString(), "New experiments"); } }