private Task <BenchmarkResult>[] RunExperiment(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0)
{
if (experiment == null)
{
throw new ArgumentNullException("experiment");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
Domain domain = domainResolver.GetDomain(experiment.DomainName);
string executable;
if (Path.IsPathRooted(experiment.Executable))
{
executable = experiment.Executable;
}
else
{
executable = Path.Combine(rootFolder, experiment.Executable);
}
if (!File.Exists(executable))
{
throw new ArgumentException("Executable not found");
}
string benchmarkFolder = string.IsNullOrEmpty(experiment.Category) ? experiment.BenchmarkDirectory : Path.Combine(experiment.BenchmarkDirectory, experiment.Category);
if (!Path.IsPathRooted(benchmarkFolder))
{
benchmarkFolder = Path.Combine(rootFolder, benchmarkFolder);
}
var benchmarks = Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories);
var results = new List <Task <BenchmarkResult> >(256);
foreach (string benchmarkFile in Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories))
{
var task =
factory.StartNew(_benchmark =>
{
string inputFullPath = (string)_benchmark;
string inputRelativePath = Utils.MakeRelativePath(benchmarkFolder, inputFullPath);
Trace.WriteLine("Running benchmark " + Path.GetFileName(inputRelativePath));
string args = experiment.Parameters;
return(RunBenchmark(id, executable, experiment.Parameters, inputRelativePath, inputFullPath,
repetitions, experiment.BenchmarkTimeout, experiment.MemoryLimitMB, null, null, domain,
normal));
}, benchmarkFile, TaskCreationOptions.LongRunning);
results.Add(task);
}
return(results.ToArray());
}
public async Task <ExperimentSummary[]> Update(string timelineName, params int[] experiments)
{
if (experiments == null)
{
throw new ArgumentNullException(nameof(experiments));
}
Trace.WriteLine("Downloading experiment results...");
var all_summaries = await DownloadSummary(timelineName);
Table timeline = all_summaries.Item1;
RecordsTable records = all_summaries.Item2;
string etag = all_summaries.Item3;
foreach (var experimentId in experiments)
{
var exp = await storage.GetExperiment(experimentId); // fails if not found
var domain = resolveDomain.GetDomain(exp.DomainName);
var results = (await storage.GetResults(experimentId)).Benchmarks;
Trace.WriteLine("Building summary for the experiment " + experimentId);
var catSummary = ExperimentSummary.Build(results, domain, ExperimentSummary.DuplicateResolution.Ignore);
var expSummary = new ExperimentSummary(experimentId, DateTimeOffset.Now, catSummary);
timeline = ExperimentSummaryStorage.AppendOrReplace(timeline, expSummary);
Trace.WriteLine("Updating records...");
records.UpdateWith(results, domain);
}
await UploadSummary(timelineName, timeline, records, all_summaries.Item3);
var resultfromTable = ExperimentSummaryStorage.LoadFromTable(timeline);
Array.Sort(resultfromTable, (el1, el2) => DateTimeOffset.Compare(el2.Date, el1.Date));
return(resultfromTable);
}
public static async Task <ExperimentPropertiesViewModel> CreateAsync(ExperimentManager manager, int id, IDomainResolver domainResolver, IUIService ui)
{
if (manager == null)
{
throw new ArgumentNullException("manager");
}
Experiment exp = await manager.TryFindExperiment(id);
if (exp == null)
{
throw new KeyNotFoundException(string.Format("There is no experiment with id {0}.", id));
}
return(new ExperimentPropertiesViewModel(exp.Definition, exp.Status, domainResolver.GetDomain(exp.Definition.DomainName ?? "Z3"), manager, ui));
}
public static async void SaveMetaCSV(string filename, ExperimentStatusViewModel[] experiments, ExperimentManager manager, IDomainResolver domainResolver, IUIService uiService)
{
if (filename == null)
{
throw new ArgumentNullException("filename");
}
if (experiments == null)
{
throw new ArgumentNullException("experiments");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (domainResolver == null)
{
throw new ArgumentNullException("domain");
}
if (uiService == null)
{
throw new ArgumentNullException("uiService");
}
var handle = uiService.StartIndicateLongOperation("Save meta csv...");
try
{
StreamWriter f = new StreamWriter(filename, false);
f.WriteLine("\"ID\",\"# Total\",\"# SAT\",\"# UNSAT\",\"# UNKNOWN\",\"# Timeout\",\"# Memout\",\"# Bug\",\"# Error\",\"# Unique\",\"Parameters\",\"Note\"");
var count = experiments.Length;
var b = new ExperimentResults[count];
b = await DownloadResultsAsync(experiments, manager);
var unique = computeUnique(experiments, b);
for (var i = 0; i < count; i++)
{
var domain = domainResolver.GetDomain(experiments[i].Definition.DomainName ?? "Z3");
var aggr = domain.Aggregate(b[i].Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds)));
var statistics = new ExperimentStatistics(aggr);
var def = experiments[i].Definition;
string ps = def.Parameters.Trim(' ');
string note = experiments[i].Note.Trim(' ');
int? sat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeySat], CultureInfo.InvariantCulture);
int? unsat = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnsat], CultureInfo.InvariantCulture);
int? unknown = statistics == null ? null : (int?)int.Parse(statistics.AggregatedResults.Properties[Z3Domain.KeyUnknown], CultureInfo.InvariantCulture);
int? bugs = statistics == null ? null : (int?)statistics.AggregatedResults.Bugs;
int? errors = statistics == null ? null : (int?)statistics.AggregatedResults.Errors;
int? timeouts = statistics == null ? null : (int?)statistics.AggregatedResults.Timeouts;
int? memouts = statistics == null ? null : (int?)statistics.AggregatedResults.MemoryOuts;
f.WriteLine(experiments[i].ID + "," +
experiments[i].BenchmarksTotal + "," +
sat + "," +
unsat + "," +
unknown + "," +
timeouts + "," +
memouts + "," +
bugs + "," +
errors + "," +
unique[i] + "," +
"\"" + ps + "\"," +
"\"" + note + "\"");
}
f.WriteLine();
f.Close();
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to save meta CSV");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
private async void SubmitNewJob(NewExperimentViewModel vm)
{
NewJobDialog dlg = new NewJobDialog();
dlg.DataContext = vm;
dlg.Owner = this;
if (dlg.ShowDialog() == true)
{
var handle = uiService.StartIndicateLongOperation("Submitting new experiment...");
try
{
vm.SaveRecentSettings();
}
catch (Exception ex)
{
uiService.ShowWarning(ex.Message, "Failed to save recent settings");
}
Tuple <string, int?, Exception>[] result;
try
{
if (!vm.Parameters.Contains("{0}"))
{
var domain = domainResolver.GetDomain(vm.Domain);
vm.Parameters = domain.AddFileNameArgument(vm.Parameters, "{0}");
}
// Submitting the new experiment
result = await managerVm.SubmitExperiments(vm);
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to submit an experiment");
return;
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
experimentsVm.Refresh();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < result.Length; i++)
{
if (i > 0)
{
sb.AppendLine();
}
var r = result[i];
if (r.Item2.HasValue)
{
sb.AppendFormat("Experiment for category {0} successfully submitted with id {1}.", r.Item1, r.Item2.Value);
}
if (r.Item3 != null)
{
sb.AppendFormat("Experiment for category {0} could not be submitted: {1}.", r.Item1, r.Item3.Message);
}
}
uiService.ShowInfo(sb.ToString(), "New experiments");
}
}
