public async Task FindExperiments()
{
ExperimentDefinition def1 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare");
ExperimentDefinition def2 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 2", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare");
ExperimentManager manager = NewManager();
var e1 = await manager.StartExperiment(def1);
var e2 = await manager.StartExperiment(def1);
var e4 = await manager.StartExperiment(def2);
var r1 = await Task.WhenAll(manager.GetResults(e1));
var r2 = await Task.WhenAll(manager.GetResults(e2));
var r4 = await Task.WhenAll(manager.GetResults(e4));
var loaded = (await manager.FindExperiments()).ToArray();
Assert.AreEqual(3, loaded.Length, "Number of found experiments (same manager)");
var manager2 = OpenManager(manager);
var loaded2 = (await manager2.FindExperiments()).ToArray();
Assert.AreEqual(3, loaded2.Length, "Number of found experiments (reloaded)");
var loaded3 = (await manager2.FindExperiments(new ExperimentManager.ExperimentFilter {
ParametersEquals = "{0} 2"
})).ToArray();
Assert.AreEqual(1, loaded3.Length, "Number of found experiments (reloaded, filtered)");
}
private async void DownloadResultsAsync()
{
if (IsFiltering)
{
return;
}
IsFiltering = true;
var handle = uiService.StartIndicateLongOperation("Comparing the experiments...");
try
{
allResults = CompareItems = null;
var t1 = Task.Run(() => manager.GetResults(id1));
var t2 = Task.Run(() => manager.GetResults(id2));
allResults1 = (await t1).Benchmarks;
allResults2 = (await t2).Benchmarks;
JoinResults();
IsFiltering = false;
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to get results for the experiments");
}
finally
{
IsFiltering = false;
uiService.StopIndicateLongOperation(handle);
}
}
private async void RefreshResultsAsync()
{
var handle = uiService.StartIndicateLongOperation("Loading experiment results...");
try
{
experimentResults = null;
allResults = Results = null;
var res = await Task.Run(() => manager.GetResults(id));
experimentResults = res;
allResults = Results = res.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray();
RefreshJobStatus();
}
catch (Exception ex)
{
experimentResults = null;
allResults = Results = null;
uiService.ShowError(ex.Message, "Failed to load experiment results");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
static async Task Run(ExperimentManager manager, ExperimentDefinition definition)
{
int id = await manager.StartExperiment(definition);
var results = manager.GetResults(id);
var filter = new ExperimentManager.ExperimentFilter
{
BenchmarkContainerEquals = definition.BenchmarkDirectory,
CategoryEquals = definition.Category,
ExecutableEquals = definition.Executable,
ParametersEquals = definition.Parameters
};
var history = (await manager.FindExperiments(filter)).Where(q => q.ID != id).ToArray();
Dictionary <string, BenchmarkResult> lastBenchmarks = new Dictionary <string, BenchmarkResult>();
if (history.Length != 0)
{
var lastResults = await manager.GetResults(history.Max(e => e.ID));
foreach (var b in lastResults.Benchmarks)
{
lastBenchmarks[b.BenchmarkFileName] = b;
}
}
var print = results.ContinueWith(task =>
{
if (task.IsFaulted)
{
PrintError(String.Format("Failed to complete benchmarks {0}", task.Exception.Message));
return;
}
BenchmarkResult[] benchmarks = task.Result.Benchmarks;
foreach (var benchmark in benchmarks)
{
BenchmarkResult lastBenchmark = null;
lastBenchmarks.TryGetValue(benchmark.BenchmarkFileName, out lastBenchmark);
if (lastBenchmark != null && lastBenchmark.Status != Measurement.ResultStatus.Success)
{
lastBenchmark = null;
}
PrintBenchmark(benchmark, lastBenchmark);
}
});
await print;
}
private static async Task <ExperimentStatistics> GetStatistics(ExperimentManager manager, int id, Measurement.Domain domain)
{
var results = await manager.GetResults(id);
var aggr = domain.Aggregate(results.Benchmarks.Select(r => new ProcessRunResults(new ProcessRunAnalysis(r.Status, r.Properties), r.CPUTime.TotalSeconds)));
return(new ExperimentStatistics(aggr));
}
public async Task RunExperiment()
{
ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_1", "csv", "{0}", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default");
ExperimentManager manager = NewManager();
var expId = await manager.StartExperiment(def);
var results = (await manager.GetResults(expId)).Benchmarks;
Assert.AreEqual(1, results.Length, "Number of completed benchmarks");
var res = results[0];
Assert.AreEqual(0, res.ExitCode, "exit code");
Assert.AreEqual(ResultStatus.Success, res.Status, "status");
Assert.IsTrue(res.CPUTime.TotalSeconds < 1, "Total runtime");
}
public async Task RunExperimentsWithCategory()
{
ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1000", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default",
category: "IdentitySquare");
ExperimentManager manager = NewManager();
var expId = await manager.StartExperiment(def);
var results = (await manager.GetResults(expId)).Benchmarks;
Assert.AreEqual(3, results.Length, "Number of completed benchmarks");
foreach (var res in results)
{
Assert.AreEqual(0, res.ExitCode, "exit code");
Assert.AreEqual(ResultStatus.Success, res.Status, "status");
Assert.IsTrue(res.TotalProcessorTime.TotalSeconds < 10, "Total runtime");
}
}
private static async Task <ExperimentResults[]> DownloadResultsAsync(ExperimentStatusViewModel[] experiments, ExperimentManager manager)
{
var count = experiments.Length;
var t = new Task <ExperimentResults> [count];
for (int i = 0; i < count; i++)
{
int index = i;
t[index] = Task.Run(() => manager.GetResults(experiments[index].ID));
}
var b = new ExperimentResults[count];
for (int j = 0; j < count; j++)
{
int index = j;
b[index] = await t[index];
}
return(b);
}
public static async void SaveOutput(string selectedPath, ExperimentStatusViewModel experiment, ExperimentManager manager, IUIService uiService)
{
if (experiment == null)
{
throw new ArgumentNullException("experiment");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (uiService == null)
{
throw new ArgumentNullException("uiService");
}
var handle = uiService.StartIndicateLongOperation("Saving output...");
try
{
string drctry = string.Format(@"{0}\{1}", selectedPath, experiment.ID.ToString());
await Task.Run(async() =>
{
double total = 0.0;
Directory.CreateDirectory(drctry);
var benchs = await manager.GetResults(experiment.ID);
var benchsVm = benchs.Benchmarks.Select(e => new BenchmarkResultViewModel(e, uiService)).ToArray();
total = benchsVm.Length;
for (int i = 0; i < total; i++)
{
UTF8Encoding enc = new UTF8Encoding();
string stdout = await benchsVm[i].GetStdOutAsync(false);
string stderr = await benchsVm[i].GetStdErrAsync(false);
string path = drctry + @"\" + experiment.Category + @"\" + benchsVm[i].Filename;
path = path.Replace("/", @"\");
Directory.CreateDirectory(path.Substring(0, path.LastIndexOf(@"\")));
if (stdout != null && stdout.Length > 0)
{
FileStream stdoutf = File.Open(path + ".out.txt", FileMode.OpenOrCreate);
stdoutf.Write(enc.GetBytes(stdout), 0, enc.GetByteCount(stdout));
stdoutf.Close();
}
if (stderr != null && stderr.Length > 0)
{
FileStream stderrf = File.Open(path + ".err.txt", FileMode.OpenOrCreate);
stderrf.Write(enc.GetBytes(stderr), 0, enc.GetByteCount(stderr));
stderrf.Close();
}
}
});
uiService.ShowInfo("Output saved to " + drctry);
}
catch (Exception ex)
{
uiService.ShowError(ex, "Failed to save output");
}
finally
{
uiService.StopIndicateLongOperation(handle);
}
}
