public Task <BenchmarkResult>[] Enqueue(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0)
{
if (experiment == null)
{
throw new ArgumentNullException("experiment");
}
return(RunExperiment(id, experiment, normal, repetitions));
}
private static ExperimentDefinition MakeRelativeDefinition(string experimentsFolder, ExperimentDefinition def)
{
string relExec = Utils.MakeRelativePath(experimentsFolder, def.Executable);
string relDirectory = Utils.MakeRelativePath(experimentsFolder, def.BenchmarkDirectory);
return(ExperimentDefinition.Create(relExec, def.BenchmarkContainerUri, relDirectory, def.BenchmarkFileExtension,
def.Parameters, def.BenchmarkTimeout, def.ExperimentTimeout, def.DomainName,
def.Category, def.MemoryLimitMB));
}
private static ExperimentDefinition RowToDefinition(ExperimentEntity row)
{
return(ExperimentDefinition.Create(
row.Executable, row.BenchmarkContainerUri, row.BenchmarkDirectory,
row.BenchmarkFileExtension, row.Parameters,
TimeSpan.FromSeconds(row.BenchmarkTimeout),
TimeSpan.FromSeconds(row.ExperimentTimeout),
row.DomainName, row.Category, row.MemoryLimitMB));
}
private Task <BenchmarkResult>[] RunExperiment(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0)
{
if (experiment == null)
{
throw new ArgumentNullException("experiment");
}
if (factory == null)
{
throw new ArgumentNullException("factory");
}
Domain domain = domainResolver.GetDomain(experiment.DomainName);
string executable;
if (Path.IsPathRooted(experiment.Executable))
{
executable = experiment.Executable;
}
else
{
executable = Path.Combine(rootFolder, experiment.Executable);
}
if (!File.Exists(executable))
{
throw new ArgumentException("Executable not found");
}
string benchmarkFolder = string.IsNullOrEmpty(experiment.Category) ? experiment.BenchmarkDirectory : Path.Combine(experiment.BenchmarkDirectory, experiment.Category);
if (!Path.IsPathRooted(benchmarkFolder))
{
benchmarkFolder = Path.Combine(rootFolder, benchmarkFolder);
}
var benchmarks = Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories);
var results = new List <Task <BenchmarkResult> >(256);
foreach (string benchmarkFile in Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories))
{
var task =
factory.StartNew(_benchmark =>
{
string inputFullPath = (string)_benchmark;
string inputRelativePath = Utils.MakeRelativePath(benchmarkFolder, inputFullPath);
Trace.WriteLine("Running benchmark " + Path.GetFileName(inputRelativePath));
string args = experiment.Parameters;
return(RunBenchmark(id, executable, experiment.Parameters, inputRelativePath, inputFullPath,
repetitions, experiment.BenchmarkTimeout, experiment.MemoryLimitMB, null, null, domain,
normal));
}, benchmarkFile, TaskCreationOptions.LongRunning);
results.Add(task);
}
return(results.ToArray());
}
public ExperimentInstance(ExperimentID id, ExperimentDefinition def, Task <BenchmarkResult>[] results)
{
if (def == null)
{
throw new ArgumentNullException("def");
}
this.id = id;
this.def = def;
this.results = results;
}
public static LocalExperimentManager NewExperiments(string experimentsFolder, ReferenceExperiment reference, IDomainResolver domainResolver)
{
ExperimentDefinition def = MakeRelativeDefinition(experimentsFolder, reference.Definition);
ReferenceExperiment relRef = new ReferenceExperiment(def, reference.Repetitions, reference.ReferenceValue);
FileStorage storage = FileStorage.Open(experimentsFolder);
storage.Clear();
storage.SaveReferenceExperiment(relRef);
LocalExperimentManager manager = new LocalExperimentManager(storage, domainResolver);
return(manager);
}
public ReferenceExperiment(ExperimentDefinition def, int repetitions, double referenceValue)
{
if (def == null)
{
throw new ArgumentNullException("def");
}
if (repetitions < 1)
{
throw new ArgumentOutOfRangeException("repetitions", "Number of repetitions must be greater than zero");
}
Definition = def;
Repetitions = repetitions;
ReferenceValue = referenceValue;
}
public override Task <Experiment> TryFindExperiment(int id)
{
ExperimentEntity entity;
if (!storage.GetExperiments().TryGetValue(id, out entity))
{
return(null);
}
ExperimentDefinition def = RowToDefinition(entity);
ExperimentStatus status = GetStatus(id, entity);
return(Task.FromResult(new Experiment {
Definition = def, Status = status
}));
}
public override async Task <ExperimentID> StartExperiment(ExperimentDefinition definition, string creator = null, string note = null, string summaryName = null)
{
definition = MakeRelativeDefinition(storage.Location, definition);
ExperimentID id = Interlocked.Increment(ref lastId);
DateTime submitted = DateTime.Now;
double normal = await asyncNormal;
var results = runner.Enqueue(id, definition, normal);
int benchmarksLeft = results.Length;
BenchmarkResult[] benchmarks = new BenchmarkResult[results.Length];
var resultsWithSave =
results.Select((task, index) =>
task.ContinueWith(benchmark =>
{
int left = Interlocked.Decrement(ref benchmarksLeft);
Trace.WriteLine(String.Format("Benchmark {0} completed, {1} left", index, left));
if (benchmark.IsCompleted && !benchmark.IsFaulted)
{
benchmarks[index] = benchmark.Result;
if (left == 0)
{
storage.AddResults(id, benchmarks);
ExperimentInstance val;
runningExperiments.TryRemove(id, out val);
}
return(benchmark.Result);
}
else
{
throw benchmark.Exception;
}
}))
.ToArray();
ExperimentInstance experiment = new ExperimentInstance(id, definition, resultsWithSave);
runningExperiments[id] = experiment;
storage.AddExperiment(id, definition, submitted, creator, note);
return(id);
}
public void AddExperiment(int id, ExperimentDefinition experiment, DateTime submitted, string creator, string note)
{
experimentsTable = experimentsTable.AddRow(new ExperimentEntity
{
ID = id,
Submitted = submitted,
Executable = experiment.Executable,
Parameters = experiment.Parameters,
BenchmarkDirectory = experiment.BenchmarkDirectory,
BenchmarkFileExtension = experiment.BenchmarkFileExtension,
Category = experiment.Category,
BenchmarkTimeout = experiment.BenchmarkTimeout.TotalSeconds,
ExperimentTimeout = experiment.ExperimentTimeout.TotalSeconds,
MemoryLimitMB = experiment.MemoryLimitMB,
GroupName = experiment.GroupName,
Note = note,
Creator = creator
});
experimentsTable.SaveUTF8Bom(Path.Combine(dir.FullName, "experiments.csv"), new WriteSettings(Delimiter.Comma, true, true));
}
public override Task <IEnumerable <Experiment> > FindExperiments(ExperimentFilter?filter = default(ExperimentFilter?))
{
IEnumerable <KeyValuePair <int, ExperimentEntity> > experiments =
storage.GetExperiments()
.ToArray();
if (filter.HasValue)
{
experiments =
experiments
.Where(q =>
{
var id = q.Key;
var e = q.Value;
return((filter.Value.BenchmarkContainerEquals == null || e.BenchmarkDirectory == filter.Value.BenchmarkContainerEquals) &&
(filter.Value.CategoryEquals == null || e.Category == null || e.Category.Contains(filter.Value.CategoryEquals)) &&
(filter.Value.ExecutableEquals == null || e.Executable == null || e.Executable == filter.Value.ExecutableEquals) &&
(filter.Value.ParametersEquals == null || e.Parameters == null || e.Parameters == filter.Value.ParametersEquals) &&
(filter.Value.NotesEquals == null || e.Note == null || e.Note.Contains(filter.Value.NotesEquals)) &&
(filter.Value.CreatorEquals == null || e.Creator == null || e.Creator.Contains(filter.Value.CreatorEquals)));
})
.OrderByDescending(q => q.Value.Submitted);
}
var results = experiments.Select(kvp =>
{
int id = kvp.Key;
ExperimentEntity expRow = kvp.Value;
ExperimentDefinition def = RowToDefinition(expRow);
ExperimentStatus status = GetStatus(id, expRow);
return(new Experiment {
Definition = def, Status = status
});
});
return(Task.FromResult(results));
}
/// <summary> /// Schedules execution of a new experiment from the given experiment definition. /// </summary> /// <param name="definition">Describes the experiment to be performed.</param> /// <returns>Identifier of the new experiment for further reference.</returns> public abstract Task <ExperimentID> StartExperiment(ExperimentDefinition definition, string creator = null, string note = null, string summaryName = null);
