public Task <BenchmarkResult>[] Enqueue(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0) { if (experiment == null) { throw new ArgumentNullException("experiment"); } return(RunExperiment(id, experiment, normal, repetitions)); }
private static ExperimentDefinition MakeRelativeDefinition(string experimentsFolder, ExperimentDefinition def) { string relExec = Utils.MakeRelativePath(experimentsFolder, def.Executable); string relDirectory = Utils.MakeRelativePath(experimentsFolder, def.BenchmarkDirectory); return(ExperimentDefinition.Create(relExec, def.BenchmarkContainerUri, relDirectory, def.BenchmarkFileExtension, def.Parameters, def.BenchmarkTimeout, def.ExperimentTimeout, def.DomainName, def.Category, def.MemoryLimitMB)); }
private static ExperimentDefinition RowToDefinition(ExperimentEntity row) { return(ExperimentDefinition.Create( row.Executable, row.BenchmarkContainerUri, row.BenchmarkDirectory, row.BenchmarkFileExtension, row.Parameters, TimeSpan.FromSeconds(row.BenchmarkTimeout), TimeSpan.FromSeconds(row.ExperimentTimeout), row.DomainName, row.Category, row.MemoryLimitMB)); }
private Task <BenchmarkResult>[] RunExperiment(ExperimentID id, ExperimentDefinition experiment, double normal, int repetitions = 0) { if (experiment == null) { throw new ArgumentNullException("experiment"); } if (factory == null) { throw new ArgumentNullException("factory"); } Domain domain = domainResolver.GetDomain(experiment.DomainName); string executable; if (Path.IsPathRooted(experiment.Executable)) { executable = experiment.Executable; } else { executable = Path.Combine(rootFolder, experiment.Executable); } if (!File.Exists(executable)) { throw new ArgumentException("Executable not found"); } string benchmarkFolder = string.IsNullOrEmpty(experiment.Category) ? experiment.BenchmarkDirectory : Path.Combine(experiment.BenchmarkDirectory, experiment.Category); if (!Path.IsPathRooted(benchmarkFolder)) { benchmarkFolder = Path.Combine(rootFolder, benchmarkFolder); } var benchmarks = Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories); var results = new List <Task <BenchmarkResult> >(256); foreach (string benchmarkFile in Directory.GetFiles(benchmarkFolder, "*." + experiment.BenchmarkFileExtension, SearchOption.AllDirectories)) { var task = factory.StartNew(_benchmark => { string inputFullPath = (string)_benchmark; string inputRelativePath = Utils.MakeRelativePath(benchmarkFolder, inputFullPath); Trace.WriteLine("Running benchmark " + Path.GetFileName(inputRelativePath)); string args = experiment.Parameters; return(RunBenchmark(id, executable, experiment.Parameters, inputRelativePath, inputFullPath, repetitions, experiment.BenchmarkTimeout, experiment.MemoryLimitMB, null, null, domain, normal)); }, benchmarkFile, TaskCreationOptions.LongRunning); results.Add(task); } return(results.ToArray()); }
public ExperimentInstance(ExperimentID id, ExperimentDefinition def, Task <BenchmarkResult>[] results) { if (def == null) { throw new ArgumentNullException("def"); } this.id = id; this.def = def; this.results = results; }
public static LocalExperimentManager NewExperiments(string experimentsFolder, ReferenceExperiment reference, IDomainResolver domainResolver) { ExperimentDefinition def = MakeRelativeDefinition(experimentsFolder, reference.Definition); ReferenceExperiment relRef = new ReferenceExperiment(def, reference.Repetitions, reference.ReferenceValue); FileStorage storage = FileStorage.Open(experimentsFolder); storage.Clear(); storage.SaveReferenceExperiment(relRef); LocalExperimentManager manager = new LocalExperimentManager(storage, domainResolver); return(manager); }
public ReferenceExperiment(ExperimentDefinition def, int repetitions, double referenceValue) { if (def == null) { throw new ArgumentNullException("def"); } if (repetitions < 1) { throw new ArgumentOutOfRangeException("repetitions", "Number of repetitions must be greater than zero"); } Definition = def; Repetitions = repetitions; ReferenceValue = referenceValue; }
public override Task <Experiment> TryFindExperiment(int id) { ExperimentEntity entity; if (!storage.GetExperiments().TryGetValue(id, out entity)) { return(null); } ExperimentDefinition def = RowToDefinition(entity); ExperimentStatus status = GetStatus(id, entity); return(Task.FromResult(new Experiment { Definition = def, Status = status })); }
public override async Task <ExperimentID> StartExperiment(ExperimentDefinition definition, string creator = null, string note = null, string summaryName = null) { definition = MakeRelativeDefinition(storage.Location, definition); ExperimentID id = Interlocked.Increment(ref lastId); DateTime submitted = DateTime.Now; double normal = await asyncNormal; var results = runner.Enqueue(id, definition, normal); int benchmarksLeft = results.Length; BenchmarkResult[] benchmarks = new BenchmarkResult[results.Length]; var resultsWithSave = results.Select((task, index) => task.ContinueWith(benchmark => { int left = Interlocked.Decrement(ref benchmarksLeft); Trace.WriteLine(String.Format("Benchmark {0} completed, {1} left", index, left)); if (benchmark.IsCompleted && !benchmark.IsFaulted) { benchmarks[index] = benchmark.Result; if (left == 0) { storage.AddResults(id, benchmarks); ExperimentInstance val; runningExperiments.TryRemove(id, out val); } return(benchmark.Result); } else { throw benchmark.Exception; } })) .ToArray(); ExperimentInstance experiment = new ExperimentInstance(id, definition, resultsWithSave); runningExperiments[id] = experiment; storage.AddExperiment(id, definition, submitted, creator, note); return(id); }
public void AddExperiment(int id, ExperimentDefinition experiment, DateTime submitted, string creator, string note) { experimentsTable = experimentsTable.AddRow(new ExperimentEntity { ID = id, Submitted = submitted, Executable = experiment.Executable, Parameters = experiment.Parameters, BenchmarkDirectory = experiment.BenchmarkDirectory, BenchmarkFileExtension = experiment.BenchmarkFileExtension, Category = experiment.Category, BenchmarkTimeout = experiment.BenchmarkTimeout.TotalSeconds, ExperimentTimeout = experiment.ExperimentTimeout.TotalSeconds, MemoryLimitMB = experiment.MemoryLimitMB, GroupName = experiment.GroupName, Note = note, Creator = creator }); experimentsTable.SaveUTF8Bom(Path.Combine(dir.FullName, "experiments.csv"), new WriteSettings(Delimiter.Comma, true, true)); }
public override Task <IEnumerable <Experiment> > FindExperiments(ExperimentFilter?filter = default(ExperimentFilter?)) { IEnumerable <KeyValuePair <int, ExperimentEntity> > experiments = storage.GetExperiments() .ToArray(); if (filter.HasValue) { experiments = experiments .Where(q => { var id = q.Key; var e = q.Value; return((filter.Value.BenchmarkContainerEquals == null || e.BenchmarkDirectory == filter.Value.BenchmarkContainerEquals) && (filter.Value.CategoryEquals == null || e.Category == null || e.Category.Contains(filter.Value.CategoryEquals)) && (filter.Value.ExecutableEquals == null || e.Executable == null || e.Executable == filter.Value.ExecutableEquals) && (filter.Value.ParametersEquals == null || e.Parameters == null || e.Parameters == filter.Value.ParametersEquals) && (filter.Value.NotesEquals == null || e.Note == null || e.Note.Contains(filter.Value.NotesEquals)) && (filter.Value.CreatorEquals == null || e.Creator == null || e.Creator.Contains(filter.Value.CreatorEquals))); }) .OrderByDescending(q => q.Value.Submitted); } var results = experiments.Select(kvp => { int id = kvp.Key; ExperimentEntity expRow = kvp.Value; ExperimentDefinition def = RowToDefinition(expRow); ExperimentStatus status = GetStatus(id, expRow); return(new Experiment { Definition = def, Status = status }); }); return(Task.FromResult(results)); }
/// <summary> /// Schedules execution of a new experiment from the given experiment definition. /// </summary> /// <param name="definition">Describes the experiment to be performed.</param> /// <returns>Identifier of the new experiment for further reference.</returns> public abstract Task <ExperimentID> StartExperiment(ExperimentDefinition definition, string creator = null, string note = null, string summaryName = null);