public CompareExperimentsViewModel(int id1, int id2, ExperimentDefinition def1, ExperimentDefinition def2, ExperimentManager manager, IUIService message)
{
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (message == null)
{
throw new ArgumentNullException("message");
}
this.manager = manager;
this.uiService = message;
this.id1 = id1;
this.id2 = id2;
this.checkIgnoreCategory = false;
this.checkIgnorePostfix = false;
this.checkIgnorePrefix = false;
this.extension1 = "." + def1.BenchmarkFileExtension.Split('|')[0];
this.extension2 = "." + def2.BenchmarkFileExtension.Split('|')[0];
this.category1 = def1.Category;
this.category2 = def2.Category;
this.sharedDirectory1 = def1.BenchmarkDirectory;
this.sharedDirectory2 = def2.BenchmarkDirectory;
this.isFiltering = false;
DownloadResultsAsync();
}
static async Task SubmitExperiment(AzureExperimentManager manager, Stream source, string fileName, string commit_hash)
{
Trace.WriteLine("Uploading new executable...");
string packageName = await manager.Storage.UploadNewExecutable(source, fileName, Settings.Creator);
Trace.WriteLine("Successfully uploaded as " + packageName);
ExperimentDefinition definition =
ExperimentDefinition.Create(
packageName,
ExperimentDefinition.DefaultContainerUri,
Settings.BenchmarkDirectory,
Settings.BenchmarkFileExtension,
Settings.Parameters,
TimeSpan.FromSeconds(Settings.BenchmarkTimeoutSeconds),
TimeSpan.FromSeconds(Settings.ExperimentTimeoutSeconds),
Settings.Domain,
Settings.BenchmarkCategory,
Settings.MemoryLimitMegabytes,
1, 0 /* Run each benchmark once (i.e. adaptive run is off) */);
Trace.WriteLine(string.Format("Starting nightly experiment in Batch pool \"{0}\"...", Settings.AzureBatchPoolId));
manager.BatchPoolID = Settings.AzureBatchPoolId;
string note = commit_hash != null?
string.Format("{0} for https://github.com/{1}/{2}/commit/{3}", Settings.ExperimentNote, Settings.GitHubOwner, Settings.GitHubZ3Repository, commit_hash) :
Settings.ExperimentNote;
var summaryName = Settings.SummaryName == "" ? null : Settings.SummaryName;
var experimentId = await manager.StartExperiment(definition, Settings.Creator, note, summaryName);
Trace.WriteLine(string.Format("Done, experiment id {0}.", experimentId));
}
public async Task FindExperiments()
{
ExperimentDefinition def1 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare");
ExperimentDefinition def2 = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 2", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default", category: "IdentitySquare");
ExperimentManager manager = NewManager();
var e1 = await manager.StartExperiment(def1);
var e2 = await manager.StartExperiment(def1);
var e4 = await manager.StartExperiment(def2);
var r1 = await Task.WhenAll(manager.GetResults(e1));
var r2 = await Task.WhenAll(manager.GetResults(e2));
var r4 = await Task.WhenAll(manager.GetResults(e4));
var loaded = (await manager.FindExperiments()).ToArray();
Assert.AreEqual(3, loaded.Length, "Number of found experiments (same manager)");
var manager2 = OpenManager(manager);
var loaded2 = (await manager2.FindExperiments()).ToArray();
Assert.AreEqual(3, loaded2.Length, "Number of found experiments (reloaded)");
var loaded3 = (await manager2.FindExperiments(new ExperimentManager.ExperimentFilter {
ParametersEquals = "{0} 2"
})).ToArray();
Assert.AreEqual(1, loaded3.Length, "Number of found experiments (reloaded, filtered)");
}
/// <summary>
/// Adds new entry to the experiments table
/// </summary>
/// <param name="experiment"></param>
/// <param name="submitted"></param>
/// <param name="creator"></param>
/// <param name="note"></param>
/// <returns>ID of newly created entry</returns>
public async Task <int> AddExperiment(ExperimentDefinition experiment, DateTime submitted, string creator, string note, string workerInformation)
{
TableQuery <NextExperimentIDEntity> idEntityQuery = QueryForNextId();
bool idChanged;
int id = -1;
do
{
idChanged = false;
var list = (await experimentsTable.ExecuteQuerySegmentedAsync(idEntityQuery, null)).ToList();
NextExperimentIDEntity nextId = null;
if (list.Count == 0)
{
nextId = new NextExperimentIDEntity();
id = 1;
nextId.Id = 2;
idChanged = !(await TryInsertTableEntity(experimentsTable, nextId));
}
else
{
nextId = list[0];
id = nextId.Id;
nextId.Id = id + 1;
idChanged = !(await TryUpdateTableEntity(experimentsTable, nextId));
}
} while (idChanged);
var row = new ExperimentEntity(id);
row.Submitted = submitted;
row.Executable = experiment.Executable;
row.DomainName = experiment.DomainName;
row.Parameters = experiment.Parameters;
row.BenchmarkContainerUri = experiment.BenchmarkContainerUri;
row.BenchmarkDirectory = experiment.BenchmarkDirectory;
row.BenchmarkFileExtension = experiment.BenchmarkFileExtension;
row.Category = experiment.Category;
row.BenchmarkTimeout = experiment.BenchmarkTimeout.TotalSeconds;
row.ExperimentTimeout = experiment.ExperimentTimeout.TotalSeconds;
row.MemoryLimitMB = experiment.MemoryLimitMB;
row.GroupName = experiment.GroupName;
row.Note = note;
row.Creator = creator;
row.WorkerInformation = workerInformation;
row.AdaptiveRunMaxRepetitions = experiment.AdaptiveRunMaxRepetitions;
row.AdaptiveRunMaxTimeInSeconds = experiment.AdaptiveRunMaxTimeInSeconds;
TableOperation insertOperation = TableOperation.Insert(row);
await experimentsTable.ExecuteAsync(insertOperation);
return(id);
}
public ExperimentPropertiesViewModel(ExperimentDefinition def, ExperimentStatus status, Domain domain, ExperimentManager manager, IUIService ui)
{
if (def == null)
{
throw new ArgumentNullException("def");
}
if (status == null)
{
throw new ArgumentNullException("status");
}
if (domain == null)
{
throw new ArgumentNullException("domain");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (ui == null)
{
throw new ArgumentNullException("ui");
}
this.id = status.ID;
this.definition = def;
this.status = status;
this.domain = domain;
this.manager = manager;
this.ui = ui;
currentNote = status.Note;
isSyncing = true;
Sync = new DelegateCommand(async _ =>
{
isSyncing = true;
Sync.RaiseCanExecuteChanged();
try
{
if (NoteChanged)
{
await SubmitNote();
}
await Refresh();
}
catch (Exception ex)
{
ui.ShowError(ex, "Failed to synchronize experiment properties");
}
finally
{
isSyncing = false;
Sync.RaiseCanExecuteChanged();
}
},
_ => !isSyncing);
Initialize();
}
private static ExperimentManager NewManager()
{
ReferenceExperiment reference = new ReferenceExperiment(
ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "reference", "csv", "{0} 10", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default"),
1, 0.17);
ExperimentManager manager = LocalExperimentManager.NewExperiments("measure" + Guid.NewGuid(), reference, domainResolver);
return(manager);
}
private Experiment ExperimentFromEntity(int id, ExperimentEntity entity)
{
var totalRuntime = TimeSpan.FromSeconds(entity.TotalRuntime);
ExperimentDefinition def = DefinitionFromEntity(entity);
ExperimentStatus status = new ExperimentStatus(
id, def.Category, entity.Submitted, entity.Creator, entity.Note,
entity.Flag, entity.CompletedBenchmarks, entity.TotalBenchmarks, totalRuntime, entity.WorkerInformation);
return(new Experiment {
Definition = def, Status = status
});
}
private ExperimentDefinition DefinitionFromEntity(ExperimentEntity experimentEntity)
{
return(ExperimentDefinition.Create(
experimentEntity.Executable,
experimentEntity.BenchmarkContainerUri,
experimentEntity.BenchmarkDirectory,
experimentEntity.BenchmarkFileExtension,
experimentEntity.Parameters,
TimeSpan.FromSeconds(experimentEntity.BenchmarkTimeout),
TimeSpan.FromSeconds(experimentEntity.ExperimentTimeout),
experimentEntity.DomainName,
experimentEntity.Category,
experimentEntity.MemoryLimitMB,
experimentEntity.AdaptiveRunMaxRepetitions,
experimentEntity.AdaptiveRunMaxTimeInSeconds));
}
public async Task RunExperiment()
{
ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_1", "csv", "{0}", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default");
ExperimentManager manager = NewManager();
var expId = await manager.StartExperiment(def);
var results = (await manager.GetResults(expId)).Benchmarks;
Assert.AreEqual(1, results.Length, "Number of completed benchmarks");
var res = results[0];
Assert.AreEqual(0, res.ExitCode, "exit code");
Assert.AreEqual(ResultStatus.Success, res.Status, "status");
Assert.IsTrue(res.CPUTime.TotalSeconds < 1, "Total runtime");
}
static async Task Run(ExperimentManager manager, ExperimentDefinition definition)
{
int id = await manager.StartExperiment(definition);
var results = manager.GetResults(id);
var filter = new ExperimentManager.ExperimentFilter
{
BenchmarkContainerEquals = definition.BenchmarkDirectory,
CategoryEquals = definition.Category,
ExecutableEquals = definition.Executable,
ParametersEquals = definition.Parameters
};
var history = (await manager.FindExperiments(filter)).Where(q => q.ID != id).ToArray();
Dictionary <string, BenchmarkResult> lastBenchmarks = new Dictionary <string, BenchmarkResult>();
if (history.Length != 0)
{
var lastResults = await manager.GetResults(history.Max(e => e.ID));
foreach (var b in lastResults.Benchmarks)
{
lastBenchmarks[b.BenchmarkFileName] = b;
}
}
var print = results.ContinueWith(task =>
{
if (task.IsFaulted)
{
PrintError(String.Format("Failed to complete benchmarks {0}", task.Exception.Message));
return;
}
BenchmarkResult[] benchmarks = task.Result.Benchmarks;
foreach (var benchmark in benchmarks)
{
BenchmarkResult lastBenchmark = null;
lastBenchmarks.TryGetValue(benchmark.BenchmarkFileName, out lastBenchmark);
if (lastBenchmark != null && lastBenchmark.Status != Measurement.ResultStatus.Success)
{
lastBenchmark = null;
}
PrintBenchmark(benchmark, lastBenchmark);
}
});
await print;
}
public async Task RunExperimentsWithCategory()
{
ExperimentDefinition def = ExperimentDefinition.Create("LinearEquationSolver.exe", ExperimentDefinition.LocalDiskContainerUri, "benchmarks_2", "csv", "{0} 1000", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(0), "default",
category: "IdentitySquare");
ExperimentManager manager = NewManager();
var expId = await manager.StartExperiment(def);
var results = (await manager.GetResults(expId)).Benchmarks;
Assert.AreEqual(3, results.Length, "Number of completed benchmarks");
foreach (var res in results)
{
Assert.AreEqual(0, res.ExitCode, "exit code");
Assert.AreEqual(ResultStatus.Success, res.Status, "status");
Assert.IsTrue(res.TotalProcessorTime.TotalSeconds < 10, "Total runtime");
}
}
public ExperimentStatusViewModel(Experiment exp, ExperimentManager manager, IUIService message)
{
if (exp == null)
{
throw new ArgumentNullException("experiment");
}
if (manager == null)
{
throw new ArgumentNullException("manager");
}
if (message == null)
{
throw new ArgumentNullException("message");
}
this.status = exp.Status;
this.definition = exp.Definition;
this.flag = status.Flag;
this.manager = manager;
this.uiService = message;
}
static void Main(string[] args)
{
Keys keys = JsonConvert.DeserializeObject <Keys>(File.ReadAllText("..\\..\\keys.json"));
var storage = new AzureExperimentStorage(keys.storageName, keys.storageKey);
var manager = AzureExperimentManager.Open(storage, keys.batchUri, keys.batchName, keys.batchKey);
//var refExp = new ReferenceExperiment(ExperimentDefinition.Create("referencez3.zip", ExperimentDefinition.DefaultContainerUri, "reference", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), "Z3", null, 2048), 20, 16.34375);
// storage.SaveReferenceExperiment(refExp).Wait();
var id = manager.StartExperiment(ExperimentDefinition.Create("z3.zip", ExperimentDefinition.DefaultContainerUri, "QF_BV", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), TimeSpan.FromSeconds(0), "Z3", "asp", 2048, 1, 0), "Dmitry K").Result;
Console.WriteLine("Experiment id:" + id);
//manager.RestartBenchmarks(159, new string[] {
// "15Puzzle/15-puzzle.init10.smt2",
// "15Puzzle/15-puzzle.init11.smt2",
// "15Puzzle/15-puzzle.init12.smt2",
// "15Puzzle/15-puzzle.init13.smt2",
// "15Puzzle/15-puzzle.init14.smt2",
// "15Puzzle/15-puzzle.init15.smt2",
// "15Puzzle/15-puzzle.init2.smt2",
// "15Puzzle/15-puzzle.init3.smt2",
// "15Puzzle/15-puzzle.init4.smt2",
// "15Puzzle/15-puzzle.init5.smt2",
// "15Puzzle/15-puzzle.init6.smt2",
// "15Puzzle/15-puzzle.init7.smt2",
// "15Puzzle/15-puzzle.init8.smt2",
// "15Puzzle/15-puzzle.init9.smt2",
// "15Puzzle/15puzzle_ins.lp.smt2"
//}).Wait();
Console.WriteLine("Done.");
Console.ReadLine();
}
public async Task <bool> Reinforce(int id, ExperimentDefinition def)
{
if (!CanStart)
{
throw new InvalidOperationException("Cannot start experiment since the manager is in read mode");
}
var refExp = await storage.GetReferenceExperiment();
ExperimentEntity ee = await storage.GetExperiment(id);
string poolId = "z3-nightly";
Regex re = new Regex(@"^.*\(pool: ([^ ]*)\)$");
Match m = re.Match(ee.WorkerInformation);
if (m.Success)
{
poolId = m.Groups[1].Value;
}
using (var bc = BatchClient.Open(batchCreds))
{
var pool = await bc.PoolOperations.GetPoolAsync(poolId);
CloudJob job = bc.JobOperations.CreateJob();
string jid_prefix = BuildJobId(id);
string jid = "";
bool have_jid = false;
int cnt = 1;
while (!have_jid)
{
try
{
jid = String.Format("{0}-{1}", jid_prefix, cnt++);
bc.JobOperations.GetJob(jid);
} catch (BatchException) {
have_jid = true;
}
}
job.Id = jid;
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.JobPreparationTask = new JobPreparationTask
{
CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%\\%AZ_BATCH_JOB_ID% /e /purge) ^& IF %ERRORLEVEL% LEQ 1 exit 0",
ResourceFiles = new List <ResourceFile>(),
WaitForSuccess = true
};
SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(48),
Permissions = SharedAccessBlobPermissions.Read
};
foreach (CloudBlockBlob blob in storage.ListAzureWorkerBlobs())
{
string sasBlobToken = blob.GetSharedAccessSignature(sasConstraints);
string blobSasUri = String.Format("{0}{1}", blob.Uri, sasBlobToken);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(blobSasUri, blob.Name));
}
string executableFolder = "exec";
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(storage.GetExecutableSasUri(def.Executable), Path.Combine(executableFolder, def.Executable)));
AzureBenchmarkStorage benchStorage = benchStorage = storage.DefaultBenchmarkStorage;
if (refExp != null)
{
string refContentFolder = "refdata";
string refBenchFolder = Path.Combine(refContentFolder, "data");
var refExpExecUri = storage.GetExecutableSasUri(refExp.Definition.Executable);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(refExpExecUri, Path.Combine(refContentFolder, refExp.Definition.Executable)));
if (refExp.Definition.BenchmarkContainerUri != ExperimentDefinition.DefaultContainerUri)
{
benchStorage = new AzureBenchmarkStorage(refExp.Definition.BenchmarkContainerUri);
}
Domain refdomain;
if (refExp.Definition.DomainName == "Z3")
{
refdomain = new Z3Domain();
}
else
{
throw new InvalidOperationException("Reference experiment uses unknown domain.");
}
SortedSet <string> extensions;
if (string.IsNullOrEmpty(refExp.Definition.BenchmarkFileExtension))
{
extensions = new SortedSet <string>(refdomain.BenchmarkExtensions.Distinct());
}
else
{
extensions = new SortedSet <string>(refExp.Definition.BenchmarkFileExtension.Split('|').Select(s => s.Trim().TrimStart('.')).Distinct());
}
foreach (CloudBlockBlob blob in benchStorage.ListBlobs(refExp.Definition.BenchmarkDirectory, refExp.Definition.Category))
{
string[] parts = blob.Name.Split('/');
string shortName = parts[parts.Length - 1];
var shortnameParts = shortName.Split('.');
if (shortnameParts.Length == 1 && !extensions.Contains(""))
{
continue;
}
var ext = shortnameParts[shortnameParts.Length - 1];
if (!extensions.Contains(ext))
{
continue;
}
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(benchStorage.GetBlobSASUri(blob), Path.Combine(refBenchFolder, shortName)));
}
}
job.Constraints = new JobConstraints();
if (def.ExperimentTimeout != TimeSpan.Zero)
{
job.Constraints.MaxWallClockTime = def.ExperimentTimeout;
}
job.Constraints.MaxTaskRetryCount = MaxTaskRetryCount;
string summaryName = ee.Creator != "Nightly" ? "" : "Z3Nightly";
AddStarterTask(id, summaryName, job, false, benchStorage.GetContainerSASUri());
}
return(true);
}
static int Main(string[] args)
{
if (args.Length < 5 || args.Length > 7)
{
return(PrintSyntax());
}
int k = 0;
bool init = false;
if (args[k] == "--init")
{
if (args.Length != 7)
{
return(PrintSyntax());
}
init = true;
k++;
}
else
{
if (args.Length != 5)
{
return(PrintSyntax());
}
}
string executable = args[k++];
string arguments = args[k++];
string benchmarkContainerUri = ExperimentDefinition.LocalDiskContainerUri;
string benchmarkDirectory = args[k++];
string category = args[k++];
string extension = args[k++];
int repetitions = 1;
double referenceValue = 1.0;
if (init)
{
repetitions = int.Parse(args[k++], CultureInfo.InvariantCulture);
}
TimeSpan timeout = TimeSpan.FromHours(1);
ExperimentDefinition definition = ExperimentDefinition.Create(executable, benchmarkContainerUri, benchmarkDirectory, extension, arguments, timeout, TimeSpan.FromSeconds(0), Measurement.Domain.Default.Name, category: category);
string version = GetVersion(executable);
if (init)
{
Print(String.Format("Initializing environment..."));
}
else
{
Print(String.Format("Measuring performance of {0} {1}...\n", executable, version));
}
IDomainResolver domainResolver = new DomainResolver(new[] { Measurement.Domain.Default });
if (init)
{
var reference = new ReferenceExperiment(definition, repetitions, referenceValue);
ExperimentManager manager = LocalExperimentManager.NewExperiments("measure", reference, domainResolver);
}
else
{
ExperimentManager manager = LocalExperimentManager.OpenExperiments("measure", domainResolver);
Run(manager, definition).Wait();
}
return(0);
}
public override async Task <ExperimentID> StartExperiment(ExperimentDefinition definition, string creator = null, string note = null, string summaryName = null)
{
if (!CanStart)
{
throw new InvalidOperationException("Cannot start experiment since the manager is in read mode");
}
var refExp = await storage.GetReferenceExperiment();
var poolId = this.BatchPoolID;
int id;
using (var bc = BatchClient.Open(batchCreds))
{
var pool = await bc.PoolOperations.GetPoolAsync(poolId);
id = await storage.AddExperiment(definition, DateTime.Now, creator, note, string.Format("{0} (pool: {1})", pool.VirtualMachineSize, poolId));
CloudJob job = bc.JobOperations.CreateJob();
job.Id = BuildJobId(id);
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.JobPreparationTask = new JobPreparationTask
{
CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%\\%AZ_BATCH_JOB_ID% /e /purge) ^& IF %ERRORLEVEL% LEQ 1 exit 0",
ResourceFiles = new List <ResourceFile>(),
WaitForSuccess = true
};
SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(48),
Permissions = SharedAccessBlobPermissions.Read
};
foreach (CloudBlockBlob blob in storage.ListAzureWorkerBlobs())
{
string sasBlobToken = blob.GetSharedAccessSignature(sasConstraints);
string blobSasUri = String.Format("{0}{1}", blob.Uri, sasBlobToken);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(blobSasUri, blob.Name));
}
string executableFolder = "exec";
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(storage.GetExecutableSasUri(definition.Executable), Path.Combine(executableFolder, definition.Executable)));
AzureBenchmarkStorage benchStorage = storage.DefaultBenchmarkStorage;
if (refExp != null)
{
string refContentFolder = "refdata";
string refBenchFolder = Path.Combine(refContentFolder, "data");
var refExpExecUri = storage.GetExecutableSasUri(refExp.Definition.Executable);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(refExpExecUri, Path.Combine(refContentFolder, refExp.Definition.Executable)));
if (refExp.Definition.BenchmarkContainerUri != ExperimentDefinition.DefaultContainerUri)
{
benchStorage = new AzureBenchmarkStorage(refExp.Definition.BenchmarkContainerUri);
}
Domain refdomain;
if (refExp.Definition.DomainName == "Z3")
{
refdomain = new Z3Domain();
}
else
{
throw new InvalidOperationException("Reference experiment uses unknown domain.");
}
SortedSet <string> extensions;
if (string.IsNullOrEmpty(refExp.Definition.BenchmarkFileExtension))
{
extensions = new SortedSet <string>(refdomain.BenchmarkExtensions.Distinct());
}
else
{
extensions = new SortedSet <string>(refExp.Definition.BenchmarkFileExtension.Split('|').Select(s => s.Trim().TrimStart('.')).Distinct());
}
foreach (CloudBlockBlob blob in benchStorage.ListBlobs(refExp.Definition.BenchmarkDirectory, refExp.Definition.Category))
{
string[] parts = blob.Name.Split('/');
string shortName = parts[parts.Length - 1];
var shortnameParts = shortName.Split('.');
if (shortnameParts.Length == 1 && !extensions.Contains(""))
{
continue;
}
var ext = shortnameParts[shortnameParts.Length - 1];
if (!extensions.Contains(ext))
{
continue;
}
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(benchStorage.GetBlobSASUri(blob), Path.Combine(refBenchFolder, shortName)));
}
}
job.Constraints = new JobConstraints();
if (definition.ExperimentTimeout != TimeSpan.Zero)
{
job.Constraints.MaxWallClockTime = definition.ExperimentTimeout;
}
job.Constraints.MaxTaskRetryCount = MaxTaskRetryCount;
AddStarterTask(id, summaryName, job, false, benchStorage.GetContainerSASUri());
}
return(id);
}
public async Task Reinforce(int id, ExperimentDefinition d)
{
await manager.Reinforce(id, d);
}
public async Task <Tuple <string, int?, Exception>[]> SubmitExperiments(NewExperimentViewModel newExperiment)
{
// Uploading package with binaries
string creator = newExperiment.Creator;
string packageName;
if (newExperiment.UseMostRecentExecutable)
{
packageName = await newExperiment.GetRecentExecutable();
if (String.IsNullOrEmpty(packageName))
{
throw new InvalidOperationException("Executable package is not available");
}
}
else if (newExperiment.UseOriginalExecutable)
{
packageName = newExperiment.Executable;
}
else // upload new executable
{
if (newExperiment.ExecutableFileNames == null || newExperiment.ExecutableFileNames.Length == 0)
{
throw new InvalidOperationException("New executable should be uploaded but no files selected");
}
if (newExperiment.ExecutableFileNames.Length == 1) // single file will be uploaded as is
{
string fileName = Path.GetFileName(newExperiment.ExecutableFileNames[0]);
using (Stream stream = File.Open(newExperiment.ExecutableFileNames[0], FileMode.Open, FileAccess.Read, FileShare.Read))
{
packageName = await manager.Storage.UploadNewExecutable(stream, fileName, creator);
}
}
else // multiple files are packed into zip
{
if (String.IsNullOrEmpty(newExperiment.MainExecutable))
{
throw new InvalidOperationException("There is no main executable selected");
}
string fileName = Path.GetFileNameWithoutExtension(newExperiment.MainExecutable) + ".zip";
using (MemoryStream stream = new MemoryStream(20 << 20))
{
Measurement.ExecutablePackage.PackToStream(newExperiment.ExecutableFileNames, newExperiment.MainExecutable, stream);
packageName = await manager.Storage.UploadNewExecutable(stream, fileName, creator);
}
}
}
int maxRepetitions = 1;
double maxTime = 0;
if (newExperiment.AllowAdaptiveRuns)
{
maxRepetitions = newExperiment.MaxRepetitions;
maxTime = newExperiment.MaxTimeForAdaptiveRuns;
}
// Submitting experiments
string[] cats = newExperiment.Categories.Split('|');
var res = new Tuple <string, int?, Exception> [cats.Length];
for (int i = 0; i < cats.Length; i++)
{
string category = cats[i].Trim();
ExperimentDefinition def =
ExperimentDefinition.Create(
packageName, newExperiment.BenchmarkContainerUri, newExperiment.BenchmarkDirectory, newExperiment.Extension, newExperiment.Parameters,
TimeSpan.FromSeconds(newExperiment.BenchmarkTimeoutSec), TimeSpan.FromSeconds(newExperiment.ExperimentTimeoutSec), newExperiment.Domain,
category, newExperiment.BenchmarkMemoryLimitMb, maxRepetitions, maxTime);
try
{
// Starts the experiment job
manager.BatchPoolID = newExperiment.Pool;
int id = await manager.StartExperiment(def, creator, newExperiment.Note);
res[i] = Tuple.Create <string, int?, Exception>(category, id, null);
}
catch (Exception ex)
{
res[i] = Tuple.Create <string, int?, Exception>(category, null, ex);
}
}
return(res);
}
public CompareExperimentsViewModel BuildComparingResults(int id1, int id2, ExperimentDefinition def1, ExperimentDefinition def2)
{
return(new CompareExperimentsViewModel(id1, id2, def1, def2, manager, uiService));
}
