/// <summary>
/// Adds new entry to the experiments table
/// </summary>
/// <param name="experiment"></param>
/// <param name="submitted"></param>
/// <param name="creator"></param>
/// <param name="note"></param>
/// <returns>ID of newly created entry</returns>
public async Task <int> AddExperiment(ExperimentDefinition experiment, DateTime submitted, string creator, string note, string workerInformation)
{
TableQuery <NextExperimentIDEntity> idEntityQuery = QueryForNextId();
bool idChanged;
int id = -1;
do
{
idChanged = false;
var list = (await experimentsTable.ExecuteQuerySegmentedAsync(idEntityQuery, null)).ToList();
NextExperimentIDEntity nextId = null;
if (list.Count == 0)
{
nextId = new NextExperimentIDEntity();
id = 1;
nextId.Id = 2;
idChanged = !(await TryInsertTableEntity(experimentsTable, nextId));
}
else
{
nextId = list[0];
id = nextId.Id;
nextId.Id = id + 1;
idChanged = !(await TryUpdateTableEntity(experimentsTable, nextId));
}
} while (idChanged);
var row = new ExperimentEntity(id);
row.Submitted = submitted;
row.Executable = experiment.Executable;
row.DomainName = experiment.DomainName;
row.Parameters = experiment.Parameters;
row.BenchmarkContainerUri = experiment.BenchmarkContainerUri;
row.BenchmarkDirectory = experiment.BenchmarkDirectory;
row.BenchmarkFileExtension = experiment.BenchmarkFileExtension;
row.Category = experiment.Category;
row.BenchmarkTimeout = experiment.BenchmarkTimeout.TotalSeconds;
row.ExperimentTimeout = experiment.ExperimentTimeout.TotalSeconds;
row.MemoryLimitMB = experiment.MemoryLimitMB;
row.GroupName = experiment.GroupName;
row.Note = note;
row.Creator = creator;
row.WorkerInformation = workerInformation;
row.AdaptiveRunMaxRepetitions = experiment.AdaptiveRunMaxRepetitions;
row.AdaptiveRunMaxTimeInSeconds = experiment.AdaptiveRunMaxTimeInSeconds;
TableOperation insertOperation = TableOperation.Insert(row);
await experimentsTable.ExecuteAsync(insertOperation);
return(id);
}
private static TableQuery <ExperimentEntity> ExperimentPointQuery(int id)
{
string experimentEntityFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, ExperimentEntity.PartitionKeyDefault),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, ExperimentEntity.ExperimentIDToString(id)));
TableQuery <ExperimentEntity> query = new TableQuery <ExperimentEntity>().Where(experimentEntityFilter);
return(query);
}
private Experiment ExperimentFromEntity(int id, ExperimentEntity entity)
{
var totalRuntime = TimeSpan.FromSeconds(entity.TotalRuntime);
ExperimentDefinition def = DefinitionFromEntity(entity);
ExperimentStatus status = new ExperimentStatus(
id, def.Category, entity.Submitted, entity.Creator, entity.Note,
entity.Flag, entity.CompletedBenchmarks, entity.TotalBenchmarks, totalRuntime, entity.WorkerInformation);
return(new Experiment {
Definition = def, Status = status
});
}
//public async Task IncreaseCompletedBenchmarks(int id, int completedBenchmarksRaise)
//{
// TableQuery<ExperimentEntity> query = ExperimentPointQuery(id);
// bool changed = false;
// do
// {
// ExperimentEntity experiment = await FirstExperimentInQuery(query);
// experiment.CompletedBenchmarks += completedBenchmarksRaise;
// changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
// } while (changed);
//}
public async Task UpdateStatusFlag(ExperimentID id, bool flag)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.Flag = flag;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
public async Task SetTotalRuntime(int id, double totalRuntime)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.TotalRuntime = totalRuntime;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
public async Task SetBenchmarksDone(int id, int done)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.CompletedBenchmarks = done;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
public async Task SetBenchmarksTotal(int id, int total)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.TotalBenchmarks = total;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
public async Task UpdateNote(int id, string note)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.Note = note;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
public async Task IncreaseCompletedBenchmarks(int id, int completedBenchmarksRaise)
{
TableQuery <ExperimentEntity> query = ExperimentPointQuery(id);
bool changed = false;
do
{
ExperimentEntity experiment = await FirstExperimentInQuery(query);
experiment.CompletedBenchmarks += completedBenchmarksRaise;
changed = !(await TryUpdateTableEntity(experimentsTable, experiment));
} while (changed);
}
private ExperimentDefinition DefinitionFromEntity(ExperimentEntity experimentEntity)
{
return(ExperimentDefinition.Create(
experimentEntity.Executable,
experimentEntity.BenchmarkContainerUri,
experimentEntity.BenchmarkDirectory,
experimentEntity.BenchmarkFileExtension,
experimentEntity.Parameters,
TimeSpan.FromSeconds(experimentEntity.BenchmarkTimeout),
TimeSpan.FromSeconds(experimentEntity.ExperimentTimeout),
experimentEntity.DomainName,
experimentEntity.Category,
experimentEntity.MemoryLimitMB,
experimentEntity.AdaptiveRunMaxRepetitions,
experimentEntity.AdaptiveRunMaxTimeInSeconds));
}
public async Task <bool> Reinforce(int id, ExperimentDefinition def)
{
if (!CanStart)
{
throw new InvalidOperationException("Cannot start experiment since the manager is in read mode");
}
var refExp = await storage.GetReferenceExperiment();
ExperimentEntity ee = await storage.GetExperiment(id);
string poolId = "z3-nightly";
Regex re = new Regex(@"^.*\(pool: ([^ ]*)\)$");
Match m = re.Match(ee.WorkerInformation);
if (m.Success)
{
poolId = m.Groups[1].Value;
}
using (var bc = BatchClient.Open(batchCreds))
{
var pool = await bc.PoolOperations.GetPoolAsync(poolId);
CloudJob job = bc.JobOperations.CreateJob();
string jid_prefix = BuildJobId(id);
string jid = "";
bool have_jid = false;
int cnt = 1;
while (!have_jid)
{
try
{
jid = String.Format("{0}-{1}", jid_prefix, cnt++);
bc.JobOperations.GetJob(jid);
} catch (BatchException) {
have_jid = true;
}
}
job.Id = jid;
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.JobPreparationTask = new JobPreparationTask
{
CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%\\%AZ_BATCH_JOB_ID% /e /purge) ^& IF %ERRORLEVEL% LEQ 1 exit 0",
ResourceFiles = new List <ResourceFile>(),
WaitForSuccess = true
};
SharedAccessBlobPolicy sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(48),
Permissions = SharedAccessBlobPermissions.Read
};
foreach (CloudBlockBlob blob in storage.ListAzureWorkerBlobs())
{
string sasBlobToken = blob.GetSharedAccessSignature(sasConstraints);
string blobSasUri = String.Format("{0}{1}", blob.Uri, sasBlobToken);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(blobSasUri, blob.Name));
}
string executableFolder = "exec";
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(storage.GetExecutableSasUri(def.Executable), Path.Combine(executableFolder, def.Executable)));
AzureBenchmarkStorage benchStorage = benchStorage = storage.DefaultBenchmarkStorage;
if (refExp != null)
{
string refContentFolder = "refdata";
string refBenchFolder = Path.Combine(refContentFolder, "data");
var refExpExecUri = storage.GetExecutableSasUri(refExp.Definition.Executable);
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(refExpExecUri, Path.Combine(refContentFolder, refExp.Definition.Executable)));
if (refExp.Definition.BenchmarkContainerUri != ExperimentDefinition.DefaultContainerUri)
{
benchStorage = new AzureBenchmarkStorage(refExp.Definition.BenchmarkContainerUri);
}
Domain refdomain;
if (refExp.Definition.DomainName == "Z3")
{
refdomain = new Z3Domain();
}
else
{
throw new InvalidOperationException("Reference experiment uses unknown domain.");
}
SortedSet <string> extensions;
if (string.IsNullOrEmpty(refExp.Definition.BenchmarkFileExtension))
{
extensions = new SortedSet <string>(refdomain.BenchmarkExtensions.Distinct());
}
else
{
extensions = new SortedSet <string>(refExp.Definition.BenchmarkFileExtension.Split('|').Select(s => s.Trim().TrimStart('.')).Distinct());
}
foreach (CloudBlockBlob blob in benchStorage.ListBlobs(refExp.Definition.BenchmarkDirectory, refExp.Definition.Category))
{
string[] parts = blob.Name.Split('/');
string shortName = parts[parts.Length - 1];
var shortnameParts = shortName.Split('.');
if (shortnameParts.Length == 1 && !extensions.Contains(""))
{
continue;
}
var ext = shortnameParts[shortnameParts.Length - 1];
if (!extensions.Contains(ext))
{
continue;
}
job.JobPreparationTask.ResourceFiles.Add(new ResourceFile(benchStorage.GetBlobSASUri(blob), Path.Combine(refBenchFolder, shortName)));
}
}
job.Constraints = new JobConstraints();
if (def.ExperimentTimeout != TimeSpan.Zero)
{
job.Constraints.MaxWallClockTime = def.ExperimentTimeout;
}
job.Constraints.MaxTaskRetryCount = MaxTaskRetryCount;
string summaryName = ee.Creator != "Nightly" ? "" : "Z3Nightly";
AddStarterTask(id, summaryName, job, false, benchStorage.GetContainerSASUri());
}
return(true);
}