private static AzureBenchmarkResult PrepareBenchmarkResult(CSVRow r, AzureExperimentStorage storage, ConcurrentDictionary <string, string> uploadedOutputs, int expId, DateTime submittedTime)
{
var properties = new Dictionary <string, string>()
{
{ "SAT", r.SAT.ToString() },
{ "UNSAT", r.UNSAT.ToString() },
{ "UNKNOWN", r.UNKNOWN.ToString() },
{ "TargetSAT", r.TargetSAT.ToString() },
{ "TargetUNSAT", r.TargetUNSAT.ToString() },
{ "TargetUNKNOWN", r.TargetUNKNOWN.ToString() },
};
var b = new AzureBenchmarkResult
{
AcquireTime = submittedTime,
BenchmarkFileName = r.Filename.Replace('\\', '/'),
ExitCode = r.ReturnValue,
ExperimentID = expId,
NormalizedRuntime = r.Runtime,
PeakMemorySizeMB = 0,
Properties = properties,
Status = ResultCodeToStatus(r.ResultCode),
StdErr = r.StdErr,
StdOut = r.StdOut,
StdErrExtStorageIdx = "",
StdOutExtStorageIdx = "",
TotalProcessorTime = TimeSpan.FromSeconds(r.Runtime),
WallClockTime = TimeSpan.FromSeconds(r.Runtime),
};
return(b);
}
public static void Init(TestContext context)
{
if (File.Exists("ConnectionString.txt"))
{
storageConnectionString = File.ReadAllText("ConnectionString.txt");
AzureExperimentStorage storage = new AzureExperimentStorage(storageConnectionString);
}
}
public async Task GetExperimentsFromCloud()
{
ValidatesConnectionString();
AzureExperimentStorage storage = new AzureExperimentStorage(storageConnectionString);
AzureExperimentManager manager = AzureExperimentManager.OpenWithoutStart(storage);
var experiments = (await manager.FindExperiments()).ToArray();
Assert.IsTrue(experiments.Length > 0);
}
static AzureBenchmarkStorage CreateBenchmarkStorage(string uri, AzureExperimentStorage experimentStorage)
{
if (uri == ExperimentDefinition.DefaultContainerUri)
{
return(experimentStorage.DefaultBenchmarkStorage);
}
else
{
return(new AzureBenchmarkStorage(uri));
}
}
static async Task CollectResults(int experimentId, AzureExperimentStorage storage)
{
Console.WriteLine("Started collection.");
var queue = storage.GetResultsQueueReference(experimentId);
List <AzureBenchmarkResult> results = new List <AzureBenchmarkResult>(); // (await storage.GetAzureExperimentResults(experimentId)).ToList();
int processedBenchmarks = 0; // goodResults.Count + badResults.Count;// results.Count;
var formatter = new BinaryFormatter();
bool completed = false;
do
{
completed = totalBenchmarksToProcess != -1 && completedTasksCount >= totalBenchmarksToProcess;
var messages = queue.GetMessages(32, TimeSpan.FromMinutes(5));
int messageCount = messages.Count();
completed = completed && messageCount == 0;
foreach (CloudQueueMessage message in messages)
{
using (var ms = new MemoryStream(message.AsBytes))
{
goodResults.Add((AzureBenchmarkResult)formatter.Deserialize(ms));
}
}
int oldCount = results.Count;
results = goodResults.Concat(badResults).ToList();
var tuple = SortCountUniqueNamesAndRemoveExactDuplicates(results);
processedBenchmarks = tuple.Item1;
results = tuple.Item2;
await storage.PutAzureExperimentResults(experimentId, results.ToArray(), AzureExperimentStorage.UploadBlobMode.CreateOrReplace);
int completedBenchmarks = totalBenchmarks == -1 ? processedBenchmarks : totalBenchmarks - totalBenchmarksToProcess + completedTasksCount;
await storage.SetCompletedBenchmarks(experimentId, completedBenchmarks);
Console.WriteLine("Setting completed benchmarks to {0}.\nTotal benchmarks: {1}\nProcessed benchmarks: {2}\nTotal to process: {3}\nCompleted tasks: {4}\nMessage count: {5}", completedBenchmarks, totalBenchmarks, processedBenchmarks, totalBenchmarksToProcess, completedTasksCount, messageCount);
foreach (CloudQueueMessage message in messages)
{
queue.DeleteMessage(message);
}
if (oldCount == results.Count)
{
Thread.Sleep(500);
}
}while (!completed);
await storage.DeleteResultsQueue(experimentId);
var totalRuntime = results.Sum(r => r.NormalizedRuntime);
await storage.SetTotalRuntime(experimentId, totalRuntime);
Console.WriteLine("Collected all results.");
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("ImportTimeline.exe <path-to-data> <storage connection string>");
return;
}
string pathToData = args[0];
string connectionString = args[1];
AzureExperimentStorage storage = null;
try
{
storage = new AzureExperimentStorage(connectionString);
}
catch (Exception ex)
{
Console.WriteLine("Failed to connect to the storage: " + ex.Message);
return;
}
Stopwatch sw = Stopwatch.StartNew();
Console.Write("Reading experiments table from {0}... ", pathToData);
var experiments = PrepareExperiments(pathToData, storage);
Console.WriteLine("{0} experiments found.", experiments.Count);
Console.WriteLine("\nUploading results tables...");
var experimentInfo = UploadResults(pathToData, experiments, storage);
//var experimentInfo = AddStatisticsNoUpload(pathToData, experiments, storage);
Console.Write("\nUploading experiments table from... ");
UploadExperiments(experiments, experimentInfo, storage);
Console.WriteLine("\nUpdating timeline...");
UpdateTimeline(new AzureSummaryManager(connectionString, new PerformanceTest.MEFDomainResolver()),
experiments.Select(e => e.Key).ToArray()).Wait();
sw.Stop();
Console.WriteLine("Done, total time is {0}", sw.Elapsed);
}
static async Task FetchSavedResults(int experimentId, AzureExperimentStorage storage)
{
var results = (await storage.GetAzureExperimentResults(experimentId)).Item1;
goodResults = new List <AzureBenchmarkResult>();
badResults = new List <AzureBenchmarkResult>();
foreach (var r in results)
{
if (r.StdErr.StartsWith(InfrastructureErrorPrefix) || (!string.IsNullOrEmpty(r.StdErrExtStorageIdx) && Utils.StreamToString(storage.ParseAzureBenchmarkResult(r).StdErr, false).StartsWith(InfrastructureErrorPrefix)))
{
badResults.Add(r);
}
else
{
goodResults.Add(r);
}
}
}
public async Task GetResultsFromCloud()
{
ValidatesConnectionString();
AzureExperimentStorage storage = new AzureExperimentStorage(storageConnectionString);
AzureExperimentManager manager = AzureExperimentManager.OpenWithoutStart(storage);
Stopwatch sw1 = Stopwatch.StartNew();
var results = await manager.GetResults(TestExperimentId);
Assert.AreEqual(TestExperimentId, results.ExperimentId);
sw1.Stop();
Trace.WriteLine("1st time: " + sw1.ElapsedMilliseconds);
Assert.AreEqual(103814, results.Benchmarks.Length);
/// Again, should read from local disk:
Stopwatch sw2 = Stopwatch.StartNew();
var results2 = await manager.GetResults(TestExperimentId);
sw2.Stop();
Trace.WriteLine("2nd time: " + sw2.ElapsedMilliseconds);
Assert.AreEqual(103814, results2.Benchmarks.Length);
}
static void Main(string[] args)
{
Keys keys = JsonConvert.DeserializeObject <Keys>(File.ReadAllText("..\\..\\keys.json"));
var storage = new AzureExperimentStorage(keys.storageName, keys.storageKey);
var manager = AzureExperimentManager.Open(storage, keys.batchUri, keys.batchName, keys.batchKey);
//var refExp = new ReferenceExperiment(ExperimentDefinition.Create("referencez3.zip", ExperimentDefinition.DefaultContainerUri, "reference", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), "Z3", null, 2048), 20, 16.34375);
// storage.SaveReferenceExperiment(refExp).Wait();
var id = manager.StartExperiment(ExperimentDefinition.Create("z3.zip", ExperimentDefinition.DefaultContainerUri, "QF_BV", "smt2", "model_validate=true -smt2 -file:{0}", TimeSpan.FromSeconds(1200), TimeSpan.FromSeconds(0), "Z3", "asp", 2048, 1, 0), "Dmitry K").Result;
Console.WriteLine("Experiment id:" + id);
//manager.RestartBenchmarks(159, new string[] {
// "15Puzzle/15-puzzle.init10.smt2",
// "15Puzzle/15-puzzle.init11.smt2",
// "15Puzzle/15-puzzle.init12.smt2",
// "15Puzzle/15-puzzle.init13.smt2",
// "15Puzzle/15-puzzle.init14.smt2",
// "15Puzzle/15-puzzle.init15.smt2",
// "15Puzzle/15-puzzle.init2.smt2",
// "15Puzzle/15-puzzle.init3.smt2",
// "15Puzzle/15-puzzle.init4.smt2",
// "15Puzzle/15-puzzle.init5.smt2",
// "15Puzzle/15-puzzle.init6.smt2",
// "15Puzzle/15-puzzle.init7.smt2",
// "15Puzzle/15-puzzle.init8.smt2",
// "15Puzzle/15-puzzle.init9.smt2",
// "15Puzzle/15puzzle_ins.lp.smt2"
//}).Wait();
Console.WriteLine("Done.");
Console.ReadLine();
}
static async Task ManageRetry(string[] args)
{
int experimentId = int.Parse(args[0], CultureInfo.InvariantCulture);
string benchmarkListBlobId = args[1];
string benchmarkContainerUri = null;
if (args.Length > 2)
{
benchmarkContainerUri = args[2];
}
string jobId = Environment.GetEnvironmentVariable(JobIdEnvVariableName);
var secretStorage = new SecretStorage(Settings.Default.AADApplicationId, Settings.Default.AADApplicationCertThumbprint, Settings.Default.KeyVaultUrl);
BatchConnectionString credentials = new BatchConnectionString(await secretStorage.GetSecret(Settings.Default.ConnectionStringSecretId));
Console.WriteLine("Retrieved credentials.");
var batchCred = new BatchSharedKeyCredentials(credentials.BatchURL, credentials.BatchAccountName, credentials.BatchAccessKey);
var storage = new AzureExperimentStorage(credentials.WithoutBatchData().ToString());
var expInfo = await storage.GetExperiment(experimentId);
if (benchmarkContainerUri == null)
{
if (expInfo.BenchmarkContainerUri != ExperimentDefinition.DefaultContainerUri)
{
throw new ArgumentException("New URI for non-default benchmark container was not provided.");
}
else
{
benchmarkContainerUri = ExperimentDefinition.DefaultContainerUri;
}
}
AzureBenchmarkStorage benchmarkStorage = CreateBenchmarkStorage(benchmarkContainerUri, storage);
var queue = await storage.CreateResultsQueue(experimentId);
Console.Write("Created queue");
// We can't tell bad results we got during previous runs on the same experiment from bad results
// we got during this run when job manager crashed, so we put them all into 'good' list.
// 'Fresh' (and, therefore, duplicate) bad results will be removed during deduplication.
goodResults = (await storage.GetAzureExperimentResults(experimentId)).Item1.ToList();
Console.WriteLine("Fetched existing results");
Domain domain = ResolveDomain(expInfo.DomainName);
string benchmarksPath = CombineBlobPath(expInfo.BenchmarkDirectory, expInfo.Category);
var benchmarkListBlob = storage.TempBlobContainer.GetBlockBlobReference(benchmarkListBlobId);
string[] benchmarkList = (await benchmarkListBlob.DownloadTextAsync()).Split('\n')
.SelectMany(s =>
{
s = s.Trim();
if (string.IsNullOrEmpty(s))
{
return new string[] { }
}
;
else
{
return new string[] { benchmarksPath + s }
};
}).ToArray();
totalBenchmarksToProcess = benchmarkList.Length;
totalBenchmarks = expInfo.TotalBenchmarks;
Console.WriteLine("Retrieved list of benchmarks to re-process. Total: {0}.", totalBenchmarksToProcess);
var collectionTask = CollectResults(experimentId, storage);
Console.WriteLine("Started collection thread.");
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
//not all experiments started
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "id,displayName";
Console.WriteLine("Listing existing tasks.");
var processedBlobs = new SortedSet <string>(batchClient.JobOperations.ListTasks(jobId, detailLevel)
.SelectMany(t =>
{
int id;
if (int.TryParse(t.Id, out id))
{
// we put benchmark file first
return(new string[] { t.DisplayName });
}
return(new string[] { });
}));
Console.WriteLine("Done!");
string outputQueueUri = storage.GetOutputQueueSASUri(experimentId, TimeSpan.FromHours(48));
string outputContainerUri = storage.GetOutputContainerSASUri(TimeSpan.FromHours(48));
string[] blobsToProcess = benchmarkList.Where(b => !processedBlobs.Contains(b)).ToArray();
if (blobsToProcess.Length > 0)
{
var starterTask = StartTasksForSegment(expInfo.BenchmarkTimeout.ToString(), experimentId, expInfo.Executable, expInfo.Parameters, expInfo.MemoryLimitMB, expInfo.DomainName, outputQueueUri, outputContainerUri, null, null, jobId, batchClient, blobsToProcess, benchmarksPath, 0, benchmarkStorage, expInfo.AdaptiveRunMaxRepetitions, expInfo.AdaptiveRunMaxTimeInSeconds);
await starterTask;
Console.WriteLine("Finished starting tasks");
}
MonitorTasksUntilCompletion(experimentId, jobId, collectionTask, batchClient);
}
Console.WriteLine("Deleting blob with benchmark list.");
await benchmarkListBlob.DeleteIfExistsAsync();
Console.WriteLine("Closing.");
}
static async Task Measure(string[] args)
{
int experimentId = int.Parse(args[0], CultureInfo.InvariantCulture);
string benchmarkId = args[1];
string executable = args[2];
string arguments = args[3];
string targetFile = args[4];
TimeSpan timeout = TimeSpan.FromSeconds(double.Parse(args[5], CultureInfo.InvariantCulture));
string domainName = args[6];
Uri outputQueueUri = new Uri(args[7]);
Uri outputBlobContainerUri = new Uri(args[8]);
int maxRepetitions = int.Parse(args[9], CultureInfo.InvariantCulture);
double maxTime = double.Parse(args[10], CultureInfo.InvariantCulture);
double memoryLimit = 0; // no limit
long? outputLimit = null;
long? errorLimit = null;
//if (args.Length > 6)
//{
// workerInfo = args[6];
if (args.Length > 11)
{
memoryLimit = double.Parse(args[11], CultureInfo.InvariantCulture);
if (args.Length > 12)
{
outputLimit = args[12] == "null" ? null : (long?)long.Parse(args[12], CultureInfo.InvariantCulture);
if (args.Length > 13)
{
errorLimit = args[13] == "null" ? null : (long?)long.Parse(args[13], CultureInfo.InvariantCulture);
}
}
}
//}
double normal = 1.0;
string workerDir = Path.Combine(Environment.GetEnvironmentVariable(SharedDirEnvVariableName), Environment.GetEnvironmentVariable(JobIdEnvVariableName));
executable = Path.Combine(workerDir, "exec", executable);
string normalFilePath = Path.Combine(workerDir, PerformanceCoefficientFileName);
if (File.Exists(normalFilePath))
{
normal = double.Parse(File.ReadAllText(normalFilePath), CultureInfo.InvariantCulture);
Trace.WriteLine(string.Format("Normal found within file: {0}", normal));
}
else
{
Trace.WriteLine("Normal not found within file, computing.");
normal = await RunReference(new string[] { });
}
Domain domain = ResolveDomain(domainName);
BenchmarkResult result = LocalExperimentRunner.RunBenchmark(
experimentId,
executable,
arguments,
benchmarkId,
Path.GetFullPath(targetFile),
0,
timeout,
memoryLimit,
outputLimit,
errorLimit,
domain,
normal,
maxRepetitions,
maxTime);
await AzureExperimentStorage.PutResult(experimentId, result, new CloudQueue(outputQueueUri), new CloudBlobContainer(outputBlobContainerUri));
}
static async Task ManageTasks(string[] args)
{
int experimentId = int.Parse(args[0], CultureInfo.InvariantCulture);
string summaryName = null;
if (args.Length > 1)
{
summaryName = args[1];
}
//Console.WriteLine(String.Format("Params are:\n id: {0}\ncontainer: {8}\ndirectory:{9}\ncategory: {1}\nextensions: {10}\ndomain: {11}\nexec: {2}\nargs: {3}\ntimeout: {4}\nmemlimit: {5}\noutlimit: {6}\nerrlimit: {7}", experimentId, benchmarkCategory, executable, arguments, timeout, memoryLimit, outputLimit, errorLimit, benchmarkContainerUri, benchmarkDirectory, extensionsString, domainString));
string jobId = Environment.GetEnvironmentVariable(JobIdEnvVariableName);
var secretStorage = new SecretStorage(Settings.Default.AADApplicationId, Settings.Default.AADApplicationCertThumbprint, Settings.Default.KeyVaultUrl);
BatchConnectionString credentials = new BatchConnectionString(await secretStorage.GetSecret(Settings.Default.ConnectionStringSecretId));
Console.WriteLine("Retrieved credentials.");
var batchCred = new BatchSharedKeyCredentials(credentials.BatchURL, credentials.BatchAccountName, credentials.BatchAccessKey);
var storage = new AzureExperimentStorage(credentials.WithoutBatchData().ToString());
var expInfo = await storage.GetExperiment(experimentId);
string benchmarkContainerUri = expInfo.BenchmarkContainerUri; // args[1];
string benchmarkDirectory = expInfo.BenchmarkDirectory; // args[2];
string benchmarkCategory = expInfo.Category; // args[3];
string extensionsString = expInfo.BenchmarkFileExtension; //args[4];
string domainString = expInfo.DomainName; // args[5];
string executable = expInfo.Executable; // args[6];
string arguments = expInfo.Parameters; // args[7];
double timeout = expInfo.BenchmarkTimeout; // TimeSpan.FromSeconds(double.Parse(args[8]));
double memoryLimit = expInfo.MemoryLimitMB; // 0; // no limit
int maxRepetitions = expInfo.AdaptiveRunMaxRepetitions;
double maxTime = expInfo.AdaptiveRunMaxTimeInSeconds;
//long? outputLimit = null;
//long? errorLimit = null;
//if (args.Length > 9)
//{
// memoryLimit = double.Parse(args[9]);
// if (args.Length > 10)
// {
// outputLimit = args[10] == "null" ? null : (long?)long.Parse(args[10]);
// if (args.Length > 11)
// {
// errorLimit = args[11] == "null" ? null : (long?)long.Parse(args[11]);
// }
// }
//}
AzureBenchmarkStorage benchmarkStorage = CreateBenchmarkStorage(benchmarkContainerUri, storage);
var queue = await storage.CreateResultsQueue(experimentId);
Console.Write("Created queue");
await FetchSavedResults(experimentId, storage);
Console.WriteLine("Fetched existing results");
var collectionTask = CollectResults(experimentId, storage);
Console.WriteLine("Started collection thread.");
Domain domain = ResolveDomain(domainString);
SortedSet <string> extensions;
if (string.IsNullOrEmpty(extensionsString))
{
extensions = new SortedSet <string>(domain.BenchmarkExtensions.Distinct());
}
else
{
extensions = new SortedSet <string>(extensionsString.Split('|').Select(s => s.Trim().TrimStart('.')).Distinct());
}
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
if (expInfo.TotalBenchmarks <= 0)
{
//not all experiments started
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "id,displayName";
Console.WriteLine("Listing existing tasks.");
var processedBlobs = new SortedSet <string>(batchClient.JobOperations.ListTasks(jobId, detailLevel)
.SelectMany(t =>
{
int id;
if (int.TryParse(t.Id, out id))
{
// we put benchmark file first
return(new string[] { t.DisplayName });
}
return(new string[] { });
}));
Console.WriteLine("Done!");
BlobContinuationToken continuationToken = null;
BlobResultSegment resultSegment = null;
List <Task> starterTasks = new List <Task>();
int totalBenchmarks = 0;
string benchmarksPath = CombineBlobPath(benchmarkDirectory, benchmarkCategory);
string outputQueueUri = storage.GetOutputQueueSASUri(experimentId, TimeSpan.FromHours(48));
string outputContainerUri = storage.GetOutputContainerSASUri(TimeSpan.FromHours(48));
do
{
resultSegment = await benchmarkStorage.ListBlobsSegmentedAsync(benchmarksPath, continuationToken);
Console.WriteLine("Got some blobs");
string[] blobNamesToProcess = resultSegment.Results.SelectMany(item =>
{
var blob = item as CloudBlockBlob;
if (blob == null || processedBlobs.Contains(blob.Name))
{
return new string[] { }
}
;
var nameParts = blob.Name.Split('/');
var shortnameParts = nameParts[nameParts.Length - 1].Split('.');
if (shortnameParts.Length == 1 && !extensions.Contains(""))
{
return new string[] { }
}
;
var ext = shortnameParts[shortnameParts.Length - 1];
if (!extensions.Contains(ext))
{
return new string[] { }
}
;
return(new string[] { blob.Name });
}).ToArray();
starterTasks.Add(StartTasksForSegment(timeout.ToString(), experimentId, executable, arguments, memoryLimit, domainString, outputQueueUri, outputContainerUri, null, null, jobId, batchClient, blobNamesToProcess, benchmarksPath, totalBenchmarks, benchmarkStorage, maxRepetitions, maxTime));
continuationToken = resultSegment.ContinuationToken;
totalBenchmarks += blobNamesToProcess.Length;
}while (continuationToken != null);
await storage.SetTotalBenchmarks(experimentId, totalBenchmarks);
Program.totalBenchmarks = totalBenchmarks;
totalBenchmarksToProcess = totalBenchmarks;
await Task.WhenAll(starterTasks.ToArray());
Console.WriteLine("Finished starting tasks");
}
else
{
Program.totalBenchmarks = expInfo.TotalBenchmarks;
totalBenchmarksToProcess = expInfo.TotalBenchmarks;
}
MonitorTasksUntilCompletion(experimentId, jobId, collectionTask, batchClient);
if (summaryName != null)
{
Trace.WriteLine(string.Format("Building summary for experiment {0} and summary name {1}...", experimentId, summaryName));
AzureSummaryManager manager = new AzureSummaryManager(credentials.WithoutBatchData().ToString(), MEFDomainResolver.Instance);
await AppendSummary(summaryName, experimentId, domain, manager);
}
else
{
Trace.WriteLine("No summary requested.");
}
Console.WriteLine("Closing.");
}
}
static ConcurrentDictionary <int, ExperimentEntity> PrepareExperiments(string pathToData, AzureExperimentStorage storage)
{
var experiments = new ConcurrentDictionary <int, ExperimentEntity>(Environment.ProcessorCount, 10000);
Directory.EnumerateFiles(pathToData, "*_meta.csv")
.AsParallel()
.ForAll(file =>
{
var metadata = new MetaData(file);
var exp = new ExperimentEntity((int)metadata.Id);
exp.Submitted = metadata.SubmissionTime;
exp.BenchmarkContainerUri = PerformanceTest.ExperimentDefinition.LocalDiskContainerUri;
exp.BenchmarkDirectory = metadata.BaseDirectory;
exp.DomainName = "Z3";
exp.BenchmarkFileExtension = "smt2";
exp.Category = "smtlib-latest";
exp.Executable = metadata.BinaryId.ToString();
exp.Parameters = metadata.Parameters;
exp.BenchmarkTimeout = metadata.Timeout;
exp.MemoryLimitMB = metadata.Memoryout / 1024.0 / 1024.0;
exp.Flag = false;
exp.Creator = "Imported from Nightly data";
exp.ExperimentTimeout = 0;
exp.GroupName = "";
exp.Note = String.Format("Cluster: {0}, cluster job id: {1}, node group: {2}, locality: {3}, finished: {4}, reference: {5}",
metadata.Cluster, metadata.ClusterJobId, metadata.Nodegroup, metadata.Locality, metadata.isFinished, metadata.Reference);
experiments.TryAdd((int)metadata.Id, exp);
});
return(experiments);
}
private static void UploadExperiments(ConcurrentDictionary <int, ExperimentEntity> experiments, IDictionary <int, TimeSpan> experimentInfo, AzureExperimentStorage storage)
{
storage.ImportExperiments(experiments.Select(e => e.Value)).Wait();
}
static IDictionary <int, TimeSpan> AddStatisticsNoUpload(string pathToData, ConcurrentDictionary <int, ExperimentEntity> experiments, AzureExperimentStorage storage)
{
List <int> missingExperiments = new List <int>();
ConcurrentDictionary <int, TimeSpan> experimentInfo = new ConcurrentDictionary <int, TimeSpan>();
Directory.EnumerateFiles(pathToData, "*.zip")
.AsParallel()
.ForAll(file =>
{
int expId = int.Parse(Path.GetFileNameWithoutExtension(file));
ExperimentEntity e;
if (!experiments.TryGetValue(expId, out e))
{
missingExperiments.Add(expId);
Console.WriteLine("Experiment {0} has results but not metadata");
return;
}
CSVData table = new CSVData(file, (uint)expId);
var totalRunTime = table.Rows.Sum(r => r.Runtime);
e.TotalRuntime = totalRunTime;
e.CompletedBenchmarks = e.TotalBenchmarks = table.Rows.Count;
Console.WriteLine("Done for {0}", expId);
});
if (missingExperiments.Count > 0)
{
Console.WriteLine("\nFollowing experiments have results but not metadata:");
foreach (var item in missingExperiments)
{
Console.WriteLine(item);
}
}
return(experimentInfo);
}
static IDictionary <int, TimeSpan> UploadResults(string pathToData, ConcurrentDictionary <int, ExperimentEntity> experiments, AzureExperimentStorage storage)
{
List <int> missingExperiments = new List <int>();
ConcurrentDictionary <string, string> uploadedOutputs = new ConcurrentDictionary <string, string>();
ConcurrentDictionary <int, TimeSpan> experimentInfo = new ConcurrentDictionary <int, TimeSpan>();
var upload =
Directory.EnumerateFiles(pathToData, "*.zip")
.AsParallel()
.Select(async file =>
{
int expId = int.Parse(Path.GetFileNameWithoutExtension(file));
Console.WriteLine("Uploading experiment {0}...", expId);
ExperimentEntity e;
if (!experiments.TryGetValue(expId, out e))
{
missingExperiments.Add(expId);
Console.WriteLine("Experiment {0} has results but not metadata");
return(0);
}
CSVData table = new CSVData(file, (uint)expId);
var entities =
table.Rows
.OrderBy(r => r.Filename)
.Select(r => PrepareBenchmarkResult(r, storage, uploadedOutputs, expId, e.Submitted))
.ToArray();
var totalRunTime = table.Rows.Sum(r => r.Runtime);
e.TotalRuntime = totalRunTime;
e.CompletedBenchmarks = e.TotalBenchmarks = table.Rows.Count;
try
{
await storage.PutAzureExperimentResults(expId, entities, AzureExperimentStorage.UploadBlobMode.CreateOrReplace);
Console.WriteLine("Done uploading results for {0}.", expId);
}
catch (Exception ex)
{
Console.WriteLine("Failed to upload experiment results of {0}: {1}", expId, ex.ToString());
}
return(0);
});
Task.WhenAll(upload).Wait();
Console.WriteLine("Done (uploaded {0} output & error blobs).", uploadedOutputs.Count);
if (missingExperiments.Count > 0)
{
Console.WriteLine("\nFollowing experiments have results but not metadata:");
foreach (var item in missingExperiments)
{
Console.WriteLine(item);
}
}
return(experimentInfo);
}
static async Task ManageTasks(string[] args)
{
int experimentId = int.Parse(args[0], CultureInfo.InvariantCulture);
string summaryName = null;
if (args.Length > 1)
{
summaryName = args[1];
}
//Console.WriteLine(String.Format("Params are:\n id: {0}\ncontainer: {8}\ndirectory:{9}\ncategory: {1}\nextensions: {10}\ndomain: {11}\nexec: {2}\nargs: {3}\ntimeout: {4}\nmemlimit: {5}\noutlimit: {6}\nerrlimit: {7}", experimentId, benchmarkCategory, executable, arguments, timeout, memoryLimit, outputLimit, errorLimit, benchmarkContainerUri, benchmarkDirectory, extensionsString, domainString));
#if DEBUG
string jobId = "cz3_exp8535";
#else
string jobId = Environment.GetEnvironmentVariable(JobIdEnvVariableName);
#endif
Console.WriteLine("Retrieving credentials...");
var secretStorage = new SecretStorage(Settings.Default.AADApplicationId, Settings.Default.AADApplicationCertThumbprint, Settings.Default.KeyVaultUrl);
BatchConnectionString credentials = new BatchConnectionString(await secretStorage.GetSecret(Settings.Default.ConnectionStringSecretId));
var batchCred = new BatchSharedKeyCredentials(credentials.BatchURL, credentials.BatchAccountName, credentials.BatchAccessKey);
var storage = new AzureExperimentStorage(credentials.WithoutBatchData().ToString());
var expInfo = await storage.GetExperiment(experimentId);
string benchmarkContainerUri = expInfo.BenchmarkContainerUri; // args[1];
string benchmarkDirectory = expInfo.BenchmarkDirectory; // args[2];
string benchmarkCategory = expInfo.Category; // args[3];
string extensionsString = expInfo.BenchmarkFileExtension; //args[4];
string domainString = expInfo.DomainName; // args[5];
string executable = expInfo.Executable; // args[6];
string arguments = expInfo.Parameters; // args[7];
double timeout = expInfo.BenchmarkTimeout; // TimeSpan.FromSeconds(double.Parse(args[8]));
double memoryLimit = expInfo.MemoryLimitMB; // 0; // no limit
int maxRepetitions = expInfo.AdaptiveRunMaxRepetitions;
double maxTime = expInfo.AdaptiveRunMaxTimeInSeconds;
long?outputLimit = 1 * (1024 * 1024); // 1 MB
long?errorLimit = 256 * 1024; // 256 KB
AzureBenchmarkStorage benchmarkStorage = CreateBenchmarkStorage(benchmarkContainerUri, storage);
var queue = await storage.CreateResultsQueue(experimentId);
DateTime before = DateTime.Now;
Console.Write("Fetching existing results...");
await FetchSavedResults(experimentId, storage);
Domain domain = ResolveDomain(domainString);
HashSet <string> extensions;
if (string.IsNullOrEmpty(extensionsString))
{
extensions = new HashSet <string>(domain.BenchmarkExtensions.Distinct());
}
else
{
extensions = new HashSet <string>(extensionsString.Split('|').Select(s => s.Trim().TrimStart('.')).Distinct());
}
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
// Exclude benchmarks that finished correctly
var processedBlobs = new HashSet <string>();
string prefix = (benchmarkDirectory.Trim('/') + "/" + benchmarkCategory.Trim('/')).Trim('/');
foreach (var r in goodResults.Select(g => prefix + "/" + g.BenchmarkFileName))
{
processedBlobs.Add(r.Trim());
}
Console.WriteLine(" took {0}.", (DateTime.Now - before));
// Exclude those that are still in progress
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.FilterClause = "(state eq 'active') or (state eq 'running') or (state eq 'preparing')";
detailLevel.SelectClause = "id,displayName";
CloudJob old_job = null;
try { old_job = batchClient.JobOperations.GetJob(jobId); } catch { /* OK */ }
if (old_job != null)
{
before = DateTime.Now;
Console.Write("Listing existing tasks...");
var ts = batchClient.JobOperations.ListTasks(jobId, detailLevel);
foreach (CloudTask t in ts)
{
int id;
if (int.TryParse(t.Id, out id))
{
string n = t.DisplayName.Trim();
if (!processedBlobs.Contains(n))
{
processedBlobs.Add(n);
}
}
}
;
Console.WriteLine(" took {0}.", (DateTime.Now - before));
// Create new job if the old one is already sealed off
switch (old_job.State)
{
case Microsoft.Azure.Batch.Common.JobState.Completed:
case Microsoft.Azure.Batch.Common.JobState.Deleting:
case Microsoft.Azure.Batch.Common.JobState.Disabled:
case Microsoft.Azure.Batch.Common.JobState.Disabling:
case Microsoft.Azure.Batch.Common.JobState.Terminating:
{
before = DateTime.Now;
Console.Write("Creating fresh job...");
PoolInformation pool_info = old_job.PoolInformation;
string new_jid;
int cnt = 1;
bool have_jid = false;
do
{
new_jid = String.Format("{0}-{1}", jobId, cnt++);
try
{
CloudJob new_job = batchClient.JobOperations.CreateJob(new_jid, pool_info);
new_job.OnAllTasksComplete = Microsoft.Azure.Batch.Common.OnAllTasksComplete.NoAction;
new_job.OnTaskFailure = old_job.OnTaskFailure;
new_job.Constraints = old_job.Constraints;
new_job.DisplayName = old_job.DisplayName;
new_job.Commit();
have_jid = true;
}
catch (Microsoft.Azure.Batch.Common.BatchException)
{
Console.Write(".");
}
}while (!have_jid);
jobId = new_jid;
Console.WriteLine(" took {0}.", (DateTime.Now - before));
break;
}
}
}
BlobContinuationToken continuationToken = null;
BlobResultSegment resultSegment = null;
before = DateTime.Now;
Console.Write("Adding tasks...");
List <Task> starterTasks = new List <Task>();
int benchmarksTotal = processedBlobs.Count();
string benchmarksPath = CombineBlobPath(benchmarkDirectory, benchmarkCategory);
string outputQueueUri = storage.GetOutputQueueSASUri(experimentId, TimeSpan.FromHours(48));
string outputContainerUri = storage.GetOutputContainerSASUri(TimeSpan.FromHours(48));
do
{
resultSegment = await benchmarkStorage.ListBlobsSegmentedAsync(benchmarksPath, continuationToken);
string[] blobNamesToProcess = resultSegment.Results.SelectMany(item =>
{
var blob = item as CloudBlockBlob;
if (blob == null || processedBlobs.Contains(blob.Name))
{
return new string[] { }
}
;
var nameParts = blob.Name.Split('/');
var shortnameParts = nameParts[nameParts.Length - 1].Split('.');
if (shortnameParts.Length == 1 && !extensions.Contains(""))
{
return new string[] { }
}
;
var ext = shortnameParts[shortnameParts.Length - 1];
if (!extensions.Contains(ext))
{
return new string[] { }
}
;
return(new string[] { blob.Name });
}).ToArray();
starterTasks.Add(StartTasksForSegment(timeout.ToString(), experimentId, executable, arguments, memoryLimit, domainString, outputQueueUri, outputContainerUri, outputLimit, errorLimit, jobId, batchClient, blobNamesToProcess, benchmarksPath, benchmarksTotal, benchmarkStorage, maxRepetitions, maxTime));
continuationToken = resultSegment.ContinuationToken;
benchmarksTotal += blobNamesToProcess.Length;
}while (continuationToken != null);
await storage.SetBenchmarksTotal(experimentId, benchmarksTotal);
Program.benchmarksTotal = benchmarksTotal;
benchmarksToProcess = benchmarksTotal - goodResults.Count;
Console.WriteLine(" took {0}.", (DateTime.Now - before));
before = DateTime.Now;
Console.Write("Waiting for tasks to start...");
await Task.WhenAll(starterTasks.ToArray());
Console.WriteLine(" took {0}.", (DateTime.Now - before));
CloudJob j = batchClient.JobOperations.GetJob(jobId);
j.OnAllTasksComplete = Microsoft.Azure.Batch.Common.OnAllTasksComplete.TerminateJob;
j.CommitChanges();
before = DateTime.Now;
Console.Write("Waiting for results...");
var collectionTask = CollectResults(experimentId, storage);
Console.WriteLine(" took {0}.", (DateTime.Now - before));
MonitorTasksUntilCompletion(experimentId, jobId, collectionTask, batchClient, domain);
if (summaryName != null && expInfo.Creator == "Nightly")
{
Trace.WriteLine(string.Format("Building summary for experiment {0} and summary name {1}...", experimentId, summaryName));
AzureSummaryManager manager = new AzureSummaryManager(credentials.WithoutBatchData().ToString(), MEFDomainResolver.Instance);
await AppendSummaryAndSendReport(summaryName, experimentId, domain, manager);
}
else
{
Trace.WriteLine("No summary requested.");
}
try
{
int?amc = storage.GetResultsQueueReference(experimentId).ApproximateMessageCount;
if (amc.HasValue && amc.Value == 0)
{
switch (batchClient.JobOperations.GetJob(jobId).State)
{
case Microsoft.Azure.Batch.Common.JobState.Completed:
case Microsoft.Azure.Batch.Common.JobState.Disabled:
Console.WriteLine("Deleting Batch job and results queue.");
await batchClient.JobOperations.DeleteJobAsync(jobId);
await storage.DeleteResultsQueue(experimentId);
break;
}
}
}
catch { /* OK */ }
Console.WriteLine("Closing.");
}
}
