static IDictionary <int, TimeSpan> UploadResults(string pathToData, ConcurrentDictionary <int, ExperimentEntity> experiments, AzureExperimentStorage storage)
{
List <int> missingExperiments = new List <int>();
ConcurrentDictionary <string, string> uploadedOutputs = new ConcurrentDictionary <string, string>();
ConcurrentDictionary <int, TimeSpan> experimentInfo = new ConcurrentDictionary <int, TimeSpan>();
var upload =
Directory.EnumerateFiles(pathToData, "*.zip")
.AsParallel()
.Select(async file =>
{
int expId = int.Parse(Path.GetFileNameWithoutExtension(file));
Console.WriteLine("Uploading experiment {0}...", expId);
ExperimentEntity e;
if (!experiments.TryGetValue(expId, out e))
{
missingExperiments.Add(expId);
Console.WriteLine("Experiment {0} has results but not metadata");
return(0);
}
CSVData table = new CSVData(file, (uint)expId);
var entities =
table.Rows
.OrderBy(r => r.Filename)
.Select(r => PrepareBenchmarkResult(r, storage, uploadedOutputs, expId, e.Submitted))
.ToArray();
var totalRunTime = table.Rows.Sum(r => r.Runtime);
e.TotalRuntime = totalRunTime;
e.CompletedBenchmarks = e.TotalBenchmarks = table.Rows.Count;
try
{
await storage.PutAzureExperimentResults(expId, entities, AzureExperimentStorage.UploadBlobMode.CreateOrReplace);
Console.WriteLine("Done uploading results for {0}.", expId);
}
catch (Exception ex)
{
Console.WriteLine("Failed to upload experiment results of {0}: {1}", expId, ex.ToString());
}
return(0);
});
Task.WhenAll(upload).Wait();
Console.WriteLine("Done (uploaded {0} output & error blobs).", uploadedOutputs.Count);
if (missingExperiments.Count > 0)
{
Console.WriteLine("\nFollowing experiments have results but not metadata:");
foreach (var item in missingExperiments)
{
Console.WriteLine(item);
}
}
return(experimentInfo);
}
static async Task CollectResults(int experimentId, AzureExperimentStorage storage)
{
Console.WriteLine("Started collection.");
var queue = storage.GetResultsQueueReference(experimentId);
List <AzureBenchmarkResult> results = new List <AzureBenchmarkResult>(); // (await storage.GetAzureExperimentResults(experimentId)).ToList();
int processedBenchmarks = 0; // goodResults.Count + badResults.Count;// results.Count;
var formatter = new BinaryFormatter();
bool completed = false;
do
{
completed = totalBenchmarksToProcess != -1 && completedTasksCount >= totalBenchmarksToProcess;
var messages = queue.GetMessages(32, TimeSpan.FromMinutes(5));
int messageCount = messages.Count();
completed = completed && messageCount == 0;
foreach (CloudQueueMessage message in messages)
{
using (var ms = new MemoryStream(message.AsBytes))
{
goodResults.Add((AzureBenchmarkResult)formatter.Deserialize(ms));
}
}
int oldCount = results.Count;
results = goodResults.Concat(badResults).ToList();
var tuple = SortCountUniqueNamesAndRemoveExactDuplicates(results);
processedBenchmarks = tuple.Item1;
results = tuple.Item2;
await storage.PutAzureExperimentResults(experimentId, results.ToArray(), AzureExperimentStorage.UploadBlobMode.CreateOrReplace);
int completedBenchmarks = totalBenchmarks == -1 ? processedBenchmarks : totalBenchmarks - totalBenchmarksToProcess + completedTasksCount;
await storage.SetCompletedBenchmarks(experimentId, completedBenchmarks);
Console.WriteLine("Setting completed benchmarks to {0}.\nTotal benchmarks: {1}\nProcessed benchmarks: {2}\nTotal to process: {3}\nCompleted tasks: {4}\nMessage count: {5}", completedBenchmarks, totalBenchmarks, processedBenchmarks, totalBenchmarksToProcess, completedTasksCount, messageCount);
foreach (CloudQueueMessage message in messages)
{
queue.DeleteMessage(message);
}
if (oldCount == results.Count)
{
Thread.Sleep(500);
}
}while (!completed);
await storage.DeleteResultsQueue(experimentId);
var totalRuntime = results.Sum(r => r.NormalizedRuntime);
await storage.SetTotalRuntime(experimentId, totalRuntime);
Console.WriteLine("Collected all results.");
}