/// <summary>
/// Constructor. Requires Azure credentials to already be set in ApsimNG.Properties.Settings.
/// </summary>
/// <param name="id"></param>
/// <param name="jobName"></param>
/// <param name="path"></param>
/// <param name="explorer"></param>
/// <param name="export"></param>
/// <param name="includeDebugFiles"></param>
/// <param name="keepOutputFiles"></param>
public AzureResultsDownloader(Guid id, string jobName, string path, AzureJobDisplayPresenter explorer, bool getResults, bool export, bool includeDebugFiles, bool keepOutputFiles, bool unzipResultFiles)
{
numBlobsComplete = 0;
jobId = id;
downloadResults = getResults;
exportToCsv = export;
saveDebugFiles = includeDebugFiles;
saveRawOutputFiles = keepOutputFiles;
outputPath = path;
rawResultsPath = outputPath + "\\" + jobName.ToString() + "_Results";
tempPath = Path.GetTempPath() + "\\" + jobId;
progressMutex = new object();
dbMutex = new object();
presenter = explorer;
unzipResults = unzipResultFiles;
try
{
// if we need to save files, create a directory under the output directory
if ((saveDebugFiles || saveRawOutputFiles || exportToCsv) && !Directory.Exists(rawResultsPath))
{
Directory.CreateDirectory(rawResultsPath);
}
}
catch (Exception err)
{
presenter.ShowError(err);
return;
}
name = jobName;
StorageCredentials storageCredentials = StorageCredentials.FromConfiguration();
BatchCredentials batchCredentials = BatchCredentials.FromConfiguration();
storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storageCredentials.Account, storageCredentials.Key), true);
var sharedCredentials = new Microsoft.Azure.Batch.Auth.BatchSharedKeyCredentials(batchCredentials.Url, batchCredentials.Account, batchCredentials.Key);
batchClient = BatchClient.Open(sharedCredentials);
blobClient = storageAccount.CreateCloudBlobClient();
blobClient.DefaultRequestOptions.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(3), 10);
}
private void Downloader_DoWork(object sender, DoWorkEventArgs e)
{
CancellationToken ct;
var outputHashLock = new object();
HashSet <string> downloadedOutputs = GetDownloadedOutputFiles();
int success = 0;
while (!IsJobComplete())
{
Thread.Sleep(10000);
}
try
{
if (downloader.CancellationPending || ct.IsCancellationRequested)
{
return;
}
// delete temp directory if it already exists and create a new one in its place
try
{
if (Directory.Exists(tempPath))
{
Directory.Delete(tempPath, true);
}
Directory.CreateDirectory(tempPath);
}
catch (Exception ex)
{
presenter.ShowError(ex.ToString());
success = 3;
}
// might be worth including a 'download all' flag (for testing purposes?), where everything in outputs gets downloaded
List <CloudBlockBlob> outputs = ListJobOutputsFromStorage().ToList();
if (outputs == null || outputs.Count < 1)
{
presenter.ShowError("No files in output container.");
return;
}
if (saveDebugFiles)
{
List <CloudBlockBlob> debugBlobs = outputs.Where(blob => debugFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList();
Download(debugBlobs, rawResultsPath, ref ct);
}
// Only download the results if the user wants a CSV or the result files themselves
if (saveRawOutputFiles || exportToCsv)
{
List <CloudBlockBlob> zipBlobs = outputs.Where(blob => zipFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList();
if (zipBlobs != null && zipBlobs.Count > 0) // if the result file are nicely zipped up for us
{
List <string> localZipFiles = Download(zipBlobs, rawResultsPath, ref ct);
if (!unzipResults)
{
// if user doesn't want to extract the results, we're done
presenter.DownloadComplete(jobId);
presenter.DisplayFinishedDownloadStatus(name, 0, outputPath);
return;
}
foreach (string archive in localZipFiles)
{
ExtractZipArchive(archive, rawResultsPath);
try
{
File.Delete(archive);
} catch
{
}
}
}
else
{
// Results are not zipped up (probably because the job was run on the old Azure job manager), so download each individual result file
List <CloudBlockBlob> resultBlobs = outputs.Where(blob => resultFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList();
Download(resultBlobs, rawResultsPath, ref ct);
}
if (exportToCsv)
{
success = SummariseResults(true);
}
// Delete the output files if the user doesn't want to keep them
if (!saveRawOutputFiles)
{
// this will delete all .db and .out files in the output directory
foreach (string resultFile in Directory.EnumerateFiles(rawResultsPath).Where(file => resultFileFormats.Contains(Path.GetExtension(file))))
{
try
{
File.Delete(resultFile);
} catch (Exception ex)
{
presenter.ShowError("Unable to delete " + resultFile + ": " + ex.ToString());
}
}
}
}
}
catch (AggregateException ae)
{
presenter.ShowError(ae.InnerException.ToString());
}
presenter.DownloadComplete(jobId);
presenter.DisplayFinishedDownloadStatus(name, success, outputPath);
}
