/// <summary> /// Constructor. Requires Azure credentials to already be set in ApsimNG.Properties.Settings. /// </summary> /// <param name="id"></param> /// <param name="jobName"></param> /// <param name="path"></param> /// <param name="explorer"></param> /// <param name="export"></param> /// <param name="includeDebugFiles"></param> /// <param name="keepOutputFiles"></param> public AzureResultsDownloader(Guid id, string jobName, string path, AzureJobDisplayPresenter explorer, bool getResults, bool export, bool includeDebugFiles, bool keepOutputFiles, bool unzipResultFiles) { numBlobsComplete = 0; jobId = id; downloadResults = getResults; exportToCsv = export; saveDebugFiles = includeDebugFiles; saveRawOutputFiles = keepOutputFiles; outputPath = path; rawResultsPath = outputPath + "\\" + jobName.ToString() + "_Results"; tempPath = Path.GetTempPath() + "\\" + jobId; progressMutex = new object(); dbMutex = new object(); presenter = explorer; unzipResults = unzipResultFiles; try { // if we need to save files, create a directory under the output directory if ((saveDebugFiles || saveRawOutputFiles || exportToCsv) && !Directory.Exists(rawResultsPath)) { Directory.CreateDirectory(rawResultsPath); } } catch (Exception err) { presenter.ShowError(err); return; } name = jobName; StorageCredentials storageCredentials = StorageCredentials.FromConfiguration(); BatchCredentials batchCredentials = BatchCredentials.FromConfiguration(); storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storageCredentials.Account, storageCredentials.Key), true); var sharedCredentials = new Microsoft.Azure.Batch.Auth.BatchSharedKeyCredentials(batchCredentials.Url, batchCredentials.Account, batchCredentials.Key); batchClient = BatchClient.Open(sharedCredentials); blobClient = storageAccount.CreateCloudBlobClient(); blobClient.DefaultRequestOptions.RetryPolicy = new Microsoft.WindowsAzure.Storage.RetryPolicies.LinearRetry(TimeSpan.FromSeconds(3), 10); }
private void Downloader_DoWork(object sender, DoWorkEventArgs e) { CancellationToken ct; var outputHashLock = new object(); HashSet <string> downloadedOutputs = GetDownloadedOutputFiles(); int success = 0; while (!IsJobComplete()) { Thread.Sleep(10000); } try { if (downloader.CancellationPending || ct.IsCancellationRequested) { return; } // delete temp directory if it already exists and create a new one in its place try { if (Directory.Exists(tempPath)) { Directory.Delete(tempPath, true); } Directory.CreateDirectory(tempPath); } catch (Exception ex) { presenter.ShowError(ex.ToString()); success = 3; } // might be worth including a 'download all' flag (for testing purposes?), where everything in outputs gets downloaded List <CloudBlockBlob> outputs = ListJobOutputsFromStorage().ToList(); if (outputs == null || outputs.Count < 1) { presenter.ShowError("No files in output container."); return; } if (saveDebugFiles) { List <CloudBlockBlob> debugBlobs = outputs.Where(blob => debugFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList(); Download(debugBlobs, rawResultsPath, ref ct); } // Only download the results if the user wants a CSV or the result files themselves if (saveRawOutputFiles || exportToCsv) { List <CloudBlockBlob> zipBlobs = outputs.Where(blob => zipFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList(); if (zipBlobs != null && zipBlobs.Count > 0) // if the result file are nicely zipped up for us { List <string> localZipFiles = Download(zipBlobs, rawResultsPath, ref ct); if (!unzipResults) { // if user doesn't want to extract the results, we're done presenter.DownloadComplete(jobId); presenter.DisplayFinishedDownloadStatus(name, 0, outputPath); return; } foreach (string archive in localZipFiles) { ExtractZipArchive(archive, rawResultsPath); try { File.Delete(archive); } catch { } } } else { // Results are not zipped up (probably because the job was run on the old Azure job manager), so download each individual result file List <CloudBlockBlob> resultBlobs = outputs.Where(blob => resultFileFormats.Contains(Path.GetExtension(blob.Name.ToLower()))).ToList(); Download(resultBlobs, rawResultsPath, ref ct); } if (exportToCsv) { success = SummariseResults(true); } // Delete the output files if the user doesn't want to keep them if (!saveRawOutputFiles) { // this will delete all .db and .out files in the output directory foreach (string resultFile in Directory.EnumerateFiles(rawResultsPath).Where(file => resultFileFormats.Contains(Path.GetExtension(file)))) { try { File.Delete(resultFile); } catch (Exception ex) { presenter.ShowError("Unable to delete " + resultFile + ": " + ex.ToString()); } } } } } catch (AggregateException ae) { presenter.ShowError(ae.InnerException.ToString()); } presenter.DownloadComplete(jobId); presenter.DisplayFinishedDownloadStatus(name, success, outputPath); }