private static void CreateDefaultBlobContainerAndSASIfNeededReturn(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact) { if ((null != filesToStage) && (filesToStage.Count > 0)) { // construct the name of the new blob container. seqArtifact.BlobContainerCreated = FileStagingNamingHelpers.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant(); // get any instance for the storage credentials FileToStage anyRealInstance = FindAtLeastOne(filesToStage); if (null != anyRealInstance) { StagingStorageAccount creds = anyRealInstance.StagingStorageAccount; string policyName = Batch.Constants.DefaultConveniencePrefix + Constants.DefaultContainerPolicyFragment; DateTime startTime = DateTime.UtcNow; DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0); seqArtifact.DefaultContainerSAS = CreateContainerWithPolicySASIfNotExist( creds.StorageAccount, creds.StorageAccountKey, creds.BlobUri, seqArtifact.BlobContainerCreated, policyName, startTime, expiredAtTime, SharedAccessBlobPermissions.Read); return; // done } } }
/// <summary> /// See <see cref="IFileStagingProvider.StageFilesAsync"/>. /// </summary> /// <param name="filesToStage">The instances of IFileStagingProvider to stage.</param> /// <param name="fileStagingArtifact">IFileStagingProvider specific staging artifacts including error/progress.</param> /// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns> public async System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, IFileStagingArtifact fileStagingArtifact) { System.Threading.Tasks.Task taskForStaticStaging = FileToStage.StageFilesInternalAsync(filesToStage, fileStagingArtifact); await taskForStaticStaging.ConfigureAwait(continueOnCapturedContext : false); return; }
/// <summary> /// Since this is the SequentialFileStagingProvider, all files are supposed to be of this type. /// Find any one and return the implementation instance. /// </summary> /// <param name="filesToStage"></param> /// <returns>Null means there was not even one.</returns> private static FileToStage FindAtLeastOne(List <IFileStagingProvider> filesToStage) { if ((null != filesToStage) && (filesToStage.Count > 0)) { foreach (IFileStagingProvider curProvider in filesToStage) { FileToStage thisIsReal = curProvider as FileToStage; if (null != thisIsReal) { return(thisIsReal); } } } return(null); }
/// <summary> /// Stages all files in the queue /// </summary> private async static System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifacts) { foreach (IFileStagingProvider currentFile in filesToStage) { // for "retry" and/or "double calls" we ignore files that have already been staged if (null == currentFile.StagedFiles) { FileToStage fts = currentFile as FileToStage; if (null != fts) { System.Threading.Tasks.Task stageTask = StageOneFileAsync(fts, seqArtifacts); await stageTask.ConfigureAwait(continueOnCapturedContext : false); } } } }
/// <summary> /// Stage a single file. /// </summary> private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts) { StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount; string containerName = seqArtifacts.BlobContainerCreated; // TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?) string blobName = Path.GetFileName(stageThisFile.LocalFileToStage); // Create the storage account with the connection string. CloudStorageAccount storageAccount = new CloudStorageAccount( new WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey), blobEndpoint: storecreds.BlobUri, queueEndpoint: null, tableEndpoint: null, fileEndpoint: null); CloudBlobClient client = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = client.GetContainerReference(containerName); ICloudBlob blob = container.GetBlockBlobReference(blobName); bool doesBlobExist; try { // fetch attributes so we can compare file lengths System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync(); await fetchTask.ConfigureAwait(continueOnCapturedContext : false); doesBlobExist = true; } catch (StorageException scex) { // check to see if blob does not exist if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode) { doesBlobExist = false; } else { throw; // unknown exception, throw to caller } } bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded if (doesBlobExist) // if the blob exists, compare { FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage); // since we don't have a hash of the contents... we check length if (blob.Properties.Length == fi.Length) { mustUploadBlob = false; } } if (mustUploadBlob) { // upload the file System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage); await uploadTask.ConfigureAwait(continueOnCapturedContext : false); } // get the SAS for the blob string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName); string nodeFileName = stageThisFile.NodeFileName; // create a new ResourceFile and populate it. This file is now staged! stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) }; }