private static async Task CreateDefaultBlobContainerAndSASIfNeededReturnAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact) { if ((null != filesToStage) && (filesToStage.Count > 0)) { // construct the name of the new blob container. seqArtifact.BlobContainerCreated = FileStagingLinkedSources.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant(); // get any instance for the storage credentials FileToStage anyRealInstance = FindAtLeastOne(filesToStage); if (null != anyRealInstance) { StagingStorageAccount creds = anyRealInstance.StagingStorageAccount; string policyName = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + Microsoft.Azure.Batch.FileStaging.Constants.DefaultContainerPolicyFragment; DateTime startTime = DateTime.UtcNow; DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0); seqArtifact.DefaultContainerSAS = await CreateContainerWithPolicySASIfNotExistAsync( creds.StorageAccount, creds.StorageAccountKey, creds.BlobUri, seqArtifact.BlobContainerCreated, policyName, startTime, expiredAtTime, SharedAccessBlobPermissions.Read); return; // done } } }
public static StagingStorageAccount GetStorageCredentialsFromEnvironment() { string storageAccountKey = TestCommon.Configuration.StorageAccountKey; string storageAccountName = TestCommon.Configuration.StorageAccountName; string storageAccountBlobEndpoint = TestCommon.Configuration.StorageAccountBlobEndpoint; StagingStorageAccount storageStagingCredentials = new StagingStorageAccount(storageAccountName, storageAccountKey, storageAccountBlobEndpoint); return(storageStagingCredentials); }
/// <summary> /// Create a container if doesn't exist, setting permission with policy, and return assosciated SAS signature /// </summary> /// <param name="account">Storage account</param> /// <param name="Key">Storage account key</param> /// <param name="blobUri">Blob endpoint URI</param> /// <param name="containerName">Name of the container to be created</param> /// <param name="policy">Name for the policy</param> /// <param name="start">Start time of the policy</param> /// <param name="end">Expire time of the policy</param> /// <param name="permissions">Blob access permissions</param> /// <returns>the SAS for the container, in full URI format.</returns>. private static async Task <string> CreateContainerWithPolicySASIfNotExistAsync(string account, string key, Uri blobUri, string containerName, string policy, DateTime start, DateTime end, string permissions) { // 1. form the credentail and initial client StagingStorageAccount stagingCredentials = new StagingStorageAccount(account, key, blobUri.ToString()); StorageSharedKeyCredential shardKeyCredentials = new StorageSharedKeyCredential(account, key); BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, stagingCredentials); // 2. create container if it doesn't exist containerClient.CreateIfNotExists(); // 3. validate policy, create/overwrite if doesn't match BlobSignedIdentifier identifier = new BlobSignedIdentifier { Id = policy, AccessPolicy = new BlobAccessPolicy { Permissions = permissions, StartsOn = start, ExpiresOn = end, }, }; var accessPolicy = (await containerClient.GetAccessPolicyAsync()).Value; bool policyFound = accessPolicy.SignedIdentifiers.Any(i => i == identifier); if (policyFound == false) { await containerClient.SetAccessPolicyAsync(PublicAccessType.BlobContainer, permissions : new List <BlobSignedIdentifier> { identifier }); } BlobSasBuilder sasBuilder = new BlobSasBuilder { BlobContainerName = containerName, StartsOn = start, ExpiresOn = end, }; sasBuilder.SetPermissions(permissions); BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri) { Sas = sasBuilder.ToSasQueryParameters(shardKeyCredentials) }; string fullSas = builder.ToString(); return(fullSas); }
/// <summary> /// Stage a single file. /// </summary> private async static Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts) { StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount; string containerName = seqArtifacts.BlobContainerCreated; // TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?) string blobName = Path.GetFileName(stageThisFile.LocalFileToStage); BlobContainerClient blobContainerClient = BlobUtilities.GetBlobContainerClient(containerName, storecreds); BlockBlobClient blobClient = blobContainerClient.GetBlockBlobClient(blobName); bool doesBlobExist = await blobClient.ExistsAsync(); bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded if (doesBlobExist) // if the blob exists, compare { FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage); var properties = await blobClient.GetPropertiesAsync(); var length = properties.Value.ContentLength; // since we don't have a hash of the contents... we check length if (length == fi.Length) { mustUploadBlob = false; } } if (mustUploadBlob) { using FileStream stream = new FileStream(stageThisFile.LocalFileToStage, FileMode.Open); // upload the file Task uploadTask = blobClient.UploadAsync(stream); await uploadTask.ConfigureAwait(continueOnCapturedContext : false); } // get the SAS for the blob string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName); string nodeFileName = stageThisFile.NodeFileName; // create a new ResourceFile and populate it. This file is now staged! stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) }; }
/// <summary> /// Specifies that a local file should be staged to blob storage. /// The specified account will be charged for storage costs. /// </summary> /// <param name="localFileToStage">The name of the local file.</param> /// <param name="storageCredentials">The storage credentials to be used when creating the default container.</param> /// <param name="nodeFileName">Optional name to be given to the file on the compute node. If this parameter is null or missing /// the name on the compute node will be set to the value of localFileToStage stripped of all path information.</param> public FileToStage(string localFileToStage, StagingStorageAccount storageCredentials, string nodeFileName = null) { this.LocalFileToStage = localFileToStage; this.StagingStorageAccount = storageCredentials; if (string.IsNullOrWhiteSpace(this.LocalFileToStage)) { throw new ArgumentOutOfRangeException("localFileToStage"); } // map null to base name of local file if (string.IsNullOrWhiteSpace(nodeFileName)) { this.NodeFileName = Path.GetFileName(this.LocalFileToStage); } else { this.NodeFileName = nodeFileName; } }
/// <summary> /// Stage a single file. /// </summary> private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts) { StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount; string containerName = seqArtifacts.BlobContainerCreated; // TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?) string blobName = Path.GetFileName(stageThisFile.LocalFileToStage); // Create the storage account with the connection string. CloudStorageAccount storageAccount = new CloudStorageAccount( new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey), blobEndpoint: storecreds.BlobUri, queueEndpoint: null, tableEndpoint: null, fileEndpoint: null); CloudBlobClient client = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = client.GetContainerReference(containerName); ICloudBlob blob = container.GetBlockBlobReference(blobName); bool doesBlobExist; try { // fetch attributes so we can compare file lengths System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync(); await fetchTask.ConfigureAwait(continueOnCapturedContext : false); doesBlobExist = true; } catch (StorageException scex) { // check to see if blob does not exist if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode) { doesBlobExist = false; } else { throw; // unknown exception, throw to caller } } bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded if (doesBlobExist) // if the blob exists, compare { FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage); // since we don't have a hash of the contents... we check length if (blob.Properties.Length == fi.Length) { mustUploadBlob = false; } } if (mustUploadBlob) { // upload the file System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage); await uploadTask.ConfigureAwait(continueOnCapturedContext : false); } // get the SAS for the blob string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName); string nodeFileName = stageThisFile.NodeFileName; // create a new ResourceFile and populate it. This file is now staged! stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) }; }