/// <summary> /// Create a container if doesn't exist, setting permission with policy, and return assosciated SAS signature /// </summary> /// <param name="account">Storage account</param> /// <param name="Key">Storage account key</param> /// <param name="blobUri">Blob endpoint URI</param> /// <param name="containerName">Name of the container to be created</param> /// <param name="policy">Name for the policy</param> /// <param name="start">Start time of the policy</param> /// <param name="end">Expire time of the policy</param> /// <param name="permissions">Blob access permissions</param> /// <returns>the SAS for the container, in full URI format.</returns>. private static async Task <string> CreateContainerWithPolicySASIfNotExistAsync(string account, string key, Uri blobUri, string containerName, string policy, DateTime start, DateTime end, string permissions) { // 1. form the credentail and initial client StagingStorageAccount stagingCredentials = new StagingStorageAccount(account, key, blobUri.ToString()); StorageSharedKeyCredential shardKeyCredentials = new StorageSharedKeyCredential(account, key); BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, stagingCredentials); // 2. create container if it doesn't exist containerClient.CreateIfNotExists(); // 3. validate policy, create/overwrite if doesn't match BlobSignedIdentifier identifier = new BlobSignedIdentifier { Id = policy, AccessPolicy = new BlobAccessPolicy { Permissions = permissions, StartsOn = start, ExpiresOn = end, }, }; var accessPolicy = (await containerClient.GetAccessPolicyAsync()).Value; bool policyFound = accessPolicy.SignedIdentifiers.Any(i => i == identifier); if (policyFound == false) { await containerClient.SetAccessPolicyAsync(PublicAccessType.BlobContainer, permissions : new List <BlobSignedIdentifier> { identifier }); } BlobSasBuilder sasBuilder = new BlobSasBuilder { BlobContainerName = containerName, StartsOn = start, ExpiresOn = end, }; sasBuilder.SetPermissions(permissions); BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri) { Sas = sasBuilder.ToSasQueryParameters(shardKeyCredentials) }; string fullSas = builder.ToString(); return(fullSas); }
public static string GetWriteableSasUri(BlobContainerClient containerClient, StagingStorageAccount storageAccount) { var sasBuilder = new BlobSasBuilder { ExpiresOn = DateTime.UtcNow.AddDays(1), BlobContainerName = containerClient.Name, }; sasBuilder.SetPermissions(BlobSasPermissions.Write); StorageSharedKeyCredential credentials = GetSharedKeyCredential(storageAccount); BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri); builder.Sas = sasBuilder.ToSasQueryParameters(credentials); string fullSas = builder.ToString(); return(fullSas); }
private Task <RepositoryResult <(string link, DateTimeOffset validUntil)> > GeSasLinkInternalAsync(string filePath, string container, int validForMinutes, BlobSasPermissions permissions, string friendlyFileName) { // Taken from the docs at // https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-dotnet if (validForMinutes <= 0) { return(Task.FromResult(RepositoryResult <(string, DateTimeOffset)> .Fail("The validity in minutes must be greater than zero"))); } var validUntil = DateTimeOffset.UtcNow.AddMinutes(validForMinutes); var sasBuilder = new BlobSasBuilder() { BlobContainerName = container, BlobName = filePath, Resource = "b", StartsOn = DateTimeOffset.UtcNow, ExpiresOn = validUntil, }; if (!string.IsNullOrWhiteSpace(friendlyFileName)) { sasBuilder.ContentDisposition = new ContentDisposition { DispositionType = "attachment", FileName = friendlyFileName }.ToString(); } else { try { var fileName = Path.GetFileName(filePath); sasBuilder.ContentDisposition = new ContentDisposition { DispositionType = "attachment", FileName = fileName }.ToString(); } catch { /* We're ignoring the case where a non-valid file name was given */ } } sasBuilder.SetPermissions(permissions); var key = new StorageSharedKeyCredential(_blobClient.AccountName, _accessKey); // Construct the full URI, including the SAS token. var blobReference = GetBlobReference(container, filePath); var blobUri = new BlobUriBuilder(blobReference.Uri) { // Use the key to get the SAS token. Sas = sasBuilder.ToSasQueryParameters(key), BlobContainerName = container, BlobName = filePath, } .ToUri(); return(Task.FromResult(RepositoryResult <(string, DateTimeOffset)> .Success((blobUri.ToString(), validUntil)))); }
/// <summary> /// This is where the work of the sourcerer happens /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> private async Task RunAsync(CancellationToken cancellationToken) { int messageCount = 0; // Get a reference to our source blob client and SAS for AzReplicate to use to access it var sourceClient = _blobServiceConnections.SourceClient; var sourceSas = _blobServiceConnections.SourceSas; // Get a reference to our destination blob client(s) and SAS(s) for AzReplicate to use to access it var destinationClients = _blobServiceConnections.DestinationClients; var destinationSas = _blobServiceConnections.DestinationSas; //Ensure the queue exists _queueClient.CreateIfNotExists(cancellationToken: cancellationToken); //iterate over each of the containers in the source account await foreach (var sourceContainer in sourceClient.GetBlobContainersAsync(BlobContainerTraits.Metadata, cancellationToken: cancellationToken)) { // get a container client to talk to the container var sourceContainerClient = sourceClient.GetBlobContainerClient(sourceContainer.Name); // iterate over each of the blobs in the source container await foreach (var sourceBlob in sourceContainerClient.GetBlobsAsync(BlobTraits.Metadata, cancellationToken: cancellationToken)) { // URI pointing to the source file with SAS var sourceUri = new BlobUriBuilder(sourceClient.Uri); sourceUri.BlobContainerName = sourceContainer.Name; sourceUri.BlobName = sourceBlob.Name; sourceUri.Query = sourceSas.ToString(); // URI pointing to the dest file with SAS // If configured with more than one dest account, files will be round robin distributed accross all distination accounts var destinationClientKey = destinationClients.ElementAt(messageCount % destinationClients.Count()).Key; var destUri = new BlobUriBuilder(destinationClients[destinationClientKey].Uri); destUri.BlobContainerName = sourceContainer.Name; destUri.BlobName = sourceBlob.Name; destUri.Query = destinationSas[destinationClientKey].ToString(); //create the message to put in the queue //we use the replicatable class to ensure that the message is in the format that //AzReplicate is expecting var message = new Replicatable { //the file we want to copy, including required SAS signature Source = sourceUri.ToString(), //the place we want AzReplicate to put the file, including required SAS signature Destination = destUri.ToString(), //Anything you pass along in the Diag Info will be available on the completer DiagnosticInfo = null, //Here we are passing along any metadata from the source Metadata = sourceBlob.Metadata.ToDictionary(x => x.Key, x => x.Value) }; //convert the message to Json and put it in the queue var serializedMessage = JsonConvert.SerializeObject(message); await _queueClient.SendMessageAsync(serializedMessage, timeToLive : TimeSpan.FromSeconds(-1), cancellationToken : cancellationToken); messageCount++; //show some progress in the logs if ((messageCount % 100) == 0) { _logger.LogInformation($"Sample AzBlobSourcerer Worker {messageCount} items added to the queue at {DateTimeOffset.UtcNow}"); } } //log that we are all done _logger.LogInformation($"Sample AzBlobSourcerer Worker Done adding items to the queue. Total message count {messageCount} at {DateTimeOffset.UtcNow}"); } }