/// <summary>
/// Create a container if doesn't exist, setting permission with policy, and return assosciated SAS signature
/// </summary>
/// <param name="account">Storage account</param>
/// <param name="Key">Storage account key</param>
/// <param name="blobUri">Blob endpoint URI</param>
/// <param name="containerName">Name of the container to be created</param>
/// <param name="policy">Name for the policy</param>
/// <param name="start">Start time of the policy</param>
/// <param name="end">Expire time of the policy</param>
/// <param name="permissions">Blob access permissions</param>
/// <returns>the SAS for the container, in full URI format.</returns>.
private static async Task <string> CreateContainerWithPolicySASIfNotExistAsync(string account, string key, Uri blobUri, string containerName, string policy, DateTime start, DateTime end, string permissions)
{
// 1. form the credentail and initial client
StagingStorageAccount stagingCredentials = new StagingStorageAccount(account, key, blobUri.ToString());
StorageSharedKeyCredential shardKeyCredentials = new StorageSharedKeyCredential(account, key);
BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, stagingCredentials);
// 2. create container if it doesn't exist
containerClient.CreateIfNotExists();
// 3. validate policy, create/overwrite if doesn't match
BlobSignedIdentifier identifier = new BlobSignedIdentifier
{
Id = policy,
AccessPolicy = new BlobAccessPolicy
{
Permissions = permissions,
StartsOn = start,
ExpiresOn = end,
},
};
var accessPolicy = (await containerClient.GetAccessPolicyAsync()).Value;
bool policyFound = accessPolicy.SignedIdentifiers.Any(i => i == identifier);
if (policyFound == false)
{
await containerClient.SetAccessPolicyAsync(PublicAccessType.BlobContainer, permissions : new List <BlobSignedIdentifier> {
identifier
});
}
BlobSasBuilder sasBuilder = new BlobSasBuilder
{
BlobContainerName = containerName,
StartsOn = start,
ExpiresOn = end,
};
sasBuilder.SetPermissions(permissions);
BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri)
{
Sas = sasBuilder.ToSasQueryParameters(shardKeyCredentials)
};
string fullSas = builder.ToString();
return(fullSas);
}
public static string GetWriteableSasUri(BlobContainerClient containerClient, StagingStorageAccount storageAccount)
{
var sasBuilder = new BlobSasBuilder
{
ExpiresOn = DateTime.UtcNow.AddDays(1),
BlobContainerName = containerClient.Name,
};
sasBuilder.SetPermissions(BlobSasPermissions.Write);
StorageSharedKeyCredential credentials = GetSharedKeyCredential(storageAccount);
BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri);
builder.Sas = sasBuilder.ToSasQueryParameters(credentials);
string fullSas = builder.ToString();
return(fullSas);
}
private Task <RepositoryResult <(string link, DateTimeOffset validUntil)> > GeSasLinkInternalAsync(string filePath, string container, int validForMinutes, BlobSasPermissions permissions, string friendlyFileName)
{
// Taken from the docs at
// https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-user-delegation-sas-create-dotnet
if (validForMinutes <= 0)
{
return(Task.FromResult(RepositoryResult <(string, DateTimeOffset)> .Fail("The validity in minutes must be greater than zero")));
}
var validUntil = DateTimeOffset.UtcNow.AddMinutes(validForMinutes);
var sasBuilder = new BlobSasBuilder()
{
BlobContainerName = container,
BlobName = filePath,
Resource = "b",
StartsOn = DateTimeOffset.UtcNow,
ExpiresOn = validUntil,
};
if (!string.IsNullOrWhiteSpace(friendlyFileName))
{
sasBuilder.ContentDisposition = new ContentDisposition
{
DispositionType = "attachment",
FileName = friendlyFileName
}.ToString();
}
else
{
try
{
var fileName = Path.GetFileName(filePath);
sasBuilder.ContentDisposition = new ContentDisposition
{
DispositionType = "attachment",
FileName = fileName
}.ToString();
}
catch { /* We're ignoring the case where a non-valid file name was given */ }
}
sasBuilder.SetPermissions(permissions);
var key = new StorageSharedKeyCredential(_blobClient.AccountName, _accessKey);
// Construct the full URI, including the SAS token.
var blobReference = GetBlobReference(container, filePath);
var blobUri = new BlobUriBuilder(blobReference.Uri)
{
// Use the key to get the SAS token.
Sas = sasBuilder.ToSasQueryParameters(key),
BlobContainerName = container,
BlobName = filePath,
}
.ToUri();
return(Task.FromResult(RepositoryResult <(string, DateTimeOffset)> .Success((blobUri.ToString(), validUntil))));
}
/// <summary>
/// This is where the work of the sourcerer happens
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
private async Task RunAsync(CancellationToken cancellationToken)
{
int messageCount = 0;
// Get a reference to our source blob client and SAS for AzReplicate to use to access it
var sourceClient = _blobServiceConnections.SourceClient;
var sourceSas = _blobServiceConnections.SourceSas;
// Get a reference to our destination blob client(s) and SAS(s) for AzReplicate to use to access it
var destinationClients = _blobServiceConnections.DestinationClients;
var destinationSas = _blobServiceConnections.DestinationSas;
//Ensure the queue exists
_queueClient.CreateIfNotExists(cancellationToken: cancellationToken);
//iterate over each of the containers in the source account
await foreach (var sourceContainer in sourceClient.GetBlobContainersAsync(BlobContainerTraits.Metadata, cancellationToken: cancellationToken))
{
// get a container client to talk to the container
var sourceContainerClient = sourceClient.GetBlobContainerClient(sourceContainer.Name);
// iterate over each of the blobs in the source container
await foreach (var sourceBlob in sourceContainerClient.GetBlobsAsync(BlobTraits.Metadata, cancellationToken: cancellationToken))
{
// URI pointing to the source file with SAS
var sourceUri = new BlobUriBuilder(sourceClient.Uri);
sourceUri.BlobContainerName = sourceContainer.Name;
sourceUri.BlobName = sourceBlob.Name;
sourceUri.Query = sourceSas.ToString();
// URI pointing to the dest file with SAS
// If configured with more than one dest account, files will be round robin distributed accross all distination accounts
var destinationClientKey = destinationClients.ElementAt(messageCount % destinationClients.Count()).Key;
var destUri = new BlobUriBuilder(destinationClients[destinationClientKey].Uri);
destUri.BlobContainerName = sourceContainer.Name;
destUri.BlobName = sourceBlob.Name;
destUri.Query = destinationSas[destinationClientKey].ToString();
//create the message to put in the queue
//we use the replicatable class to ensure that the message is in the format that
//AzReplicate is expecting
var message = new Replicatable
{
//the file we want to copy, including required SAS signature
Source = sourceUri.ToString(),
//the place we want AzReplicate to put the file, including required SAS signature
Destination = destUri.ToString(),
//Anything you pass along in the Diag Info will be available on the completer
DiagnosticInfo = null,
//Here we are passing along any metadata from the source
Metadata = sourceBlob.Metadata.ToDictionary(x => x.Key, x => x.Value)
};
//convert the message to Json and put it in the queue
var serializedMessage = JsonConvert.SerializeObject(message);
await _queueClient.SendMessageAsync(serializedMessage, timeToLive : TimeSpan.FromSeconds(-1), cancellationToken : cancellationToken);
messageCount++;
//show some progress in the logs
if ((messageCount % 100) == 0)
{
_logger.LogInformation($"Sample AzBlobSourcerer Worker {messageCount} items added to the queue at {DateTimeOffset.UtcNow}");
}
}
//log that we are all done
_logger.LogInformation($"Sample AzBlobSourcerer Worker Done adding items to the queue. Total message count {messageCount} at {DateTimeOffset.UtcNow}");
}
}