private static async Task CreateDefaultBlobContainerAndSASIfNeededReturnAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact)
{
if ((null != filesToStage) && (filesToStage.Count > 0))
{
// construct the name of the new blob container.
seqArtifact.BlobContainerCreated = FileStagingLinkedSources.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant();
// get any instance for the storage credentials
FileToStage anyRealInstance = FindAtLeastOne(filesToStage);
if (null != anyRealInstance)
{
StagingStorageAccount creds = anyRealInstance.StagingStorageAccount;
string policyName = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + Microsoft.Azure.Batch.FileStaging.Constants.DefaultContainerPolicyFragment;
DateTime startTime = DateTime.UtcNow;
DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0);
seqArtifact.DefaultContainerSAS = await CreateContainerWithPolicySASIfNotExistAsync(
creds.StorageAccount,
creds.StorageAccountKey,
creds.BlobUri,
seqArtifact.BlobContainerCreated,
policyName,
startTime,
expiredAtTime,
SharedAccessBlobPermissions.Read);
return; // done
}
}
}
public static StagingStorageAccount GetStorageCredentialsFromEnvironment()
{
string storageAccountKey = TestCommon.Configuration.StorageAccountKey;
string storageAccountName = TestCommon.Configuration.StorageAccountName;
string storageAccountBlobEndpoint = TestCommon.Configuration.StorageAccountBlobEndpoint;
StagingStorageAccount storageStagingCredentials = new StagingStorageAccount(storageAccountName, storageAccountKey, storageAccountBlobEndpoint);
return(storageStagingCredentials);
}
/// <summary>
/// Create a container if doesn't exist, setting permission with policy, and return assosciated SAS signature
/// </summary>
/// <param name="account">Storage account</param>
/// <param name="Key">Storage account key</param>
/// <param name="blobUri">Blob endpoint URI</param>
/// <param name="containerName">Name of the container to be created</param>
/// <param name="policy">Name for the policy</param>
/// <param name="start">Start time of the policy</param>
/// <param name="end">Expire time of the policy</param>
/// <param name="permissions">Blob access permissions</param>
/// <returns>the SAS for the container, in full URI format.</returns>.
private static async Task <string> CreateContainerWithPolicySASIfNotExistAsync(string account, string key, Uri blobUri, string containerName, string policy, DateTime start, DateTime end, string permissions)
{
// 1. form the credentail and initial client
StagingStorageAccount stagingCredentials = new StagingStorageAccount(account, key, blobUri.ToString());
StorageSharedKeyCredential shardKeyCredentials = new StorageSharedKeyCredential(account, key);
BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, stagingCredentials);
// 2. create container if it doesn't exist
containerClient.CreateIfNotExists();
// 3. validate policy, create/overwrite if doesn't match
BlobSignedIdentifier identifier = new BlobSignedIdentifier
{
Id = policy,
AccessPolicy = new BlobAccessPolicy
{
Permissions = permissions,
StartsOn = start,
ExpiresOn = end,
},
};
var accessPolicy = (await containerClient.GetAccessPolicyAsync()).Value;
bool policyFound = accessPolicy.SignedIdentifiers.Any(i => i == identifier);
if (policyFound == false)
{
await containerClient.SetAccessPolicyAsync(PublicAccessType.BlobContainer, permissions : new List <BlobSignedIdentifier> {
identifier
});
}
BlobSasBuilder sasBuilder = new BlobSasBuilder
{
BlobContainerName = containerName,
StartsOn = start,
ExpiresOn = end,
};
sasBuilder.SetPermissions(permissions);
BlobUriBuilder builder = new BlobUriBuilder(containerClient.Uri)
{
Sas = sasBuilder.ToSasQueryParameters(shardKeyCredentials)
};
string fullSas = builder.ToString();
return(fullSas);
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
BlobContainerClient blobContainerClient = BlobUtilities.GetBlobContainerClient(containerName, storecreds);
BlockBlobClient blobClient = blobContainerClient.GetBlockBlobClient(blobName);
bool doesBlobExist = await blobClient.ExistsAsync();
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
var properties = await blobClient.GetPropertiesAsync();
var length = properties.Value.ContentLength;
// since we don't have a hash of the contents... we check length
if (length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
using FileStream stream = new FileStream(stageThisFile.LocalFileToStage, FileMode.Open);
// upload the file
Task uploadTask = blobClient.UploadAsync(stream);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
/// <summary>
/// Specifies that a local file should be staged to blob storage.
/// The specified account will be charged for storage costs.
/// </summary>
/// <param name="localFileToStage">The name of the local file.</param>
/// <param name="storageCredentials">The storage credentials to be used when creating the default container.</param>
/// <param name="nodeFileName">Optional name to be given to the file on the compute node. If this parameter is null or missing
/// the name on the compute node will be set to the value of localFileToStage stripped of all path information.</param>
public FileToStage(string localFileToStage, StagingStorageAccount storageCredentials, string nodeFileName = null)
{
this.LocalFileToStage = localFileToStage;
this.StagingStorageAccount = storageCredentials;
if (string.IsNullOrWhiteSpace(this.LocalFileToStage))
{
throw new ArgumentOutOfRangeException("localFileToStage");
}
// map null to base name of local file
if (string.IsNullOrWhiteSpace(nodeFileName))
{
this.NodeFileName = Path.GetFileName(this.LocalFileToStage);
}
else
{
this.NodeFileName = nodeFileName;
}
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}