private static void CreateDefaultBlobContainerAndSASIfNeededReturn(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact)
{
if ((null != filesToStage) && (filesToStage.Count > 0))
{
// construct the name of the new blob container.
seqArtifact.BlobContainerCreated = FileStagingNamingHelpers.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant();
// get any instance for the storage credentials
FileToStage anyRealInstance = FindAtLeastOne(filesToStage);
if (null != anyRealInstance)
{
StagingStorageAccount creds = anyRealInstance.StagingStorageAccount;
string policyName = Batch.Constants.DefaultConveniencePrefix + Constants.DefaultContainerPolicyFragment;
DateTime startTime = DateTime.UtcNow;
DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0);
seqArtifact.DefaultContainerSAS = CreateContainerWithPolicySASIfNotExist(
creds.StorageAccount,
creds.StorageAccountKey,
creds.BlobUri,
seqArtifact.BlobContainerCreated,
policyName,
startTime,
expiredAtTime,
SharedAccessBlobPermissions.Read);
return; // done
}
}
}
/// <summary>
/// Specifies that a local file should be staged to blob storage.
/// The specified account will be charged for storage costs.
/// </summary>
/// <param name="localFileToStage">The name of the local file.</param>
/// <param name="storageCredentials">The storage credentials to be used when creating the default container.</param>
/// <param name="nodeFileName">Optional name to be given to the file on the compute node. If this parameter is null or missing
/// the name on the compute node will be set to the value of localFileToStage stripped of all path information.</param>
public FileToStage(string localFileToStage, StagingStorageAccount storageCredentials, string nodeFileName = null)
{
this.LocalFileToStage = localFileToStage;
this.StagingStorageAccount = storageCredentials;
if (string.IsNullOrWhiteSpace(this.LocalFileToStage))
{
throw new ArgumentOutOfRangeException("localFileToStage");
}
// map null to base name of local file
if (string.IsNullOrWhiteSpace(nodeFileName))
{
this.NodeFileName = Path.GetFileName(this.LocalFileToStage);
}
else
{
this.NodeFileName = nodeFileName;
}
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}