private static void CreateDefaultBlobContainerAndSASIfNeededReturn(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact)
{
if ((null != filesToStage) && (filesToStage.Count > 0))
{
// construct the name of the new blob container.
seqArtifact.BlobContainerCreated = FileStagingNamingHelpers.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant();
// get any instance for the storage credentials
FileToStage anyRealInstance = FindAtLeastOne(filesToStage);
if (null != anyRealInstance)
{
StagingStorageAccount creds = anyRealInstance.StagingStorageAccount;
string policyName = Batch.Constants.DefaultConveniencePrefix + Constants.DefaultContainerPolicyFragment;
DateTime startTime = DateTime.UtcNow;
DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0);
seqArtifact.DefaultContainerSAS = CreateContainerWithPolicySASIfNotExist(
creds.StorageAccount,
creds.StorageAccountKey,
creds.BlobUri,
seqArtifact.BlobContainerCreated,
policyName,
startTime,
expiredAtTime,
SharedAccessBlobPermissions.Read);
return; // done
}
}
}
/// <summary>
/// See <see cref="IFileStagingProvider.StageFilesAsync"/>.
/// </summary>
/// <param name="filesToStage">The instances of IFileStagingProvider to stage.</param>
/// <param name="fileStagingArtifact">IFileStagingProvider specific staging artifacts including error/progress.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
public async System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, IFileStagingArtifact fileStagingArtifact)
{
System.Threading.Tasks.Task taskForStaticStaging = FileToStage.StageFilesInternalAsync(filesToStage, fileStagingArtifact);
await taskForStaticStaging.ConfigureAwait(continueOnCapturedContext : false);
return;
}
/// <summary>
/// Since this is the SequentialFileStagingProvider, all files are supposed to be of this type.
/// Find any one and return the implementation instance.
/// </summary>
/// <param name="filesToStage"></param>
/// <returns>Null means there was not even one.</returns>
private static FileToStage FindAtLeastOne(List <IFileStagingProvider> filesToStage)
{
if ((null != filesToStage) && (filesToStage.Count > 0))
{
foreach (IFileStagingProvider curProvider in filesToStage)
{
FileToStage thisIsReal = curProvider as FileToStage;
if (null != thisIsReal)
{
return(thisIsReal);
}
}
}
return(null);
}
/// <summary>
/// Stages all files in the queue
/// </summary>
private async static System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifacts)
{
foreach (IFileStagingProvider currentFile in filesToStage)
{
// for "retry" and/or "double calls" we ignore files that have already been staged
if (null == currentFile.StagedFiles)
{
FileToStage fts = currentFile as FileToStage;
if (null != fts)
{
System.Threading.Tasks.Task stageTask = StageOneFileAsync(fts, seqArtifacts);
await stageTask.ConfigureAwait(continueOnCapturedContext : false);
}
}
}
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
