/// <summary>
/// Stage a single file.
/// </summary>
private async static Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
BlobContainerClient blobContainerClient = BlobUtilities.GetBlobContainerClient(containerName, storecreds);
BlockBlobClient blobClient = blobContainerClient.GetBlockBlobClient(blobName);
bool doesBlobExist = await blobClient.ExistsAsync();
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
var properties = await blobClient.GetPropertiesAsync();
var length = properties.Value.ContentLength;
// since we don't have a hash of the contents... we check length
if (length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
using FileStream stream = new FileStream(stageThisFile.LocalFileToStage, FileMode.Open);
// upload the file
Task uploadTask = blobClient.UploadAsync(stream);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
/// <summary>
/// Starts an asynchronous call to stage the given files.
/// </summary>
private static async System.Threading.Tasks.Task StageFilesInternalAsync(List <IFileStagingProvider> filesToStage, IFileStagingArtifact fileStagingArtifact)
{
if (null == filesToStage)
{
throw new ArgumentNullException("filesToStage");
}
if (null == fileStagingArtifact)
{
throw new ArgumentNullException("filesStagingArtifact");
}
SequentialFileStagingArtifact seqArtifact = (SequentialFileStagingArtifact)fileStagingArtifact;
// is there any work to do?
if (null == FindAtLeastOne(filesToStage))
{
return; // now work to do. none of the files belong to this provider
}
// is there any work to do
if ((null == filesToStage) || (filesToStage.Count <= 0))
{
return; // we are done
}
// create a Run task to create the blob containers if needed
System.Threading.Tasks.Task createContainerTask = System.Threading.Tasks.Task.Run(async() => { await CreateDefaultBlobContainerAndSASIfNeededReturnAsync(filesToStage, seqArtifact); });
// wait for container to be created
await createContainerTask.ConfigureAwait(continueOnCapturedContext : false);
// begin staging the files
System.Threading.Tasks.Task stageTask = StageFilesAsync(filesToStage, seqArtifact);
// wait for files to be staged
await stageTask.ConfigureAwait(continueOnCapturedContext : false);
}
/// <summary>
/// Stage a single file.
/// </summary>
private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
{
StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
string containerName = seqArtifacts.BlobContainerCreated;
// TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);
// Create the storage account with the connection string.
CloudStorageAccount storageAccount = new CloudStorageAccount(
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
blobEndpoint: storecreds.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference(containerName);
ICloudBlob blob = container.GetBlockBlobReference(blobName);
bool doesBlobExist;
try
{
// fetch attributes so we can compare file lengths
System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();
await fetchTask.ConfigureAwait(continueOnCapturedContext : false);
doesBlobExist = true;
}
catch (StorageException scex)
{
// check to see if blob does not exist
if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
{
doesBlobExist = false;
}
else
{
throw; // unknown exception, throw to caller
}
}
bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded
if (doesBlobExist) // if the blob exists, compare
{
FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);
// since we don't have a hash of the contents... we check length
if (blob.Properties.Length == fi.Length)
{
mustUploadBlob = false;
}
}
if (mustUploadBlob)
{
// upload the file
System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);
await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
}
// get the SAS for the blob
string blobSAS = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
string nodeFileName = stageThisFile.NodeFileName;
// create a new ResourceFile and populate it. This file is now staged!
stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
}
/// <summary>
/// Stages all files in the queue
/// </summary>
private async static System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifacts)
{
foreach (IFileStagingProvider currentFile in filesToStage)
{
// for "retry" and/or "double calls" we ignore files that have already been staged
if (null == currentFile.StagedFiles)
{
FileToStage fts = currentFile as FileToStage;
if (null != fts)
{
System.Threading.Tasks.Task stageTask = StageOneFileAsync(fts, seqArtifacts);
await stageTask.ConfigureAwait(continueOnCapturedContext : false);
}
}
}
}
private static async Task CreateDefaultBlobContainerAndSASIfNeededReturnAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact)
{
if ((null != filesToStage) && (filesToStage.Count > 0))
{
// construct the name of the new blob container.
seqArtifact.BlobContainerCreated = FileStagingLinkedSources.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant();
// get any instance for the storage credentials
FileToStage anyRealInstance = FindAtLeastOne(filesToStage);
if (null != anyRealInstance)
{
StagingStorageAccount creds = anyRealInstance.StagingStorageAccount;
string policyName = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + Microsoft.Azure.Batch.FileStaging.Constants.DefaultContainerPolicyFragment;
DateTime startTime = DateTime.UtcNow;
DateTime expiredAtTime = startTime + new TimeSpan(24 /* hrs*/, 0, 0);
seqArtifact.DefaultContainerSAS = await CreateContainerWithPolicySASIfNotExistAsync(
creds.StorageAccount,
creds.StorageAccountKey,
creds.BlobUri,
seqArtifact.BlobContainerCreated,
policyName,
startTime,
expiredAtTime,
SharedAccessBlobPermissions.Read);
return; // done
}
}
}
