예제 #1
0
        /// <summary>
        /// Stage a single file.
        /// </summary>
        private async static Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
        {
            StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
            string containerName             = seqArtifacts.BlobContainerCreated;

            // TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
            string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);

            BlobContainerClient blobContainerClient = BlobUtilities.GetBlobContainerClient(containerName, storecreds);
            BlockBlobClient     blobClient          = blobContainerClient.GetBlockBlobClient(blobName);

            bool doesBlobExist = await blobClient.ExistsAsync();

            bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded

            if (doesBlobExist)          // if the blob exists, compare
            {
                FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);

                var properties = await blobClient.GetPropertiesAsync();

                var length = properties.Value.ContentLength;
                // since we don't have a hash of the contents... we check length
                if (length == fi.Length)
                {
                    mustUploadBlob = false;
                }
            }

            if (mustUploadBlob)
            {
                using FileStream stream = new FileStream(stageThisFile.LocalFileToStage, FileMode.Open);

                // upload the file
                Task uploadTask = blobClient.UploadAsync(stream);

                await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
            }

            // get the SAS for the blob
            string blobSAS      = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
            string nodeFileName = stageThisFile.NodeFileName;

            // create a new ResourceFile and populate it.  This file is now staged!
            stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
        }
        /// <summary>
        /// Starts an asynchronous call to stage the given files.
        /// </summary>
        private static async System.Threading.Tasks.Task StageFilesInternalAsync(List <IFileStagingProvider> filesToStage, IFileStagingArtifact fileStagingArtifact)
        {
            if (null == filesToStage)
            {
                throw new ArgumentNullException("filesToStage");
            }

            if (null == fileStagingArtifact)
            {
                throw new ArgumentNullException("filesStagingArtifact");
            }

            SequentialFileStagingArtifact seqArtifact = (SequentialFileStagingArtifact)fileStagingArtifact;

            // is there any work to do?
            if (null == FindAtLeastOne(filesToStage))
            {
                return;  // now work to do.  none of the files belong to this provider
            }

            // is there any work to do
            if ((null == filesToStage) || (filesToStage.Count <= 0))
            {
                return;  // we are done
            }

            // create a Run task to create the blob containers if needed
            System.Threading.Tasks.Task createContainerTask = System.Threading.Tasks.Task.Run(async() => { await CreateDefaultBlobContainerAndSASIfNeededReturnAsync(filesToStage, seqArtifact); });

            // wait for container to be created
            await createContainerTask.ConfigureAwait(continueOnCapturedContext : false);

            // begin staging the files
            System.Threading.Tasks.Task stageTask = StageFilesAsync(filesToStage, seqArtifact);

            // wait for files to be staged
            await stageTask.ConfigureAwait(continueOnCapturedContext : false);
        }
        /// <summary>
        /// Stage a single file.
        /// </summary>
        private async static System.Threading.Tasks.Task StageOneFileAsync(FileToStage stageThisFile, SequentialFileStagingArtifact seqArtifacts)
        {
            StagingStorageAccount storecreds = stageThisFile.StagingStorageAccount;
            string containerName             = seqArtifacts.BlobContainerCreated;

            // TODO: this flattens all files to the top of the compute node/task relative file directory. solve the hiearchy problem (virt dirs?)
            string blobName = Path.GetFileName(stageThisFile.LocalFileToStage);

            // Create the storage account with the connection string.
            CloudStorageAccount storageAccount = new CloudStorageAccount(
                new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storecreds.StorageAccount, storecreds.StorageAccountKey),
                blobEndpoint: storecreds.BlobUri,
                queueEndpoint: null,
                tableEndpoint: null,
                fileEndpoint: null);

            CloudBlobClient    client    = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer container = client.GetContainerReference(containerName);
            ICloudBlob         blob      = container.GetBlockBlobReference(blobName);
            bool doesBlobExist;

            try
            {
                // fetch attributes so we can compare file lengths
                System.Threading.Tasks.Task fetchTask = blob.FetchAttributesAsync();

                await fetchTask.ConfigureAwait(continueOnCapturedContext : false);

                doesBlobExist = true;
            }
            catch (StorageException scex)
            {
                // check to see if blob does not exist
                if ((int)System.Net.HttpStatusCode.NotFound == scex.RequestInformation.HttpStatusCode)
                {
                    doesBlobExist = false;
                }
                else
                {
                    throw;  // unknown exception, throw to caller
                }
            }

            bool mustUploadBlob = true; // we do not re-upload blobs if they have already been uploaded

            if (doesBlobExist)          // if the blob exists, compare
            {
                FileInfo fi = new FileInfo(stageThisFile.LocalFileToStage);

                // since we don't have a hash of the contents... we check length
                if (blob.Properties.Length == fi.Length)
                {
                    mustUploadBlob = false;
                }
            }

            if (mustUploadBlob)
            {
                // upload the file
                System.Threading.Tasks.Task uploadTask = blob.UploadFromFileAsync(stageThisFile.LocalFileToStage);

                await uploadTask.ConfigureAwait(continueOnCapturedContext : false);
            }

            // get the SAS for the blob
            string blobSAS      = ConstructBlobSource(seqArtifacts.DefaultContainerSAS, blobName);
            string nodeFileName = stageThisFile.NodeFileName;

            // create a new ResourceFile and populate it.  This file is now staged!
            stageThisFile.StagedFiles = new ResourceFile[] { ResourceFile.FromUrl(blobSAS, nodeFileName) };
        }
        /// <summary>
        /// Stages all files in the queue
        /// </summary>
        private async static System.Threading.Tasks.Task StageFilesAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifacts)
        {
            foreach (IFileStagingProvider currentFile in filesToStage)
            {
                // for "retry" and/or "double calls" we ignore files that have already been staged
                if (null == currentFile.StagedFiles)
                {
                    FileToStage fts = currentFile as FileToStage;

                    if (null != fts)
                    {
                        System.Threading.Tasks.Task stageTask = StageOneFileAsync(fts, seqArtifacts);

                        await stageTask.ConfigureAwait(continueOnCapturedContext : false);
                    }
                }
            }
        }
        private static async Task CreateDefaultBlobContainerAndSASIfNeededReturnAsync(List <IFileStagingProvider> filesToStage, SequentialFileStagingArtifact seqArtifact)
        {
            if ((null != filesToStage) && (filesToStage.Count > 0))
            {
                // construct the name of the new blob container.
                seqArtifact.BlobContainerCreated = FileStagingLinkedSources.ConstructDefaultName(seqArtifact.NamingFragment).ToLowerInvariant();

                // get any instance for the storage credentials
                FileToStage anyRealInstance = FindAtLeastOne(filesToStage);

                if (null != anyRealInstance)
                {
                    StagingStorageAccount creds = anyRealInstance.StagingStorageAccount;
                    string   policyName         = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + Microsoft.Azure.Batch.FileStaging.Constants.DefaultContainerPolicyFragment;
                    DateTime startTime          = DateTime.UtcNow;
                    DateTime expiredAtTime      = startTime + new TimeSpan(24 /* hrs*/, 0, 0);

                    seqArtifact.DefaultContainerSAS = await CreateContainerWithPolicySASIfNotExistAsync(
                        creds.StorageAccount,
                        creds.StorageAccountKey,
                        creds.BlobUri,
                        seqArtifact.BlobContainerCreated,
                        policyName,
                        startTime,
                        expiredAtTime,
                        SharedAccessBlobPermissions.Read);

                    return;  // done
                }
            }
        }