コード例 #1
0
 public Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType  = null,
     string subDirectory = "",
     Func <string> getSharedAccessSignature = null)
 {
     SetConnectionLimits(url);
     return(Task.Factory.StartNew(
                () => UploadFileToBlob(
                    cancellationToken,
                    url,
                    localFile,
                    contentType,
                    subDirectory,
                    fileEncryption,
                    client,
                    retryPolicy,
                    getSharedAccessSignature),
                cancellationToken));
 }
コード例 #2
0
 public Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType = null,
     string subDirectory = "",
     Func<string> getSharedAccessSignature = null,
     int parallelTransferThreadCount = 10,
     int numberOfConcurrentTransfers = default(int))
 {
     SetConnectionLimits(url, numberOfConcurrentTransfers);
     return Task.Factory.StartNew(
         () => UploadFileToBlob(
             cancellationToken,
             url,
             localFile,
             contentType,
             subDirectory,
             fileEncryption,
             client,
             retryPolicy,
             getSharedAccessSignature,
             parallelTransferThreadCount),
             cancellationToken);
 }
コード例 #3
0
        /// <summary>
        /// Downloads the specified blob to the specified location.
        /// </summary>
        /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param>
        /// <param name="localFile">The full path where file will be saved.</param>
        /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param>
        /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
        /// <param name="client">The azure client to access a blob.</param>
        /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <returns>A task that downloads the specified blob.</returns>
        public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
        {
            SetMaxConnectionLimit(uri);
            Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy));

            return(task);
        }
コード例 #4
0
        /// <summary>
        /// Uploads the destinationPath with given path asynchronously
        /// </summary>
        /// <param name="path">The path of a destinationPath to upload</param>
        /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to upload files.</param>
        /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param>
        /// <param name="token">A <see cref="CancellationToken"/> to use for canceling upload operation.</param>
        /// <returns>A function delegate that returns the future result to be available through the Task.</returns>
        public Task UploadAsync(string path, BlobTransferClient blobTransferClient, ILocator locator, CancellationToken token)
        {
            if (blobTransferClient == null)
            {
                throw new ArgumentNullException("blobTransferClient");
            }

            if (locator == null)
            {
                throw new ArgumentNullException("locator");
            }

            if (path == null)
            {
                throw new ArgumentNullException("path");
            }

            if (!File.Exists(path))
            {
                throw new FileNotFoundException(path);
            }

            ValidateFileName(path);

            IContentKey          contentKeyData       = null;
            FileEncryption       fileEncryption       = null;
            AssetCreationOptions assetCreationOptions = this.Asset.Options;

            if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
            {
                contentKeyData = this.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault();
                if (contentKeyData == null)
                {
                    throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, this.Asset.Id));
                }

                fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id));
            }

            EventHandler <BlobTransferProgressChangedEventArgs> handler = (s, e) => OnUploadProgressChanged(path, e);

            blobTransferClient.TransferProgressChanged += handler;

            MediaRetryPolicy retryPolicy = this.GetMediaContext().MediaServicesClassFactory.GetBlobStorageClientRetryPolicy();

            return(blobTransferClient.UploadBlob(
                       new Uri(locator.BaseUri),
                       path,
                       null,
                       fileEncryption,
                       token,
                       retryPolicy.AsAzureStorageClientRetryPolicy(),
                       () => locator.ContentAccessComponent)
                   .ContinueWith(
                       ts =>
            {
                blobTransferClient.TransferProgressChanged -= handler;
                this.PostUploadAction(ts, path, fileEncryption, assetCreationOptions, token);
            }));
        }
コード例 #5
0
        public Task DownloadBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func<string> getSharedAccessSignature = null,
            long start = 0,
            long length = -1,
            int parallelTransferThreadCount = 10,
            int numberOfConcurrentTransfers = 2)
        {
            if (client != null && getSharedAccessSignature != null)
            {
                throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null");
            }

            SetConnectionLimits(uri, Environment.ProcessorCount * numberOfConcurrentTransfers * parallelTransferThreadCount);

            Task task =
                Task.Factory.StartNew(
                    () =>
                        DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client,
                            cancellationToken, retryPolicy, getSharedAccessSignature, start: start, length: length,
                            parallelTransferThreadCount: parallelTransferThreadCount));
            return task;
        }
コード例 #6
0
 /// <summary>
 /// Initializes this instance.
 /// </summary>
 public void Init()
 {
     if (!this.HasFileEncryption)
     {
         this._fileEncryption = new FileEncryption();
     }
 }
コード例 #7
0
 public Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType = null,
     string subDirectory = "",
     Func<string> getSharedAccessSignature = null)
 {
     SetConnectionLimits(url);
     return Task.Factory.StartNew(
         () => UploadFileToBlob(
             cancellationToken,
             url,
             localFile,
             contentType,
             subDirectory,
             fileEncryption,
             client,
             retryPolicy,
             getSharedAccessSignature),
         cancellationToken);
 }
コード例 #8
0
 /// <summary>
 /// Initializes this instance.
 /// </summary>
 public void Init()
 {
     if (!this.HasFileEncryption)
     {
         this._fileEncryption = new FileEncryption();
     }
 }
コード例 #9
0
        public Task DownloadBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature = null,
            long start  = 0,
            long length = -1,
            int parallelTransferThreadCount = 10,
            int numberOfConcurrentTransfers = default(int))
        {
            if (client != null && getSharedAccessSignature != null)
            {
                throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null");
            }

            SetConnectionLimits(uri, numberOfConcurrentTransfers);

            Task task =
                Task.Factory.StartNew(
                    () =>
                    DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client,
                                         cancellationToken, retryPolicy, getSharedAccessSignature, start: start, length: length,
                                         parallelTransferThreadCount: parallelTransferThreadCount));

            return(task);
        }
コード例 #10
0
 public Task UploadBlob(
     Uri url,
     string name,
     Stream stream,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType  = null,
     string subDirectory = "",
     Func <string> getSharedAccessSignature = null,
     int parallelTransferThreadCount        = 10,
     int numberOfConcurrentTransfers        = default(int))
 {
     SetConnectionLimits(url, numberOfConcurrentTransfers);
     return(Task.Factory.StartNew(
                () =>
                UploadFileToBlob(
                    cancellationToken,
                    url,
                    name,
                    stream,
                    contentType,
                    subDirectory,
                    fileEncryption,
                    client,
                    retryPolicy,
                    getSharedAccessSignature,
                    parallelTransferThreadCount),
                cancellationToken));
 }
コード例 #11
0
        /// <summary>
        /// Downloads the file asynchronously .
        /// </summary>
        /// <param name="destinationPath">The path to download the file to.</param>
        /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to download files.</param>
        /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param>
        /// <param name="retryPolicy">The retry policy.</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <returns>A function delegate that returns the future result to be available through the Task.</returns>
        internal Task DownloadToFileAsync(string destinationPath, BlobTransferClient blobTransferClient, ILocator locator, IRetryPolicy retryPolicy, CancellationToken cancellationToken)
        {
            FileEncryption fileEncryption = this.GetFileEncryption();

            cancellationToken.Register(() => this.Cleanup(null, fileEncryption, null, null));
            return(Task.Factory.StartNew(() =>
            {
                cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null));
                ulong iv = Convert.ToUInt64(this.InitializationVector, CultureInfo.InvariantCulture);
                UriBuilder uriBuilder = new UriBuilder(locator.Path);
                uriBuilder.Path += String.Concat("/", Name);
                blobTransferClient.TransferProgressChanged += this.OnDownloadBlobTransferProgressChanged;
                blobTransferClient.DownloadBlob(uriBuilder.Uri, destinationPath, fileEncryption, iv, cancellationToken, retryPolicy).Wait(cancellationToken);
                cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null));
            },
                                         cancellationToken)
                   .ContinueWith(
                       t =>
            {
                t.ThrowIfFaulted(() => this.Cleanup(null, fileEncryption, null, null));
                cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null));
                this.Cleanup(null, fileEncryption, null, null);
            },
                       cancellationToken));
        }
コード例 #12
0
 private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response,
                                 KeyValuePair <long, int> blockOffsetAndLength, ref long bytesDownloaded)
 {
     using (Stream stream = response.GetResponseStream())
     {
         int offsetInChunk = 0;
         int remaining     = blockOffsetAndLength.Value;
         while (remaining > 0)
         {
             int read = stream.Read(buffer, offsetInChunk, remaining);
             lock (lockobject)
             {
                 fs.Position = blockOffsetAndLength.Key + offsetInChunk;
                 if (fileEncryption != null)
                 {
                     lock (fileEncryption)
                     {
                         using (
                             FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk)
                             )
                         {
                             encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk);
                         }
                     }
                 }
                 fs.Write(buffer, offsetInChunk, read);
             }
             offsetInChunk += read;
             remaining     -= read;
             Interlocked.Add(ref bytesDownloaded, read);
         }
     }
     return(bytesDownloaded);
 }
コード例 #13
0
        /// <summary>
        /// Uploads file to a blob storage.
        /// </summary>
        /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
        /// <param name="localFile">The full path of local file.</param>
        /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
        /// then client can be null</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="contentType">Content type of the blob</param>
        /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <returns></returns>
        public virtual Task UploadBlob(
            Uri url,
            string localFile,
            FileEncryption fileEncryption,
            CancellationToken cancellationToken,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            string contentType  = null,
            string subDirectory = "",
            Func <string> getSharedAccessSignature = null
            )
        {
            var fs = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);

            return(UploadBlob(
                       url,
                       localFile,
                       fs,
                       fileEncryption,
                       cancellationToken,
                       client,
                       retryPolicy,
                       contentType,
                       subDirectory,
                       getSharedAccessSignature)
                   .ContinueWith(t => fs.Dispose()));
        }
コード例 #14
0
        /// <summary>
        /// Adds the encryption metadata to the file info.
        /// </summary>
        /// <param name="file">The file information to update.</param>
        /// <param name="fileEncryption">The file encryption to use.</param>
        internal static void AddEncryptionMetadataToAssetFile(AssetFileData file, FileEncryption fileEncryption)
        {
            ulong iv = fileEncryption.GetInitializationVectorForFile(file.Name);

            file.IsEncrypted          = true;
            file.EncryptionKeyId      = fileEncryption.GetKeyIdentifierAsString();
            file.EncryptionScheme     = FileEncryption.SchemeName;
            file.EncryptionVersion    = FileEncryption.SchemeVersion;
            file.InitializationVector = iv.ToString(CultureInfo.InvariantCulture);
        }
コード例 #15
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns></returns>
 public Task UploadBlob(
     Uri url,
     string localFile,
     string contentType,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     IRetryPolicy retryPolicy)
 {
     return UploadBlob(url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType);
 }
コード例 #16
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns></returns>
 public Task UploadBlob(
     Uri url,
     string localFile,
     string contentType,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     IRetryPolicy retryPolicy)
 {
     return(UploadBlob(url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType));
 }
コード例 #17
0
		/// <summary>
		/// Uploads file to a blob storage.
		/// </summary>
		/// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
		/// <param name="localFile">The full path of local file.</param>
		/// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
		/// <param name="cancellationToken">The cancellation token.</param>
		/// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
		/// then client can be null</param>
		/// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
		/// <param name="contentType">Content type of the blob</param>
		/// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
		/// <returns></returns>
		public virtual Task UploadBlob(
			Uri url, 
			string localFile, 
			FileEncryption fileEncryption, 
			CancellationToken cancellationToken, 
			CloudBlobClient client, 
			IRetryPolicy retryPolicy,
			string contentType = null,
			string subDirectory = "",
			Func<string> getSharedAccessSignature = null
            )
		{

			if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
			{
				// The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
				Thread.Sleep(_forceSharedAccessSignatureRetry);
			}

			BlobUploader blobuploader = new BlobUploader(new MemoryManagerFactory());

			blobuploader.TransferCompleted += (sender, args) =>
			{
				if (TransferCompleted != null)
				{
					TransferCompleted(sender, args);
				}
				else if (args.Error != null)
				{
					throw args.Error;
				}
			};

			blobuploader.TransferProgressChanged += (sender, args) =>
			{
				if (TransferProgressChanged != null)
				{
					TransferProgressChanged(sender, args);
				}
			};

		    return blobuploader.UploadBlob(
		        url,
		        localFile,
		        fileEncryption,
		        cancellationToken,
		        client,
		        retryPolicy,
		        contentType,
		        subDirectory,
		        getSharedAccessSignature: getSharedAccessSignature,
		        parallelTransferThreadCount: ParallelTransferThreadCount,
		        numberOfConcurrentTransfers: NumberOfConcurrentTransfers);
		}
コード例 #18
0
 /// <summary>
 /// Downloads the specified blob to the specified location.
 /// </summary>
 /// <param name="uri">The blob url  from which file  needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path where file will be saved </param>
 /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param>
 /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns></returns>
 public virtual Task DownloadBlob(
     Uri uri,
     string localFile,
     FileEncryption fileEncryption,
     ulong initializationVector,
     CancellationToken cancellationToken,
     IRetryPolicy retryPolicy,
     Func <string> getSharedAccessSignature = null)
 {
     return(DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy, getSharedAccessSignature));
 }
コード例 #19
0
        /// <summary>
        /// Downloads the specified blob to the specified location.
        /// </summary>
        /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param>
        /// <param name="localFile">The full path where file will be saved.</param>
        /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param>
        /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
        /// <param name="client">The azure client to access a blob.</param>
        /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <returns>A task that downloads the specified blob.</returns>
        public Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature = null)
        {
            if (client != null && getSharedAccessSignature != null)
            {
                throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null");
            }

            SetMaxConnectionLimit(uri);
            Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature));

            return(task);
        }
コード例 #20
0
 protected void ApplyEncryptionTransform(FileEncryption fileEncryption, string fileName, long beginFilePosition, byte[] buffer, int bytesToWrite)
 {
     if (fileEncryption != null)
     {
         lock (fileEncryption)
         {
             using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(fileName, beginFilePosition))
             {
                 encryptor.TransformBlock(inputBuffer: buffer, inputOffset: 0, inputCount: bytesToWrite, outputBuffer: buffer, outputOffset: 0);
             }
         }
     }
 }
コード例 #21
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
 /// then client can be null</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
 /// <returns></returns>
 public virtual Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType  = null,
     string subDirectory = "")
 {
     SetMaxConnectionLimit(url);
     return(Task.Factory.StartNew(
                () => UploadFileToBlob(cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy),
                cancellationToken));
 }
        /// <summary>
        /// Creates the storage content key.
        /// </summary>
        /// <param name="fileEncryption">The file encryption.</param>
        /// <param name="cert">The cert.</param>
        /// <returns>The content key.</returns>
        internal static ContentKeyData CreateStorageContentKey(FileEncryption fileEncryption, X509Certificate2 cert)
        {
            byte[] encryptedContentKey = fileEncryption.EncryptContentKeyToCertificate(cert);

            ContentKeyData contentKeyData = new ContentKeyData
            {
                Id = fileEncryption.GetKeyIdentifierAsString(),
                EncryptedContentKey = Convert.ToBase64String(encryptedContentKey),
                ContentKeyType      = (int)ContentKeyType.StorageEncryption,
                ProtectionKeyId     = cert.Thumbprint,
                ProtectionKeyType   = (int)ProtectionKeyType.X509CertificateThumbprint,
                Checksum            = fileEncryption.GetChecksum()
            };

            return(contentKeyData);
        }
コード例 #23
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded. If blob has private write permissions then
 /// appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
 /// <returns></returns>
 public virtual Task UploadBlob(
     Uri url,
     string localFile,
     string contentType,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     IRetryPolicy retryPolicy,
     Func <string> getSharedAccessSignature = null)
 {
     return(UploadBlob(
                url,
                localFile,
                fileEncryption,
                cancellationToken,
                null,
                retryPolicy,
                contentType,
                getSharedAccessSignature: getSharedAccessSignature));
 }
コード例 #24
0
        public Task DownloadBlob(
            Uri uri, 
            string localFile, 
            FileEncryption fileEncryption, 
            ulong initializationVector, 
            CloudBlobClient client, 
            CancellationToken cancellationToken, 
            IRetryPolicy retryPolicy, 
            Func<string> getSharedAccessSignature = null,
            long start = 0,
            long length = -1)
        {
            if (client != null && getSharedAccessSignature != null)
            {
                throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null");
            }

            SetConnectionLimits(uri);

            Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature, start:start, length:length));
            return task;
        }
コード例 #25
0
        private void PostUploadAction(Task task, string path, FileEncryption fileEncryption, AssetCreationOptions assetCreationOptions, CancellationToken token)
        {
            try
            {
                task.ThrowIfFaulted();
                token.ThrowIfCancellationRequested();

                FileInfo fileInfo = new FileInfo(path);

                //Updating Name based on file name to avoid exceptions trying to download file.Mapping to storage account is through file name
                this.Name = fileInfo.Name;

                // Set the ContentFileSize base on the local file size
                this.ContentFileSize = fileInfo.Length;

                // Update the files associated with the asset with the encryption related metadata.
                if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
                {
                    AssetBaseCollection.AddEncryptionMetadataToAssetFile(this, fileEncryption);
                }
                else if (assetCreationOptions.HasFlag(AssetCreationOptions.CommonEncryptionProtected))
                {
                    AssetBaseCollection.SetAssetFileForCommonEncryption(this);
                }
                else if (assetCreationOptions.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected))
                {
                    AssetBaseCollection.SetAssetFileForEnvelopeEncryption(this);
                }
                this.Update();
            }
            finally
            {
                if (fileEncryption != null)
                {
                    fileEncryption.Dispose();
                }
            }
        }
        private static void SetEncryptionSettings(IIngestManifestAsset ingestManifestAsset, AssetCreationOptions options, IngestManifestFileData data)
        {
            if (options.HasFlag(AssetCreationOptions.StorageEncrypted))
            {
                var contentKeyData = ingestManifestAsset.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault();
                if (contentKeyData == null)
                {
                    throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, ingestManifestAsset.Asset.Id));
                }
                using (var fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id)))
                {
                    if (!fileEncryption.IsInitializationVectorPresent(data.Name))
                    {
                        fileEncryption.CreateInitializationVectorForFile(data.Name);
                    }
                    ulong iv = fileEncryption.GetInitializationVectorForFile(data.Name);

                    data.IsEncrypted          = true;
                    data.EncryptionKeyId      = fileEncryption.GetKeyIdentifierAsString();
                    data.EncryptionScheme     = FileEncryption.SchemeName;
                    data.EncryptionVersion    = FileEncryption.SchemeVersion;
                    data.InitializationVector = iv.ToString(CultureInfo.InvariantCulture);
                }
            }
            else if (options.HasFlag(AssetCreationOptions.CommonEncryptionProtected))
            {
                data.IsEncrypted       = true;
                data.EncryptionScheme  = CommonEncryption.SchemeName;
                data.EncryptionVersion = CommonEncryption.SchemeVersion;
            }
            else if (options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected))
            {
                data.IsEncrypted       = true;
                data.EncryptionScheme  = EnvelopeEncryption.SchemeName;
                data.EncryptionVersion = EnvelopeEncryption.SchemeVersion;
            }
        }
コード例 #27
0
        public override Task<IAssetFile> CreateAsync(string name, CancellationToken cancelation)
        {
            if (_parentAsset == null)
            {
                throw new InvalidOperationException(StringTable.AssetFileCreateParentAssetIsNull);
            }

            if (string.IsNullOrWhiteSpace(name))
            {
                throw new ArgumentException(String.Format(CultureInfo.InvariantCulture, StringTable.ErrorCreatingAssetFileEmptyFileName));
            }
            cancelation.ThrowIfCancellationRequested();
            IMediaDataServiceContext dataContext = null;
            AssetFileData assetFile = null;
            return Task.Factory.StartNew(() =>
            {
                cancelation.ThrowIfCancellationRequested();
                dataContext = MediaContext.MediaServicesClassFactory.CreateDataServiceContext();

                FileEncryption fileEncryption = null;

                // Set a MIME type based on the extension of the file name
                string mimeType = AssetFileData.GetMimeType(name);
                assetFile = new AssetFileData
                {
                    Name = name,
                    ParentAssetId = _parentAsset.Id,
                    MimeType = mimeType,
                };
                try
                {
                    // Update the files associated with the asset with the encryption related metadata.
                    if (_parentAsset.Options.HasFlag(AssetCreationOptions.StorageEncrypted))
                    {
                        IContentKey storageEncryptionKey = _parentAsset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault();
                        cancelation.ThrowIfCancellationRequested();
                        if (storageEncryptionKey == null)
                        {
                            throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, _parentAsset.Id));
                        }
                        fileEncryption = new FileEncryption(storageEncryptionKey.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(storageEncryptionKey.Id));
                        AssetBaseCollection.AddEncryptionMetadataToAssetFile(assetFile, fileEncryption);
                    }
                    else if (_parentAsset.Options.HasFlag(AssetCreationOptions.CommonEncryptionProtected))
                    {
                        AssetBaseCollection.SetAssetFileForCommonEncryption(assetFile);
                    }
                    else if (_parentAsset.Options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected))
                    {
                        AssetBaseCollection.SetAssetFileForEnvelopeEncryption(assetFile);
                    }
                }
                finally
                {
                    if (fileEncryption != null)
                    {
                        fileEncryption.Dispose();
                    }
                }
                dataContext.AddObject(FileSet, assetFile);
                cancelation.ThrowIfCancellationRequested();
                cancelation.ThrowIfCancellationRequested();
                MediaRetryPolicy retryPolicy = this.MediaContext.MediaServicesClassFactory.GetSaveChangesRetryPolicy(dataContext as IRetryPolicyAdapter);
                return retryPolicy.ExecuteAsync<IMediaDataServiceResponse>(() =>
                {
                    cancelation.ThrowIfCancellationRequested();
                    return dataContext.SaveChangesAsync(assetFile);

                }, cancelation).Result;
            }, cancelation)
            .ContinueWith<IAssetFile>(t =>
            {

                t.ThrowIfFaulted();
                AssetFileData data = (AssetFileData)t.Result.AsyncState;
                return data;
            }, cancelation);
        }
コード例 #28
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
 /// then client can be null</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
 /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
 /// <returns></returns>
 public virtual Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType = null,
     string subDirectory = "",
     Func<string> getSharedAccessSignature = null
     )
 {
     var fs = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
     return UploadBlob(
             url,
             localFile,
             fs,
             fileEncryption,
             cancellationToken,
             client,
             retryPolicy,
             contentType,
             subDirectory,
             getSharedAccessSignature)
         .ContinueWith(t => fs.Dispose());
 }
コード例 #29
0
        /// <summary>
        /// Uploads the destinationPath with given path asynchronously
        /// </summary>
        /// <param name="path">The path of a destinationPath to upload</param>
        /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to upload files.</param>
        /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param>
        /// <param name="token">A <see cref="CancellationToken"/> to use for canceling upload operation.</param>
        /// <returns>A function delegate that returns the future result to be available through the Task.</returns>
        public Task UploadAsync(string path, BlobTransferClient blobTransferClient, ILocator locator, CancellationToken token)
        {
            if (blobTransferClient == null)
            {
                throw new ArgumentNullException("blobTransferClient");
            }

            if (locator == null)
            {
                throw new ArgumentNullException("locator");
            }

            if (path == null)
            {
                throw new ArgumentNullException("path");
            }

            if (!File.Exists(path))
            {
                throw new FileNotFoundException(path);
            }

            ValidateFileName(path);

            IContentKey contentKeyData = null;
            FileEncryption fileEncryption = null;
            AssetCreationOptions assetCreationOptions = this.Asset.Options;
            if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
            {
                contentKeyData = this.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault();
                if (contentKeyData == null)
                {
                    throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, this.Asset.Id));
                }

                fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id));
            }

            EventHandler<BlobTransferProgressChangedEventArgs> handler = (s, e) => OnUploadProgressChanged(path, e);

            blobTransferClient.TransferProgressChanged += handler;

            return blobTransferClient.UploadBlob(
                new Uri(locator.Path),
                path,
                null,
                fileEncryption,
                token,
                AzureStorageClientRetryPolicyFactory.DefaultPolicy.AsAzureStorageClientRetryPolicy())
                .ContinueWith(
                ts=>
                {
                    blobTransferClient.TransferProgressChanged -= handler;
                    this.PostUploadAction(ts, path, fileEncryption, assetCreationOptions, token);
                });
        }
コード例 #30
0
        private void Cleanup(BlobTransferClient blobTransfer, FileEncryption fileEncryption, ILocator locator, IAccessPolicy accessPolicy)
        {
            lock (this._lock)
            {
                if (blobTransfer != null)
                {
                    try
                    {
                        blobTransfer.TransferProgressChanged -= this.OnDownloadBlobTransferProgressChanged;
                    }
                    catch
                    {
                    }
                    finally
                    {
                        blobTransfer = null;
                    }
                }

                if (fileEncryption != null)
                {
                    try
                    {
                        fileEncryption.Dispose();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        fileEncryption = null;
                    }
                }

                if (locator != null)
                {
                    try
                    {
                        locator.Delete();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        locator = null;
                    }
                }

                if (accessPolicy != null)
                {
                    try
                    {
                        accessPolicy.Delete();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        accessPolicy = null;
                    }
                }
            }
        }
コード例 #31
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subFolder,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
            {
            }

            Exception lastException = null;
            CloudBlockBlob blob = null;
            // stats from azurescope show 10 to be an optimal number of transfer threads
            int numThreads = ParallelTransferThreadCount;
            var file = new FileInfo(localFile);
            long fileSize = file.Length;

            int maxBlockSize = GetBlockSize(fileSize);

            // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
            // the 'key' is block id and 'value' is the block length.
            List<string> blockList;
            var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
            int exceptionCount = 0;

            blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType);
            blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy });

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            var options = new BlobRequestOptions
            {
                RetryPolicy = retryPolicy,
                ServerTimeout = TimeSpan.FromSeconds(90)
            };

            // Launch threads to upload blocks.
            var tasks = new List<Task>();
            long bytesSent = 0;
            Action action = () =>
            {

                List<Exception> exceptions = new List<Exception>();

                if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
                {
                    Thread.Sleep(_forceSharedAccessSignatureRetry);
                }

                if (queue.Count > 0)
                {
                    FileStream fs = null;

                    try
                    {
                        fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);

                        KeyValuePair<int, int> blockIdAndLength;
                        while (queue.TryDequeue(out blockIdAndLength))
                        {
                            cancellationToken.ThrowIfCancellationRequested();

                            try
                            {
                                var buffer = new byte[blockIdAndLength.Value];
                                var binaryReader = new BinaryReader(fs);

                                // move the file system reader to the proper position
                                fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
                                int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);

                                if (fileEncryption != null)
                                {
                                    lock (fileEncryption)
                                    {
                                        using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
                                        {
                                            encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                        }
                                    }
                                }

                                using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
                                {
                                    string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
                                    string blockHash = GetMd5HashFromStream(buffer);
                                    if (blob != null) blob.PutBlock(blockIdString, ms, blockHash, options: options);
                                }

                                Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
                                var progress = (int)((double)bytesSent / file.Length * 100);
                                var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
                                OnTaskProgressChanged(eArgs);
                            }
                            catch (StorageException ex)
                            {
                                TimeSpan tm;
                                exceptionCount++;
                                exceptions.Add(ex);
                                if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
                                {
                                    lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
                                    throw lastException;
                                }
                                Thread.Sleep(tm);

                                queue.Enqueue(blockIdAndLength);
                            }
                            catch (IOException ex)
                            {
                                TimeSpan tm;
                                exceptionCount++;
                                exceptions.Add(ex);
                                if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
                                {
                                    lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
                                        exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
                                    throw lastException;
                                }

                                // dispose existing file stream
                                if (fs != null)
                                {
                                    fs.Close();
                                }

                                Thread.Sleep(tm);

                                // try to reopen the file stream again
                                fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
                                queue.Enqueue(blockIdAndLength);
                            }
                        }
                    }
                    finally
                    {
                        if (fs != null)
                        {
                            fs.Close();
                        }
                    }
                }
            };

            for (int idxThread = 0; idxThread < numThreads; idxThread++)
            {
                tasks.Add(Task.Factory.StartNew(
                    action,
                    cancellationToken,
                    TaskCreationOptions.AttachedToParent,
                    TaskScheduler.Current));
            }

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
            {
                if (result.Any(t => t.IsFaulted))
                {
                    return;
                }
                blob.PutBlockList(blockList, options: options);

            }, TaskContinuationOptions.None).Wait(cancellationToken);

            TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
        }
コード例 #32
0
        private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
        {
            int numThreads = ParallelTransferThreadCount;
            List<Exception> exceptions = new List<Exception>();
            AggregateException aggregateException = null;
            long bytesDownloaded = 0;

            CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy);

            long blobLength = blob.Properties.Length;
            int bufferLength = GetBlockSize(blobLength);
            var queue = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads);

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
            {
                var tasks = new List<Task>();

                Action action = () =>
                {
                    KeyValuePair<long, int> blockOffsetAndLength;
                    int exceptionPerThread = 0;
                    // A buffer to fill per read request.
                    var buffer = new byte[bufferLength];

                    if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
                    {
                        // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
                        Thread.Sleep(_forceSharedAccessSignatureRetry);
                    }

                    while (queue.TryDequeue(out blockOffsetAndLength))
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            break;
                        }

                        try
                        {
                            var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob);

                            using (var response = blobGetRequest.GetResponse() as HttpWebResponse)
                            {
                                if (response != null)
                                {
                                    ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded);
                                    var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100);
                                    // raise the progress changed event
                                    var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null);
                                    OnTaskProgressChanged(eArgs);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            var webEx = ex as WebException;
                            bool ok = (webEx != null) || ex is ObjectDisposedException;
                            if (!ok)
                            {
                                throw;
                            }

                            if (webEx != null)
                            {
                                if (webEx.Response is HttpWebResponse)
                                {
                                    var httpex = (HttpWebResponse)webEx.Response;
                                    if (httpex.StatusCode == HttpStatusCode.Forbidden)
                                    {
                                       blob = InitializeCloudBlockBlob(uri, null, retryPolicy);
                                    }
                                }
                            }

                            TimeSpan tm;
                            exceptionPerThread++;
                            exceptions.Add(ex);
                            if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext()))
                            {
                                aggregateException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptions.Count), exceptions);
                                throw aggregateException;
                            }

                            Thread.Sleep(tm);
                            // Add block back to queue
                            queue.Enqueue(blockOffsetAndLength);
                        }
                    }
                };

                // Launch threads to download chunks.
                for (int idxThread = 0; idxThread < numThreads; idxThread++)
                {
                    tasks.Add(Task.Factory.StartNew(action));
                }

                if (cancellationToken.IsCancellationRequested)
                {
                    TaskCompletedCallback(true, aggregateException, BlobTransferType.Download, localFile, uri);
                    cancellationToken.ThrowIfCancellationRequested();
                }
                Task.WaitAll(tasks.ToArray(), cancellationToken);
                TaskCompletedCallback(cancellationToken.IsCancellationRequested, aggregateException, BlobTransferType.Download, localFile, uri);
            }
        }
コード例 #33
0
 /// <summary>
 /// Downloads the specified blob to the specified location.
 /// </summary>
 /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param>
 /// <param name="localFile">The full path where file will be saved.</param>
 /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param>
 /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
 /// <param name="client">The azure client to access a blob.</param>
 /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns>A task that downloads the specified blob.</returns>
 public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
 {
     SetMaxConnectionLimit(uri);
     Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy));
     return task;
 }
コード例 #34
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken, 
            Uri uri, 
            string localFile, 
            string contentType, 
            string subDirectory, 
            FileEncryption fileEncryption, 
            CloudBlobClient client, 
            IRetryPolicy retryPolicy, 
            Func<string> getSharedAccessSignature,
            bool shouldDoFileIO = true)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
            {
            }

            SetConnectionLimits(uri);

            ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions = new BlobRequestOptions
                {
                    RetryPolicy = retryPolicy,
                    ServerTimeout = TimeSpan.FromSeconds(90)
                };

            CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);
            BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));

            FileInfo file = new FileInfo(localFile);
            long fileSize = file.Length;

            if (fileSize == 0)
            {
                blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
            else
            {
                int numThreads = Environment.ProcessorCount*ParallelUploadDownloadThreadCountMultiplier;
                int blockSize = GetBlockSize(fileSize);

                BlobTransferContext transferContext = new BlobTransferContext();
                transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);

                transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
                for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                {
                    transferContext.BlocksForFileIO[i] = null;
                }
                transferContext.BlockSize = blockSize;
                transferContext.CancellationToken = cancellationToken;
                transferContext.Blob = blob;
                transferContext.BlobRequestOptions = blobRequestOptions;
                transferContext.Length = fileSize;
                transferContext.LocalFilePath = localFile;
                transferContext.OnComplete = () => uploadCompletedSignal.Set();
                transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
                transferContext.Client = client;
                transferContext.RetryPolicy = retryPolicy;
                transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                transferContext.ShouldDoFileIO = shouldDoFileIO;
                transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
                transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                transferContext.Exceptions = new ConcurrentBag<Exception>();
                transferContext.FileEncryption = fileEncryption;
                transferContext.ContentType = contentType;
                transferContext.BlobSubFolder = subDirectory;
                transferContext.NextFileIOBlock = 0;
                transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>();

                using (
                    FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                {
                    RunUploadLoop(transferContext, stream, numThreads);
                }

                transferContext.MemoryManager.ReleaseUnusedBuffers();

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
        }
コード例 #35
0
        private static Action GetEncryptionAction(
            CancellationToken cancellationToken,
            FileEncryption fileEncryption,
            IngestManifestFileData file,
            string destinationPath,
            FileInfo fileInfo,
            ConcurrentQueue <Tuple <int, int> > queue,
            int maxBlockSize)
        {
            Action action = () =>
            {
                cancellationToken.ThrowIfCancellationRequested();

                if (queue.Count > 0)
                {
                    using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read))
                    {
                        Tuple <int, int> blockIdAndLength;
                        while (queue.TryDequeue(out blockIdAndLength))
                        {
                            cancellationToken.ThrowIfCancellationRequested();
                            var buffer       = new byte[blockIdAndLength.Item2];
                            var binaryReader = new BinaryReader(fs);

                            // Move the file system reader to the proper position.
                            fs.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
                            int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2);

                            while (readSize != blockIdAndLength.Item2)
                            {
                                readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize);
                            }

                            bool lockWasTakenForEncode = false;
                            try
                            {
                                Monitor.Enter(fileEncryption, ref lockWasTakenForEncode);
                                using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1 * (long)maxBlockSize))
                                {
                                    encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                }
                            }
                            finally
                            {
                                if (lockWasTakenForEncode)
                                {
                                    Monitor.Exit(fileEncryption);
                                }
                            }

                            bool lockWasTakenForWrite = false;
                            try
                            {
                                Monitor.Enter(file, ref lockWasTakenForWrite);
                                using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write))
                                {
                                    writeStream.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
                                    writeStream.Write(buffer, 0, readSize);
                                }
                            }
                            finally
                            {
                                if (lockWasTakenForWrite)
                                {
                                    Monitor.Exit(file);
                                }
                            }
                        }
                    }
                }
            };

            return(action);
        }
コード例 #36
0
        private void AssetEncryptAction(string outputPath, bool overwriteExistingEncryptedFiles, CancellationToken cancellationToken, ConcurrentDictionary <string, IContentKey> keys, IIngestManifestAsset asset)
        {
            cancellationToken.ThrowIfCancellationRequested();

            List <Task> encryptTasks = new List <Task>();

            AssetCreationOptions assetCreationOptions = asset.Asset.Options;

            if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
            {
                IContentKey contentKeyData = keys[asset.Id];
                var         fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id));


                foreach (IngestManifestFileData file in asset.IngestManifestFiles)
                {
                    ulong iv = Convert.ToUInt64(file.InitializationVector, CultureInfo.InvariantCulture);
                    fileEncryption.SetInitializationVectorForFile(file.Name, iv);

                    FileInfo fileInfo = null;
                    fileInfo = TrackedFilesPaths.ContainsKey(file.Id) ? new FileInfo(TrackedFilesPaths[file.Id]) : new FileInfo(file.Name);

                    string destinationPath = Path.Combine(outputPath, fileInfo.Name);
                    if (File.Exists(destinationPath))
                    {
                        if (overwriteExistingEncryptedFiles)
                        {
                            File.Delete(destinationPath);
                        }
                        else
                        {
                            throw new IOException(string.Format(CultureInfo.InvariantCulture, StringTable.BulkIngestFileExists, destinationPath));
                        }
                    }

                    long fileSize     = fileInfo.Length;
                    int  maxBlockSize = GetBlockSize(fileSize);

                    int numThreads = MaxNumberOfEncryptionThreadsForFilePerCore * Environment.ProcessorCount;
                    ConcurrentQueue <Tuple <int, int> > queue = PrepareUploadQueue(maxBlockSize, fileSize);
                    if (queue.Count < numThreads)
                    {
                        numThreads = queue.Count;
                    }


                    File.Create(destinationPath).Dispose();

                    Action action = GetEncryptionAction(cancellationToken, fileEncryption, file, destinationPath, fileInfo, queue, maxBlockSize);
                    for (int i = 0; i < numThreads; i++)
                    {
                        encryptTasks.Add(Task.Factory.StartNew((action), cancellationToken));
                    }
                }
                try
                {
                    Task.WaitAll(encryptTasks.ToArray());
                }
                finally
                {
                    fileEncryption.Dispose();
                }
            }
        }
コード例 #37
0
		/// <summary>
		/// Downloads the specified blob to the specified location.
		/// </summary>
		/// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param>
		/// <param name="localFile">The full path where file will be saved.</param>
		/// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param>
		/// <param name="initializationVector">The initialization vector if encryption has been used.</param>
		/// <param name="client">The azure client to access a blob.</param>
		/// <param name="cancellationToken">The cancellation token to cancel the download operation.</param>
		/// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
		/// <param name="start">Start pos to download</param>
		/// <param name="length">Number of bytes to download, -1 to download all.</param>
		/// <returns>A task that downloads the specified blob.</returns>
		public virtual Task DownloadBlob(
			Uri uri, 
			string localFile, 
			FileEncryption fileEncryption, 
			ulong initializationVector, 
			CloudBlobClient client, 
			CancellationToken cancellationToken, 
			IRetryPolicy retryPolicy,
			Func<string> getSharedAccessSignature = null, 
			long start = 0, 
			long length = -1
            )
		{
			if (client != null && getSharedAccessSignature != null)
			{
				throw new InvalidOperationException("The arguments client and sharedAccessSignature cannot both be non-null");
			}

			if (start < 0)
			{
				throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Initial offset {0} to download a file must not be negative", start));
			}

			// To download the asset file as a whole or byte-range, the localFile should not be empty
			if (string.IsNullOrEmpty(localFile))
			{
				throw new ArgumentException("Parameter localFile must be set with a full path");
			}

			if (length < -1)
			{
				throw new ArgumentException("Parameter length must be equals or greater than -1");
			}

			if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
			{
				// The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
				Thread.Sleep(_forceSharedAccessSignatureRetry);
			}

			BlobDownloader blobDownloader = new BlobDownloader(new MemoryManagerFactory());

			blobDownloader.TransferCompleted += (sender, args) =>
				{
					if (TransferCompleted != null)
					{
						TransferCompleted(sender, args);
					}
					else if (args.Error != null)
					{
						throw args.Error;
					}
				};

			blobDownloader.TransferProgressChanged += (sender, args) =>
				{
					if (TransferProgressChanged != null)
					{
						TransferProgressChanged(sender, args);
					}
				};

			return blobDownloader.DownloadBlob(
				uri,
				localFile,
				fileEncryption,
				initializationVector,
				client,
				cancellationToken,
				retryPolicy,
				getSharedAccessSignature,
				start,
                length,
                parallelTransferThreadCount: ParallelTransferThreadCount,
                numberOfConcurrentTransfers: NumberOfConcurrentTransfers);
		}
コード例 #38
0
        /// <summary>
        /// Uploads file to a blob storage.
        /// </summary>
        /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
        /// <param name="localFile">The full path of local file.</param>
        /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
        /// then client can be null</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="contentType">Content type of the blob</param>
        /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <returns></returns>
        public virtual Task UploadBlob(
			Uri url, 
			string localFile, 
			FileEncryption fileEncryption, 
			CancellationToken cancellationToken, 
			CloudBlobClient client, 
			IRetryPolicy retryPolicy,
			string contentType = null,
			string subDirectory = "",
			Func<string> getSharedAccessSignature = null
            )
        {
            if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
            {
                // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
                Thread.Sleep(_forceSharedAccessSignatureRetry);
            }

            BlobUploader blobuploader = new BlobUploader(new MemoryManagerFactory());

            blobuploader.TransferCompleted += (sender, args) =>
            {
                if (TransferCompleted != null)
                {
                    TransferCompleted(sender, args);
                }
                else if (args.Error != null)
                {
                    throw args.Error;
                }
            };

            blobuploader.TransferProgressChanged += (sender, args) =>
            {
                if (TransferProgressChanged != null)
                {
                    TransferProgressChanged(sender, args);
                }
            };

            return blobuploader.UploadBlob(
                url,
                localFile,
                fileEncryption,
                cancellationToken,
                client,
                retryPolicy,
                contentType,
                subDirectory,
                getSharedAccessSignature: getSharedAccessSignature,
                parallelTransferThreadCount: ParallelTransferThreadCount,
                numberOfConcurrentTransfers: NumberOfConcurrentTransfers);
        }
コード例 #39
0
        /// <summary>
        /// Uploads file to a blob storage.
        /// </summary>
        /// <param name="url">The URL where file needs to be uploaded. If blob has private write permissions then 
        /// appropriate sas url need to be passed</param>
        /// <param name="localFile">The full path of local file.</param>
        /// <param name="contentType">Content type of the blob</param>
        /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <returns></returns>
        public virtual Task UploadBlob(
			Uri url,
			string localFile,
			string contentType,
			FileEncryption fileEncryption,
			CancellationToken cancellationToken,
			IRetryPolicy retryPolicy,
			Func<string> getSharedAccessSignature = null)
        {
            return UploadBlob(
                url,
                localFile,
                fileEncryption,
                cancellationToken,
                null,
                retryPolicy,
                contentType,
                getSharedAccessSignature: getSharedAccessSignature)
            ;
        }
コード例 #40
0
        /// <summary>
        /// Downloads the specified blob to the specified location.
        /// </summary>
        /// <param name="uri">The blob url  from which file  needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param>
        /// <param name="localFile">The full path where file will be saved </param>
        /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param>
        /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
        /// <param name="cancellationToken">The cancellation token.</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <returns></returns>
        public virtual Task DownloadBlob(
			Uri uri, 
			string localFile, 
			FileEncryption fileEncryption, 
			ulong initializationVector, 
			CancellationToken cancellationToken,
			IRetryPolicy retryPolicy,
			Func<string> getSharedAccessSignature = null
            )
        {
            return DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy, getSharedAccessSignature);
        }
コード例 #41
0
 /// <summary>
 /// Downloads the specified blob to the specified location.
 /// </summary>
 /// <param name="uri">The blob url  from which file  needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path where file will be saved </param>
 /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param>
 /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns></returns>
 public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
 {
     return DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy);
 }
コード例 #42
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string name,
            Stream stream,
            string contentType,
            string subDirectory,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            int parallelTransferThreadCount,
            bool shouldDoFileIO = true)
        {
            BlobTransferContext transferContext = new BlobTransferContext();

            transferContext.Exceptions = new ConcurrentBag <Exception>();
            try
            {
                ManualResetEvent   uploadCompletedSignal = new ManualResetEvent(false);
                BlobRequestOptions blobRequestOptions    = new BlobRequestOptions
                {
                    RetryPolicy   = retryPolicy,
                    ServerTimeout = TimeSpan.FromSeconds(90)
                };

                CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, name, contentType, getSharedAccessSignature);

                transferContext.Length         = stream.Length;
                transferContext.LocalFilePath  = name;
                transferContext.OnComplete     = () => uploadCompletedSignal.Set();
                transferContext.Blob           = blob;
                transferContext.FileEncryption = fileEncryption;
                if (stream.Length == 0)
                {
                    blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
                }
                else if (stream.Length < cloudBlockBlobUploadDownloadSizeLimit)
                {
                    AccessCondition  accessCondition  = AccessCondition.GenerateEmptyCondition();
                    OperationContext operationContext = new OperationContext();
                    operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);

                    using (var memoryStream = new MemoryStream())
                    {
                        stream.CopyTo(memoryStream);
                        byte[] fileContent = memoryStream.ToArray();
                        ApplyEncryptionTransform(
                            transferContext.FileEncryption,
                            Path.GetFileName(transferContext.LocalFilePath),
                            0,
                            fileContent,
                            Convert.ToInt32(stream.Length));

                        using (var uploadMemoryStream = new MemoryStream(fileContent))
                        {
                            blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
                        }
                    }
                    InvokeProgressCallback(transferContext, stream.Length, stream.Length);
                    transferContext.OnComplete();
                }
                else
                {
                    int numThreads = parallelTransferThreadCount;
                    int blockSize  = GetBlockSize(stream.Length);

                    transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(stream.Length, blockSize, ref numThreads);

                    transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                    for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                    {
                        transferContext.BlocksForFileIO[i] = null;
                    }
                    transferContext.BlockSize                = blockSize;
                    transferContext.CancellationToken        = cancellationToken;
                    transferContext.BlobRequestOptions       = blobRequestOptions;
                    transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                    transferContext.Client                   = client;
                    transferContext.RetryPolicy              = retryPolicy;
                    transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                    transferContext.ShouldDoFileIO           = shouldDoFileIO;
                    transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                    transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                    transferContext.ContentType              = contentType;
                    transferContext.BlobSubFolder            = subDirectory;
                    transferContext.NextFileIOBlock          = 0;
                    transferContext.PartialFileIOState       = new ConcurrentDictionary <long, int>();

                    RunUploadLoop(transferContext, stream, numThreads);
                }
            }
            catch (Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();
                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                        ? new AggregateException(transferContext.Exceptions)
                        : null,
                    BlobTransferType.Upload,
                    name,
                    uri);
            }
        }
コード例 #43
0
 /// <summary>
 /// Uploads file to a blob storage.
 /// </summary>
 /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path of local file.</param>
 /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url,
 /// then client can be null</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <param name="contentType">Content type of the blob</param>
 /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param>
 /// <returns></returns>
 public virtual Task UploadBlob(
     Uri url,
     string localFile,
     FileEncryption fileEncryption,
     CancellationToken cancellationToken,
     CloudBlobClient client,
     IRetryPolicy retryPolicy,
     string contentType = null,
     string subDirectory = "")
 {
     SetMaxConnectionLimit(url);
     return Task.Factory.StartNew(
         () => UploadFileToBlob(cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy),
         cancellationToken);
 }
コード例 #44
0
        public override Task <IAssetFile> CreateAsync(string name, CancellationToken cancelation)
        {
            if (_parentAsset == null)
            {
                throw new InvalidOperationException(StringTable.AssetFileCreateParentAssetIsNull);
            }

            if (string.IsNullOrWhiteSpace(name))
            {
                throw new ArgumentException(String.Format(CultureInfo.InvariantCulture, StringTable.ErrorCreatingAssetFileEmptyFileName));
            }

            cancelation.ThrowIfCancellationRequested();
            IMediaDataServiceContext dataContext = null;
            AssetFileData            assetFile   = null;

            return(Task.Factory.StartNew(() =>
            {
                cancelation.ThrowIfCancellationRequested();
                dataContext = MediaContext.MediaServicesClassFactory.CreateDataServiceContext();

                FileEncryption fileEncryption = null;

                // Set a MIME type based on the extension of the file name
                string mimeType = AssetFileData.GetMimeType(name);
                assetFile = new AssetFileData
                {
                    Name = name,
                    ParentAssetId = _parentAsset.Id,
                    MimeType = mimeType,
                };
                try
                {
                    // Update the files associated with the asset with the encryption related metadata.
                    if (_parentAsset.Options.HasFlag(AssetCreationOptions.StorageEncrypted))
                    {
                        IContentKey storageEncryptionKey = _parentAsset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault();
                        cancelation.ThrowIfCancellationRequested();
                        if (storageEncryptionKey == null)
                        {
                            throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, _parentAsset.Id));
                        }
                        fileEncryption = new FileEncryption(storageEncryptionKey.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(storageEncryptionKey.Id));
                        AssetBaseCollection.AddEncryptionMetadataToAssetFile(assetFile, fileEncryption);
                    }
                    else if (_parentAsset.Options.HasFlag(AssetCreationOptions.CommonEncryptionProtected))
                    {
                        AssetBaseCollection.SetAssetFileForCommonEncryption(assetFile);
                    }
                    else if (_parentAsset.Options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected))
                    {
                        AssetBaseCollection.SetAssetFileForEnvelopeEncryption(assetFile);
                    }
                }
                finally
                {
                    if (fileEncryption != null)
                    {
                        fileEncryption.Dispose();
                    }
                }
                dataContext.AddObject(FileSet, assetFile);
                cancelation.ThrowIfCancellationRequested();
                cancelation.ThrowIfCancellationRequested();
                MediaRetryPolicy retryPolicy = this.MediaContext.MediaServicesClassFactory.GetSaveChangesRetryPolicy(dataContext as IRetryPolicyAdapter);
                return retryPolicy.ExecuteAsync <IMediaDataServiceResponse>(() =>
                {
                    cancelation.ThrowIfCancellationRequested();
                    return dataContext.SaveChangesAsync(assetFile);
                }, cancelation).Result;
            }, cancelation)
                   .ContinueWith <IAssetFile>(t =>
            {
                t.ThrowIfFaulted();
                AssetFileData data = (AssetFileData)t.Result.AsyncState;
                return data;
            }, cancelation));
        }
コード例 #45
0
 private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response,
     KeyValuePair<long, int> blockOffsetAndLength, ref long bytesDownloaded)
 {
     using (Stream stream = response.GetResponseStream())
     {
         int offsetInChunk = 0;
         int remaining = blockOffsetAndLength.Value;
         while (remaining > 0)
         {
             int read = stream.Read(buffer, offsetInChunk, remaining);
             lock (lockobject)
             {
                 fs.Position = blockOffsetAndLength.Key + offsetInChunk;
                 if (fileEncryption != null)
                 {
                     lock (fileEncryption)
                     {
                         using (
                             FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk)
                             )
                         {
                             encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk);
                         }
                     }
                 }
                 fs.Write(buffer, offsetInChunk, read);
             }
             offsetInChunk += read;
             remaining -= read;
             Interlocked.Add(ref bytesDownloaded, read);
         }
     }
     return bytesDownloaded;
 }
コード例 #46
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subDirectory,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func<string> getSharedAccessSignature,
            int parallelTransferThreadCount,
            bool shouldDoFileIO = true)
        {
            BlobTransferContext transferContext = new BlobTransferContext();
            transferContext.Exceptions = new ConcurrentBag<Exception>();
            try
            {
                    ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false);
                    BlobRequestOptions blobRequestOptions = new BlobRequestOptions
                    {
                        RetryPolicy = retryPolicy,
                        ServerTimeout = TimeSpan.FromSeconds(90)
                    };
                    //attempt to open the file first so that we throw an exception before getting into the async work
                    using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    {
                    }
                    CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType,
                        getSharedAccessSignature);
                    BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));

                    FileInfo file = new FileInfo(localFile);
                    long fileSize = file.Length;
                    transferContext.Length = fileSize;
                    transferContext.LocalFilePath = localFile;
                    transferContext.OnComplete = () => uploadCompletedSignal.Set();
                    transferContext.Blob = blob;
                    transferContext.FileEncryption = fileEncryption;
                    if (fileSize == 0)
                    {
                        blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
                    }
                    else if (fileSize < cloudBlockBlobUploadDownloadSizeLimit)
                    {
                        AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition();
                        OperationContext operationContext = new OperationContext();
                        operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                        using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                        {
                               using (var memoryStream = new MemoryStream())
                                     {
                                         fileStream.CopyTo(memoryStream);
                                         byte[] fileContent = memoryStream.ToArray();
                                         ApplyEncryptionTransform(
                                                     transferContext.FileEncryption,
                                                     Path.GetFileName(transferContext.LocalFilePath),
                                                     0,
                                                     fileContent,
                                                     Convert.ToInt32(fileStream.Length));
                                         using (var uploadMemoryStream = new MemoryStream(fileContent))
                                         {
                                             blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
                                         }
                                     }
                        }
                        InvokeProgressCallback(transferContext, fileSize, fileSize);
                        transferContext.OnComplete();
                    }
                    else
                    {
                        int numThreads = parallelTransferThreadCount;
                        int blockSize = GetBlockSize(fileSize);

                        transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);

                        transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
                        for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                        {
                            transferContext.BlocksForFileIO[i] = null;
                        }
                        transferContext.BlockSize = blockSize;
                        transferContext.CancellationToken = cancellationToken;
                        transferContext.BlobRequestOptions = blobRequestOptions;
                        transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
                        transferContext.Client = client;
                        transferContext.RetryPolicy = retryPolicy;
                        transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                        transferContext.ShouldDoFileIO = shouldDoFileIO;
                        transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
                        transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                        transferContext.ContentType = contentType;
                        transferContext.BlobSubFolder = subDirectory;
                        transferContext.NextFileIOBlock = 0;
                        transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>();

                        using (
                            FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read,
                                FileShare.ReadWrite))
                        {
                            RunUploadLoop(transferContext, stream, numThreads);
                        }
                }
            }
            catch (Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();

                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                        ? new AggregateException(transferContext.Exceptions)
                        : null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
        }
コード例 #47
0
 /// <summary>
 /// Downloads the specified blob to the specified location.
 /// </summary>
 /// <param name="uri">The blob url  from which file  needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param>
 /// <param name="localFile">The full path where file will be saved </param>
 /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param>
 /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
 /// <param name="cancellationToken">The cancellation token.</param>
 /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
 /// <returns></returns>
 public Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy)
 {
     return(DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy));
 }
        private static Action GetEncryptionAction(
            CancellationToken cancellationToken,
            FileEncryption fileEncryption,
            IngestManifestFileData file,
            string destinationPath,
            FileInfo fileInfo,
            ConcurrentQueue<Tuple<int, int>> queue,
            int maxBlockSize)
        {
            Action action = () =>
                                {
                                    cancellationToken.ThrowIfCancellationRequested();

                                    if (queue.Count > 0)
                                    {
                                        using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read))
                                        {
                                            Tuple<int, int> blockIdAndLength;
                                            while (queue.TryDequeue(out blockIdAndLength))
                                            {
                                                cancellationToken.ThrowIfCancellationRequested();
                                                var buffer = new byte[blockIdAndLength.Item2];
                                                var binaryReader = new BinaryReader(fs);

                                                // Move the file system reader to the proper position.
                                                fs.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin);
                                                int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2);

                                                while (readSize != blockIdAndLength.Item2)
                                                {
                                                    readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize);
                                                }

                                                bool lockWasTakenForEncode = false;
                                                try
                                                {
                                                    Monitor.Enter(fileEncryption, ref lockWasTakenForEncode);
                                                    using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1*(long) maxBlockSize))
                                                    {
                                                        encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                                    }
                                                }
                                                finally
                                                {
                                                    if (lockWasTakenForEncode) Monitor.Exit(fileEncryption);
                                                }

                                                bool lockWasTakenForWrite = false;
                                                try
                                                {
                                                    Monitor.Enter(file, ref lockWasTakenForWrite);
                                                    using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write))
                                                    {
                                                        writeStream.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin);
                                                        writeStream.Write(buffer, 0, readSize);
                                                    }
                                                }
                                                finally
                                                {
                                                    if (lockWasTakenForWrite) Monitor.Exit(file);
                                                }
                                            }
                                        }
                                    }
                                };
            return action;
        }
コード例 #49
0
        private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature)
        {
            int       exceptionCount  = 0;
            int       numThreads      = ParallelTransferThreadCount;
            Exception lastException   = null;
            long      bytesDownloaded = 0;

            CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);

            long blobLength   = blob.Properties.Length;
            int  bufferLength = GetBlockSize(blobLength);
            var  queue        = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads);

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
            {
                var tasks = new List <Task>();

                Action action = () =>
                {
                    KeyValuePair <long, int> blockOffsetAndLength;
                    int exceptionPerThread = 0;
                    int sasRetry           = 0;
                    // A buffer to fill per read request.
                    var buffer = new byte[bufferLength];

                    if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
                    {
                        // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
                        Thread.Sleep(_forceSharedAccessSignatureRetry);
                    }

                    while (queue.TryDequeue(out blockOffsetAndLength))
                    {
                        if (cancellationToken.IsCancellationRequested)
                        {
                            break;
                        }

                        try
                        {
                            var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob);

                            using (var response = blobGetRequest.GetResponse() as HttpWebResponse)
                            {
                                if (response != null)
                                {
                                    ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded);
                                    var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100);
                                    // raise the progress changed event
                                    var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null);
                                    OnTaskProgressChanged(eArgs);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            var  webEx          = ex as WebException;
                            bool ok             = (webEx != null) || ex is ObjectDisposedException;
                            bool isSasException = false;
                            if (!ok)
                            {
                                throw;
                            }

                            if (webEx != null && getSharedAccessSignature != null)
                            {
                                if (webEx.Response is HttpWebResponse)
                                {
                                    var httpex = (HttpWebResponse)webEx.Response;
                                    if (httpex.StatusCode == HttpStatusCode.Forbidden)
                                    {
                                        sasRetry++;
                                        if (sasRetry > MaxSasSignatureRetry)
                                        {
                                            throw;
                                        }
                                        isSasException = true;
                                    }
                                }
                            }

                            if (isSasException)
                            {
                                blob = InitializeCloudBlockBlob(uri, client, retryPolicy,
                                                                getSharedAccessSignature);
                            }
                            else
                            {
                                TimeSpan tm;
                                exceptionCount++;
                                exceptionPerThread++;
                                if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext()))
                                {
                                    lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptionCount), ex);
                                    break;
                                }

                                Thread.Sleep(tm);
                            }
                            // Add block back to queue
                            queue.Enqueue(blockOffsetAndLength);
                        }
                    }
                };


                // Launch threads to download chunks.
                for (int idxThread = 0; idxThread < numThreads; idxThread++)
                {
                    tasks.Add(Task.Factory.StartNew(action));
                }

                if (cancellationToken.IsCancellationRequested)
                {
                    TaskCompletedCallback(true, lastException, BlobTransferType.Download, localFile, uri);
                    cancellationToken.ThrowIfCancellationRequested();
                }
                Task.WaitAll(tasks.ToArray(), cancellationToken);
                TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Download, localFile, uri);
            }
        }
        private void AssetEncryptAction(string outputPath, bool overwriteExistingEncryptedFiles, CancellationToken cancellationToken, ConcurrentDictionary<string, IContentKey> keys, IIngestManifestAsset asset)
        {
            cancellationToken.ThrowIfCancellationRequested();

            List<Task> encryptTasks = new List<Task>();

            AssetCreationOptions assetCreationOptions = asset.Asset.Options;
            if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
            {
                IContentKey contentKeyData = keys[asset.Id];
                var fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id));

                foreach (IngestManifestFileData file in asset.IngestManifestFiles)
                {
                    ulong iv = Convert.ToUInt64(file.InitializationVector, CultureInfo.InvariantCulture);
                    fileEncryption.SetInitializationVectorForFile(file.Name, iv);

                    FileInfo fileInfo = null;
                    fileInfo = TrackedFilesPaths.ContainsKey(file.Id) ? new FileInfo(TrackedFilesPaths[file.Id]) : new FileInfo(file.Name);

                    string destinationPath = Path.Combine(outputPath, fileInfo.Name);
                    if (File.Exists(destinationPath))
                    {
                        if (overwriteExistingEncryptedFiles)
                        {
                            File.Delete(destinationPath);
                        }
                        else
                        {
                            throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, StringTable.BulkIngestFileExists, destinationPath));
                        }
                    }

                    long fileSize = fileInfo.Length;
                    int maxBlockSize = GetBlockSize(fileSize);

                    int numThreads = MaxNumberOfEncryptionThreadsForFilePerCore*Environment.ProcessorCount;
                    ConcurrentQueue<Tuple<int, int>> queue = PrepareUploadQueue(maxBlockSize, fileSize);
                    if (queue.Count < numThreads)
                    {
                        numThreads = queue.Count;
                    }

                    File.Create(destinationPath).Dispose();

                    Action action = GetEncryptionAction(cancellationToken, fileEncryption, file, destinationPath, fileInfo, queue, maxBlockSize);
                    for (int i = 0; i < numThreads; i++)
                    {
                        encryptTasks.Add(Task.Factory.StartNew((action), cancellationToken));
                    }

                }
                try
                {
                    Task.WaitAll(encryptTasks.ToArray());
                }
                finally
                {
                    fileEncryption.Dispose();
                }
            }
        }
コード例 #51
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subFolder,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
            {
            }

            Exception      lastException = null;
            CloudBlockBlob blob          = null;
            // stats from azurescope show 10 to be an optimal number of transfer threads
            int  numThreads = ParallelTransferThreadCount;
            var  file       = new FileInfo(localFile);
            long fileSize   = file.Length;

            int maxBlockSize = GetBlockSize(fileSize);

            // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
            // the 'key' is block id and 'value' is the block length.
            List <string> blockList;
            var           queue          = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
            int           exceptionCount = 0;

            blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
            blob.DeleteIfExists(options: new BlobRequestOptions()
            {
                RetryPolicy = retryPolicy
            });

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            var options = new BlobRequestOptions
            {
                RetryPolicy   = retryPolicy,
                ServerTimeout = TimeSpan.FromSeconds(90)
            };

            // Launch threads to upload blocks.
            var    tasks     = new List <Task>();
            long   bytesSent = 0;
            Action action    = () =>
            {
                List <Exception> exceptions = new List <Exception>();
                int sasRetry = 0;

                if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
                {
                    Thread.Sleep(_forceSharedAccessSignatureRetry);
                }

                if (queue.Count > 0)
                {
                    FileStream fs = null;

                    try
                    {
                        fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);

                        KeyValuePair <int, int> blockIdAndLength;
                        while (queue.TryDequeue(out blockIdAndLength))
                        {
                            cancellationToken.ThrowIfCancellationRequested();

                            try
                            {
                                var buffer       = new byte[blockIdAndLength.Value];
                                var binaryReader = new BinaryReader(fs);

                                // move the file system reader to the proper position
                                fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
                                int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);

                                if (fileEncryption != null)
                                {
                                    lock (fileEncryption)
                                    {
                                        using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
                                        {
                                            encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                        }
                                    }
                                }

                                using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
                                {
                                    string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
                                    string blockHash     = GetMd5HashFromStream(buffer);
                                    if (blob != null)
                                    {
                                        blob.PutBlock(blockIdString, ms, blockHash, options: options);
                                    }
                                }

                                Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
                                var progress = (int)((double)bytesSent / file.Length * 100);
                                var eArgs    = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
                                OnTaskProgressChanged(eArgs);
                            }
                            catch (StorageException ex)
                            {
                                if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
                                {
                                    sasRetry++;
                                    if (sasRetry > MaxSasSignatureRetry)
                                    {
                                        throw;
                                    }
                                    blob = GetCloudBlockBlob(uri, client, subFolder, localFile,
                                                             contentType, getSharedAccessSignature);
                                }
                                else
                                {
                                    TimeSpan tm;
                                    exceptionCount++;
                                    exceptions.Add(ex);
                                    if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
                                    {
                                        lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
                                        throw lastException;
                                    }
                                    Thread.Sleep(tm);
                                }
                                queue.Enqueue(blockIdAndLength);
                            }
                            catch (IOException ex)
                            {
                                TimeSpan tm;
                                exceptionCount++;
                                exceptions.Add(ex);
                                if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
                                {
                                    lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
                                                                                         exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
                                    throw lastException;
                                }

                                // dispose existing file stream
                                if (fs != null)
                                {
                                    fs.Close();
                                }

                                Thread.Sleep(tm);

                                // try to reopen the file stream again
                                fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
                                queue.Enqueue(blockIdAndLength);
                            }
                        }
                    }
                    finally
                    {
                        if (fs != null)
                        {
                            fs.Close();
                        }
                    }
                }
            };

            for (int idxThread = 0; idxThread < numThreads; idxThread++)
            {
                tasks.Add(Task.Factory.StartNew(
                              action,
                              cancellationToken,
                              TaskCreationOptions.AttachedToParent,
                              TaskScheduler.Current));
            }

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
            {
                if (result.Any(t => t.IsFaulted))
                {
                    return;
                }
                try
                {
                    blob.PutBlockList(blockList, options: options);
                }
                catch (StorageException ex)
                {
                    if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
                    {
                        blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
                        blob.PutBlockList(blockList, options: options);
                    }
                    else
                    {
                        throw;
                    }
                }
            }, TaskContinuationOptions.None).Wait(cancellationToken);

            TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
        }
コード例 #52
0
        private void PostUploadAction(Task task,string path, FileEncryption fileEncryption, AssetCreationOptions assetCreationOptions, CancellationToken token)
        {
            try
            {
                task.ThrowIfFaulted();
                token.ThrowIfCancellationRequested();

                FileInfo fileInfo = new FileInfo(path);

                //Updating Name based on file name to avoid exceptions trying to download file.Mapping to storage account is through file name
                this.Name = fileInfo.Name;

                // Set the ContentFileSize base on the local file size
                this.ContentFileSize = fileInfo.Length;

                // Update the files associated with the asset with the encryption related metadata.
                if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted))
                {
                    AssetBaseCollection.AddEncryptionMetadataToAssetFile(this, fileEncryption);
                }
                else if (assetCreationOptions.HasFlag(AssetCreationOptions.CommonEncryptionProtected))
                {
                    AssetBaseCollection.SetAssetFileForCommonEncryption(this);
                }
                else if (assetCreationOptions.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected))
                {
                    AssetBaseCollection.SetAssetFileForEnvelopeEncryption(this);
                }
                this.Update();
            }
            finally
            {
                if (fileEncryption != null)
                {
                    fileEncryption.Dispose();
                }
            }
        }
コード例 #53
0
        private void DownloadFileFromBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func<string> getSharedAccessSignature,
            bool shouldDoFileIO = true,
            long start = 0,
            long length = -1,
            int parallelTransferThreadCount = 10)
        {
            int numThreads = Environment.ProcessorCount * parallelTransferThreadCount;
            ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy };
            CloudBlockBlob blob = null;
            BlobTransferContext transferContext = new BlobTransferContext();
            transferContext.Exceptions = new ConcurrentBag<Exception>();
            try
            {
                blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);

                long initialOffset = start;
                long sizeToDownload = blob.Properties.Length;

                if (length != -1)
                {
                    if (length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                            "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
                    }

                    if (start + length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                            "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
                            blob.Properties.Length));
                    }

                    sizeToDownload = length;
                }
                if (sizeToDownload == 0)
                {
                    using (FileStream stream =
                        new FileStream(
                            localFile,
                            FileMode.OpenOrCreate,
                            FileAccess.Write,
                            FileShare.Read
                            ))
                    {
                    }

                }
                else
                {
                    int blockSize = GetBlockSize(blob.Properties.Length);

                    transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
                        ref numThreads, initialOffset);

                    transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>();
                    for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                    {
                        transferContext.BlocksForFileIO[i] = null;
                    }
                    transferContext.BlockSize = blockSize;
                    transferContext.CancellationToken = cancellationToken;
                    transferContext.Blob = blob;
                    transferContext.BlobRequestOptions = blobRequestOptions;
                    transferContext.Length = sizeToDownload;

                    transferContext.LocalFilePath = localFile;
                    transferContext.OnComplete = () => downloadCompletedSignal.Set();
                    transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize);
                    transferContext.Client = client;
                    transferContext.RetryPolicy = retryPolicy;
                    transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                    transferContext.ShouldDoFileIO = shouldDoFileIO;
                    transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>();
                    transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                    transferContext.FileEncryption = fileEncryption;
                    transferContext.InitializationVector = initializationVector;
                    transferContext.InitialOffset = start;

                    using (FileStream stream = new FileStream(
                        transferContext.LocalFilePath,
                        FileMode.OpenOrCreate,
                        FileAccess.Write,
                        FileShare.Read
                        ))
                    {
                        stream.SetLength(sizeToDownload);
                        RunDownloadLoop(transferContext, stream, numThreads);
                    }
                }
            }
            catch(Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                 // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();
                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                    TaskCompletedCallback(
                        cancellationToken.IsCancellationRequested,
                        transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                            ? new AggregateException(transferContext.Exceptions)
                            : null,
                        BlobTransferType.Download,
                        localFile,
                        uri);
            }
        }
コード例 #54
0
        private void Cleanup(BlobTransferClient blobTransfer, FileEncryption fileEncryption, ILocator locator, IAccessPolicy accessPolicy)
        {
            lock (this._lock)
            {
                if (blobTransfer != null)
                {
                    try
                    {
                        blobTransfer.TransferProgressChanged -= this.OnDownloadBlobTransferProgressChanged;
                    }
                    catch
                    {
                    }
                    finally
                    {
                        blobTransfer = null;
                    }
                }

                if (fileEncryption != null)
                {
                    try
                    {
                        fileEncryption.Dispose();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        fileEncryption = null;
                    }
                }

                if (locator != null)
                {
                    try
                    {
                        locator.Delete();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        locator = null;
                    }
                }

                if (accessPolicy != null)
                {
                    try
                    {
                        accessPolicy.Delete();
                    }
                    catch
                    {
                    }
                    finally
                    {
                        accessPolicy = null;
                    }
                }
            }
        }
コード例 #55
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subDirectory,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            bool shouldDoFileIO = true)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
            {
            }

            SetConnectionLimits(uri);

            ManualResetEvent   uploadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions    = new BlobRequestOptions
            {
                RetryPolicy   = retryPolicy,
                ServerTimeout = TimeSpan.FromSeconds(90)
            };

            CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);

            BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));

            FileInfo file     = new FileInfo(localFile);
            long     fileSize = file.Length;

            if (fileSize == 0)
            {
                blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
            else
            {
                int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
                int blockSize  = GetBlockSize(fileSize);

                BlobTransferContext transferContext = new BlobTransferContext();
                transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);

                transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                {
                    transferContext.BlocksForFileIO[i] = null;
                }
                transferContext.BlockSize         = blockSize;
                transferContext.CancellationToken = cancellationToken;
                transferContext.Blob = blob;
                transferContext.BlobRequestOptions       = blobRequestOptions;
                transferContext.Length                   = fileSize;
                transferContext.LocalFilePath            = localFile;
                transferContext.OnComplete               = () => uploadCompletedSignal.Set();
                transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                transferContext.Client                   = client;
                transferContext.RetryPolicy              = retryPolicy;
                transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                transferContext.ShouldDoFileIO           = shouldDoFileIO;
                transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                transferContext.Exceptions               = new ConcurrentBag <Exception>();
                transferContext.FileEncryption           = fileEncryption;
                transferContext.ContentType              = contentType;
                transferContext.BlobSubFolder            = subDirectory;
                transferContext.NextFileIOBlock          = 0;
                transferContext.PartialFileIOState       = new ConcurrentDictionary <long, int>();

                using (
                    FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                {
                    RunUploadLoop(transferContext, stream, numThreads);
                }

                transferContext.MemoryManager.ReleaseUnusedBuffers();

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
        }
コード例 #56
0
        /// <summary>
        /// Downloads the specified blob to the specified location.
        /// </summary>
        /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param>
        /// <param name="localFile">The full path where file will be saved.</param>
        /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param>
        /// <param name="initializationVector">The initialization vector if encryption has been used.</param>
        /// <param name="client">The azure client to access a blob.</param>
        /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param>
        /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param>
        /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param>
        /// <param name="start">Start pos to download</param>
        /// <param name="length">Number of bytes to download, -1 to download all.</param>
        /// <returns>A task that downloads the specified blob.</returns>
        public virtual Task DownloadBlob(
			Uri uri, 
			string localFile, 
			FileEncryption fileEncryption, 
			ulong initializationVector, 
			CloudBlobClient client, 
			CancellationToken cancellationToken, 
			IRetryPolicy retryPolicy,
			Func<string> getSharedAccessSignature = null, 
			long start = 0, 
			long length = -1
            )
        {
            if (client != null && getSharedAccessSignature != null)
            {
                throw new InvalidOperationException("The arguments client and sharedAccessSignature cannot both be non-null");
            }

            if (start < 0)
            {
                throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Initial offset {0} to download a file must not be negative", start));
            }

            // To download the asset file as a whole or byte-range, the localFile should not be empty
            if (string.IsNullOrEmpty(localFile))
            {
                throw new ArgumentException("Parameter localFile must be set with a full path");
            }

            if (length < -1)
            {
                throw new ArgumentException("Parameter length must be equals or greater than -1");
            }

            if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
            {
                // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path
                Thread.Sleep(_forceSharedAccessSignatureRetry);
            }

            BlobDownloader blobDownloader = new BlobDownloader(new MemoryManagerFactory());

            blobDownloader.TransferCompleted += (sender, args) =>
                {
                    if (TransferCompleted != null)
                    {
                        TransferCompleted(sender, args);
                    }
                    else if (args.Error != null)
                    {
                        throw args.Error;
                    }
                };

            blobDownloader.TransferProgressChanged += (sender, args) =>
                {
                    if (TransferProgressChanged != null)
                    {
                        TransferProgressChanged(sender, args);
                    }
                };

            return blobDownloader.DownloadBlob(
                uri,
                localFile,
                fileEncryption,
                initializationVector,
                client,
                cancellationToken,
                retryPolicy,
                getSharedAccessSignature,
                start,
                length,
                parallelTransferThreadCount: ParallelTransferThreadCount,
                numberOfConcurrentTransfers: NumberOfConcurrentTransfers);
        }