public Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func <string> getSharedAccessSignature = null) { SetConnectionLimits(url); return(Task.Factory.StartNew( () => UploadFileToBlob( cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy, getSharedAccessSignature), cancellationToken)); }
public Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func<string> getSharedAccessSignature = null, int parallelTransferThreadCount = 10, int numberOfConcurrentTransfers = default(int)) { SetConnectionLimits(url, numberOfConcurrentTransfers); return Task.Factory.StartNew( () => UploadFileToBlob( cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy, getSharedAccessSignature, parallelTransferThreadCount), cancellationToken); }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param> /// <param name="localFile">The full path where file will be saved.</param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="client">The azure client to access a blob.</param> /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns>A task that downloads the specified blob.</returns> public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { SetMaxConnectionLimit(uri); Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy)); return(task); }
/// <summary> /// Uploads the destinationPath with given path asynchronously /// </summary> /// <param name="path">The path of a destinationPath to upload</param> /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to upload files.</param> /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param> /// <param name="token">A <see cref="CancellationToken"/> to use for canceling upload operation.</param> /// <returns>A function delegate that returns the future result to be available through the Task.</returns> public Task UploadAsync(string path, BlobTransferClient blobTransferClient, ILocator locator, CancellationToken token) { if (blobTransferClient == null) { throw new ArgumentNullException("blobTransferClient"); } if (locator == null) { throw new ArgumentNullException("locator"); } if (path == null) { throw new ArgumentNullException("path"); } if (!File.Exists(path)) { throw new FileNotFoundException(path); } ValidateFileName(path); IContentKey contentKeyData = null; FileEncryption fileEncryption = null; AssetCreationOptions assetCreationOptions = this.Asset.Options; if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { contentKeyData = this.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault(); if (contentKeyData == null) { throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, this.Asset.Id)); } fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id)); } EventHandler <BlobTransferProgressChangedEventArgs> handler = (s, e) => OnUploadProgressChanged(path, e); blobTransferClient.TransferProgressChanged += handler; MediaRetryPolicy retryPolicy = this.GetMediaContext().MediaServicesClassFactory.GetBlobStorageClientRetryPolicy(); return(blobTransferClient.UploadBlob( new Uri(locator.BaseUri), path, null, fileEncryption, token, retryPolicy.AsAzureStorageClientRetryPolicy(), () => locator.ContentAccessComponent) .ContinueWith( ts => { blobTransferClient.TransferProgressChanged -= handler; this.PostUploadAction(ts, path, fileEncryption, assetCreationOptions, token); })); }
public Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature = null, long start = 0, long length = -1, int parallelTransferThreadCount = 10, int numberOfConcurrentTransfers = 2) { if (client != null && getSharedAccessSignature != null) { throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null"); } SetConnectionLimits(uri, Environment.ProcessorCount * numberOfConcurrentTransfers * parallelTransferThreadCount); Task task = Task.Factory.StartNew( () => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature, start: start, length: length, parallelTransferThreadCount: parallelTransferThreadCount)); return task; }
/// <summary> /// Initializes this instance. /// </summary> public void Init() { if (!this.HasFileEncryption) { this._fileEncryption = new FileEncryption(); } }
public Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func<string> getSharedAccessSignature = null) { SetConnectionLimits(url); return Task.Factory.StartNew( () => UploadFileToBlob( cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy, getSharedAccessSignature), cancellationToken); }
public Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature = null, long start = 0, long length = -1, int parallelTransferThreadCount = 10, int numberOfConcurrentTransfers = default(int)) { if (client != null && getSharedAccessSignature != null) { throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null"); } SetConnectionLimits(uri, numberOfConcurrentTransfers); Task task = Task.Factory.StartNew( () => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature, start: start, length: length, parallelTransferThreadCount: parallelTransferThreadCount)); return(task); }
public Task UploadBlob( Uri url, string name, Stream stream, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func <string> getSharedAccessSignature = null, int parallelTransferThreadCount = 10, int numberOfConcurrentTransfers = default(int)) { SetConnectionLimits(url, numberOfConcurrentTransfers); return(Task.Factory.StartNew( () => UploadFileToBlob( cancellationToken, url, name, stream, contentType, subDirectory, fileEncryption, client, retryPolicy, getSharedAccessSignature, parallelTransferThreadCount), cancellationToken)); }
/// <summary> /// Downloads the file asynchronously . /// </summary> /// <param name="destinationPath">The path to download the file to.</param> /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to download files.</param> /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param> /// <param name="retryPolicy">The retry policy.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A function delegate that returns the future result to be available through the Task.</returns> internal Task DownloadToFileAsync(string destinationPath, BlobTransferClient blobTransferClient, ILocator locator, IRetryPolicy retryPolicy, CancellationToken cancellationToken) { FileEncryption fileEncryption = this.GetFileEncryption(); cancellationToken.Register(() => this.Cleanup(null, fileEncryption, null, null)); return(Task.Factory.StartNew(() => { cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null)); ulong iv = Convert.ToUInt64(this.InitializationVector, CultureInfo.InvariantCulture); UriBuilder uriBuilder = new UriBuilder(locator.Path); uriBuilder.Path += String.Concat("/", Name); blobTransferClient.TransferProgressChanged += this.OnDownloadBlobTransferProgressChanged; blobTransferClient.DownloadBlob(uriBuilder.Uri, destinationPath, fileEncryption, iv, cancellationToken, retryPolicy).Wait(cancellationToken); cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null)); }, cancellationToken) .ContinueWith( t => { t.ThrowIfFaulted(() => this.Cleanup(null, fileEncryption, null, null)); cancellationToken.ThrowIfCancellationRequested(() => this.Cleanup(null, fileEncryption, null, null)); this.Cleanup(null, fileEncryption, null, null); }, cancellationToken)); }
private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response, KeyValuePair <long, int> blockOffsetAndLength, ref long bytesDownloaded) { using (Stream stream = response.GetResponseStream()) { int offsetInChunk = 0; int remaining = blockOffsetAndLength.Value; while (remaining > 0) { int read = stream.Read(buffer, offsetInChunk, remaining); lock (lockobject) { fs.Position = blockOffsetAndLength.Key + offsetInChunk; if (fileEncryption != null) { lock (fileEncryption) { using ( FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk) ) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk); } } } fs.Write(buffer, offsetInChunk, read); } offsetInChunk += read; remaining -= read; Interlocked.Add(ref bytesDownloaded, read); } } return(bytesDownloaded); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url, /// then client can be null</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="contentType">Content type of the blob</param> /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func <string> getSharedAccessSignature = null ) { var fs = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); return(UploadBlob( url, localFile, fs, fileEncryption, cancellationToken, client, retryPolicy, contentType, subDirectory, getSharedAccessSignature) .ContinueWith(t => fs.Dispose())); }
/// <summary> /// Adds the encryption metadata to the file info. /// </summary> /// <param name="file">The file information to update.</param> /// <param name="fileEncryption">The file encryption to use.</param> internal static void AddEncryptionMetadataToAssetFile(AssetFileData file, FileEncryption fileEncryption) { ulong iv = fileEncryption.GetInitializationVectorForFile(file.Name); file.IsEncrypted = true; file.EncryptionKeyId = fileEncryption.GetKeyIdentifierAsString(); file.EncryptionScheme = FileEncryption.SchemeName; file.EncryptionVersion = FileEncryption.SchemeVersion; file.InitializationVector = iv.ToString(CultureInfo.InvariantCulture); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="contentType">Content type of the blob</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns></returns> public Task UploadBlob( Uri url, string localFile, string contentType, FileEncryption fileEncryption, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { return UploadBlob(url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="contentType">Content type of the blob</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns></returns> public Task UploadBlob( Uri url, string localFile, string contentType, FileEncryption fileEncryption, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { return(UploadBlob(url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType)); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url, /// then client can be null</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="contentType">Content type of the blob</param> /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func<string> getSharedAccessSignature = null ) { if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path Thread.Sleep(_forceSharedAccessSignatureRetry); } BlobUploader blobuploader = new BlobUploader(new MemoryManagerFactory()); blobuploader.TransferCompleted += (sender, args) => { if (TransferCompleted != null) { TransferCompleted(sender, args); } else if (args.Error != null) { throw args.Error; } }; blobuploader.TransferProgressChanged += (sender, args) => { if (TransferProgressChanged != null) { TransferProgressChanged(sender, args); } }; return blobuploader.UploadBlob( url, localFile, fileEncryption, cancellationToken, client, retryPolicy, contentType, subDirectory, getSharedAccessSignature: getSharedAccessSignature, parallelTransferThreadCount: ParallelTransferThreadCount, numberOfConcurrentTransfers: NumberOfConcurrentTransfers); }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path where file will be saved </param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns></returns> public virtual Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature = null) { return(DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy, getSharedAccessSignature)); }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param> /// <param name="localFile">The full path where file will be saved.</param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="client">The azure client to access a blob.</param> /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns>A task that downloads the specified blob.</returns> public Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature = null) { if (client != null && getSharedAccessSignature != null) { throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null"); } SetMaxConnectionLimit(uri); Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature)); return(task); }
protected void ApplyEncryptionTransform(FileEncryption fileEncryption, string fileName, long beginFilePosition, byte[] buffer, int bytesToWrite) { if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(fileName, beginFilePosition)) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: 0, inputCount: bytesToWrite, outputBuffer: buffer, outputOffset: 0); } } } }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url, /// then client can be null</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="contentType">Content type of the blob</param> /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "") { SetMaxConnectionLimit(url); return(Task.Factory.StartNew( () => UploadFileToBlob(cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy), cancellationToken)); }
/// <summary> /// Creates the storage content key. /// </summary> /// <param name="fileEncryption">The file encryption.</param> /// <param name="cert">The cert.</param> /// <returns>The content key.</returns> internal static ContentKeyData CreateStorageContentKey(FileEncryption fileEncryption, X509Certificate2 cert) { byte[] encryptedContentKey = fileEncryption.EncryptContentKeyToCertificate(cert); ContentKeyData contentKeyData = new ContentKeyData { Id = fileEncryption.GetKeyIdentifierAsString(), EncryptedContentKey = Convert.ToBase64String(encryptedContentKey), ContentKeyType = (int)ContentKeyType.StorageEncryption, ProtectionKeyId = cert.Thumbprint, ProtectionKeyType = (int)ProtectionKeyType.X509CertificateThumbprint, Checksum = fileEncryption.GetChecksum() }; return(contentKeyData); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded. If blob has private write permissions then /// appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="contentType">Content type of the blob</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, string contentType, FileEncryption fileEncryption, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature = null) { return(UploadBlob( url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType, getSharedAccessSignature: getSharedAccessSignature)); }
public Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature = null, long start = 0, long length = -1) { if (client != null && getSharedAccessSignature != null) { throw new InvalidOperationException("The arguments client and getSharedAccessSignature cannot both be non-null"); } SetConnectionLimits(uri); Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature, start:start, length:length)); return task; }
private void PostUploadAction(Task task, string path, FileEncryption fileEncryption, AssetCreationOptions assetCreationOptions, CancellationToken token) { try { task.ThrowIfFaulted(); token.ThrowIfCancellationRequested(); FileInfo fileInfo = new FileInfo(path); //Updating Name based on file name to avoid exceptions trying to download file.Mapping to storage account is through file name this.Name = fileInfo.Name; // Set the ContentFileSize base on the local file size this.ContentFileSize = fileInfo.Length; // Update the files associated with the asset with the encryption related metadata. if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { AssetBaseCollection.AddEncryptionMetadataToAssetFile(this, fileEncryption); } else if (assetCreationOptions.HasFlag(AssetCreationOptions.CommonEncryptionProtected)) { AssetBaseCollection.SetAssetFileForCommonEncryption(this); } else if (assetCreationOptions.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected)) { AssetBaseCollection.SetAssetFileForEnvelopeEncryption(this); } this.Update(); } finally { if (fileEncryption != null) { fileEncryption.Dispose(); } } }
private static void SetEncryptionSettings(IIngestManifestAsset ingestManifestAsset, AssetCreationOptions options, IngestManifestFileData data) { if (options.HasFlag(AssetCreationOptions.StorageEncrypted)) { var contentKeyData = ingestManifestAsset.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault(); if (contentKeyData == null) { throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, ingestManifestAsset.Asset.Id)); } using (var fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id))) { if (!fileEncryption.IsInitializationVectorPresent(data.Name)) { fileEncryption.CreateInitializationVectorForFile(data.Name); } ulong iv = fileEncryption.GetInitializationVectorForFile(data.Name); data.IsEncrypted = true; data.EncryptionKeyId = fileEncryption.GetKeyIdentifierAsString(); data.EncryptionScheme = FileEncryption.SchemeName; data.EncryptionVersion = FileEncryption.SchemeVersion; data.InitializationVector = iv.ToString(CultureInfo.InvariantCulture); } } else if (options.HasFlag(AssetCreationOptions.CommonEncryptionProtected)) { data.IsEncrypted = true; data.EncryptionScheme = CommonEncryption.SchemeName; data.EncryptionVersion = CommonEncryption.SchemeVersion; } else if (options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected)) { data.IsEncrypted = true; data.EncryptionScheme = EnvelopeEncryption.SchemeName; data.EncryptionVersion = EnvelopeEncryption.SchemeVersion; } }
public override Task<IAssetFile> CreateAsync(string name, CancellationToken cancelation) { if (_parentAsset == null) { throw new InvalidOperationException(StringTable.AssetFileCreateParentAssetIsNull); } if (string.IsNullOrWhiteSpace(name)) { throw new ArgumentException(String.Format(CultureInfo.InvariantCulture, StringTable.ErrorCreatingAssetFileEmptyFileName)); } cancelation.ThrowIfCancellationRequested(); IMediaDataServiceContext dataContext = null; AssetFileData assetFile = null; return Task.Factory.StartNew(() => { cancelation.ThrowIfCancellationRequested(); dataContext = MediaContext.MediaServicesClassFactory.CreateDataServiceContext(); FileEncryption fileEncryption = null; // Set a MIME type based on the extension of the file name string mimeType = AssetFileData.GetMimeType(name); assetFile = new AssetFileData { Name = name, ParentAssetId = _parentAsset.Id, MimeType = mimeType, }; try { // Update the files associated with the asset with the encryption related metadata. if (_parentAsset.Options.HasFlag(AssetCreationOptions.StorageEncrypted)) { IContentKey storageEncryptionKey = _parentAsset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault(); cancelation.ThrowIfCancellationRequested(); if (storageEncryptionKey == null) { throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, _parentAsset.Id)); } fileEncryption = new FileEncryption(storageEncryptionKey.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(storageEncryptionKey.Id)); AssetBaseCollection.AddEncryptionMetadataToAssetFile(assetFile, fileEncryption); } else if (_parentAsset.Options.HasFlag(AssetCreationOptions.CommonEncryptionProtected)) { AssetBaseCollection.SetAssetFileForCommonEncryption(assetFile); } else if (_parentAsset.Options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected)) { AssetBaseCollection.SetAssetFileForEnvelopeEncryption(assetFile); } } finally { if (fileEncryption != null) { fileEncryption.Dispose(); } } dataContext.AddObject(FileSet, assetFile); cancelation.ThrowIfCancellationRequested(); cancelation.ThrowIfCancellationRequested(); MediaRetryPolicy retryPolicy = this.MediaContext.MediaServicesClassFactory.GetSaveChangesRetryPolicy(dataContext as IRetryPolicyAdapter); return retryPolicy.ExecuteAsync<IMediaDataServiceResponse>(() => { cancelation.ThrowIfCancellationRequested(); return dataContext.SaveChangesAsync(assetFile); }, cancelation).Result; }, cancelation) .ContinueWith<IAssetFile>(t => { t.ThrowIfFaulted(); AssetFileData data = (AssetFileData)t.Result.AsyncState; return data; }, cancelation); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url, /// then client can be null</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="contentType">Content type of the blob</param> /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "", Func<string> getSharedAccessSignature = null ) { var fs = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); return UploadBlob( url, localFile, fs, fileEncryption, cancellationToken, client, retryPolicy, contentType, subDirectory, getSharedAccessSignature) .ContinueWith(t => fs.Dispose()); }
/// <summary> /// Uploads the destinationPath with given path asynchronously /// </summary> /// <param name="path">The path of a destinationPath to upload</param> /// <param name="blobTransferClient">The <see cref="BlobTransferClient"/> which is used to upload files.</param> /// <param name="locator">An asset <see cref="ILocator"/> which defines permissions associated with the Asset.</param> /// <param name="token">A <see cref="CancellationToken"/> to use for canceling upload operation.</param> /// <returns>A function delegate that returns the future result to be available through the Task.</returns> public Task UploadAsync(string path, BlobTransferClient blobTransferClient, ILocator locator, CancellationToken token) { if (blobTransferClient == null) { throw new ArgumentNullException("blobTransferClient"); } if (locator == null) { throw new ArgumentNullException("locator"); } if (path == null) { throw new ArgumentNullException("path"); } if (!File.Exists(path)) { throw new FileNotFoundException(path); } ValidateFileName(path); IContentKey contentKeyData = null; FileEncryption fileEncryption = null; AssetCreationOptions assetCreationOptions = this.Asset.Options; if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { contentKeyData = this.Asset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault(); if (contentKeyData == null) { throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, this.Asset.Id)); } fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id)); } EventHandler<BlobTransferProgressChangedEventArgs> handler = (s, e) => OnUploadProgressChanged(path, e); blobTransferClient.TransferProgressChanged += handler; return blobTransferClient.UploadBlob( new Uri(locator.Path), path, null, fileEncryption, token, AzureStorageClientRetryPolicyFactory.DefaultPolicy.AsAzureStorageClientRetryPolicy()) .ContinueWith( ts=> { blobTransferClient.TransferProgressChanged -= handler; this.PostUploadAction(ts, path, fileEncryption, assetCreationOptions, token); }); }
private void Cleanup(BlobTransferClient blobTransfer, FileEncryption fileEncryption, ILocator locator, IAccessPolicy accessPolicy) { lock (this._lock) { if (blobTransfer != null) { try { blobTransfer.TransferProgressChanged -= this.OnDownloadBlobTransferProgressChanged; } catch { } finally { blobTransfer = null; } } if (fileEncryption != null) { try { fileEncryption.Dispose(); } catch { } finally { fileEncryption = null; } } if (locator != null) { try { locator.Delete(); } catch { } finally { locator = null; } } if (accessPolicy != null) { try { accessPolicy.Delete(); } catch { } finally { accessPolicy = null; } } } }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subFolder, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read)) { } Exception lastException = null; CloudBlockBlob blob = null; // stats from azurescope show 10 to be an optimal number of transfer threads int numThreads = ParallelTransferThreadCount; var file = new FileInfo(localFile); long fileSize = file.Length; int maxBlockSize = GetBlockSize(fileSize); // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where // the 'key' is block id and 'value' is the block length. List<string> blockList; var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList); int exceptionCount = 0; blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType); blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy }); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } var options = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; // Launch threads to upload blocks. var tasks = new List<Task>(); long bytesSent = 0; Action action = () => { List<Exception> exceptions = new List<Exception>(); if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { Thread.Sleep(_forceSharedAccessSignatureRetry); } if (queue.Count > 0) { FileStream fs = null; try { fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); KeyValuePair<int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); try { var buffer = new byte[blockIdAndLength.Value]; var binaryReader = new BinaryReader(fs); // move the file system reader to the proper position fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value); if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } } using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value)) { string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture)))); string blockHash = GetMd5HashFromStream(buffer); if (blob != null) blob.PutBlock(blockIdString, ms, blockHash, options: options); } Interlocked.Add(ref bytesSent, blockIdAndLength.Value); var progress = (int)((double)bytesSent / file.Length * 100); var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null); OnTaskProgressChanged(eArgs); } catch (StorageException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions); throw lastException; } Thread.Sleep(tm); queue.Enqueue(blockIdAndLength); } catch (IOException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.", exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions); throw lastException; } // dispose existing file stream if (fs != null) { fs.Close(); } Thread.Sleep(tm); // try to reopen the file stream again fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); queue.Enqueue(blockIdAndLength); } } } finally { if (fs != null) { fs.Close(); } } } }; for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew( action, cancellationToken, TaskCreationOptions.AttachedToParent, TaskScheduler.Current)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) => { if (result.Any(t => t.IsFaulted)) { return; } blob.PutBlockList(blockList, options: options); }, TaskContinuationOptions.None).Wait(cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri); }
private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { int numThreads = ParallelTransferThreadCount; List<Exception> exceptions = new List<Exception>(); AggregateException aggregateException = null; long bytesDownloaded = 0; CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy); long blobLength = blob.Properties.Length; int bufferLength = GetBlockSize(blobLength); var queue = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { var tasks = new List<Task>(); Action action = () => { KeyValuePair<long, int> blockOffsetAndLength; int exceptionPerThread = 0; // A buffer to fill per read request. var buffer = new byte[bufferLength]; if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path Thread.Sleep(_forceSharedAccessSignatureRetry); } while (queue.TryDequeue(out blockOffsetAndLength)) { if (cancellationToken.IsCancellationRequested) { break; } try { var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob); using (var response = blobGetRequest.GetResponse() as HttpWebResponse) { if (response != null) { ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded); var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100); // raise the progress changed event var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null); OnTaskProgressChanged(eArgs); } } } catch (Exception ex) { var webEx = ex as WebException; bool ok = (webEx != null) || ex is ObjectDisposedException; if (!ok) { throw; } if (webEx != null) { if (webEx.Response is HttpWebResponse) { var httpex = (HttpWebResponse)webEx.Response; if (httpex.StatusCode == HttpStatusCode.Forbidden) { blob = InitializeCloudBlockBlob(uri, null, retryPolicy); } } } TimeSpan tm; exceptionPerThread++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext())) { aggregateException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptions.Count), exceptions); throw aggregateException; } Thread.Sleep(tm); // Add block back to queue queue.Enqueue(blockOffsetAndLength); } } }; // Launch threads to download chunks. for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew(action)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, aggregateException, BlobTransferType.Download, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.WaitAll(tasks.ToArray(), cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, aggregateException, BlobTransferType.Download, localFile, uri); } }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param> /// <param name="localFile">The full path where file will be saved.</param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="client">The azure client to access a blob.</param> /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns>A task that downloads the specified blob.</returns> public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { SetMaxConnectionLimit(uri); Task task = Task.Factory.StartNew(() => DownloadFileFromBlob(uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy)); return task; }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subDirectory, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature, bool shouldDoFileIO = true) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { } SetConnectionLimits(uri); ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature); BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions)); FileInfo file = new FileInfo(localFile); long fileSize = file.Length; if (fileSize == 0) { blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions); TaskCompletedCallback( cancellationToken.IsCancellationRequested, null, BlobTransferType.Upload, localFile, uri); } else { int numThreads = Environment.ProcessorCount*ParallelUploadDownloadThreadCountMultiplier; int blockSize = GetBlockSize(fileSize); BlobTransferContext transferContext = new BlobTransferContext(); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads); transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.Blob = blob; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.Length = fileSize; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => uploadCompletedSignal.Set(); transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); transferContext.Exceptions = new ConcurrentBag<Exception>(); transferContext.FileEncryption = fileEncryption; transferContext.ContentType = contentType; transferContext.BlobSubFolder = subDirectory; transferContext.NextFileIOBlock = 0; transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>(); using ( FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { RunUploadLoop(transferContext, stream, numThreads); } transferContext.MemoryManager.ReleaseUnusedBuffers(); TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Upload, localFile, uri); } }
private static Action GetEncryptionAction( CancellationToken cancellationToken, FileEncryption fileEncryption, IngestManifestFileData file, string destinationPath, FileInfo fileInfo, ConcurrentQueue <Tuple <int, int> > queue, int maxBlockSize) { Action action = () => { cancellationToken.ThrowIfCancellationRequested(); if (queue.Count > 0) { using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read)) { Tuple <int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); var buffer = new byte[blockIdAndLength.Item2]; var binaryReader = new BinaryReader(fs); // Move the file system reader to the proper position. fs.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2); while (readSize != blockIdAndLength.Item2) { readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize); } bool lockWasTakenForEncode = false; try { Monitor.Enter(fileEncryption, ref lockWasTakenForEncode); using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1 * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } finally { if (lockWasTakenForEncode) { Monitor.Exit(fileEncryption); } } bool lockWasTakenForWrite = false; try { Monitor.Enter(file, ref lockWasTakenForWrite); using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write)) { writeStream.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin); writeStream.Write(buffer, 0, readSize); } } finally { if (lockWasTakenForWrite) { Monitor.Exit(file); } } } } } }; return(action); }
private void AssetEncryptAction(string outputPath, bool overwriteExistingEncryptedFiles, CancellationToken cancellationToken, ConcurrentDictionary <string, IContentKey> keys, IIngestManifestAsset asset) { cancellationToken.ThrowIfCancellationRequested(); List <Task> encryptTasks = new List <Task>(); AssetCreationOptions assetCreationOptions = asset.Asset.Options; if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { IContentKey contentKeyData = keys[asset.Id]; var fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id)); foreach (IngestManifestFileData file in asset.IngestManifestFiles) { ulong iv = Convert.ToUInt64(file.InitializationVector, CultureInfo.InvariantCulture); fileEncryption.SetInitializationVectorForFile(file.Name, iv); FileInfo fileInfo = null; fileInfo = TrackedFilesPaths.ContainsKey(file.Id) ? new FileInfo(TrackedFilesPaths[file.Id]) : new FileInfo(file.Name); string destinationPath = Path.Combine(outputPath, fileInfo.Name); if (File.Exists(destinationPath)) { if (overwriteExistingEncryptedFiles) { File.Delete(destinationPath); } else { throw new IOException(string.Format(CultureInfo.InvariantCulture, StringTable.BulkIngestFileExists, destinationPath)); } } long fileSize = fileInfo.Length; int maxBlockSize = GetBlockSize(fileSize); int numThreads = MaxNumberOfEncryptionThreadsForFilePerCore * Environment.ProcessorCount; ConcurrentQueue <Tuple <int, int> > queue = PrepareUploadQueue(maxBlockSize, fileSize); if (queue.Count < numThreads) { numThreads = queue.Count; } File.Create(destinationPath).Dispose(); Action action = GetEncryptionAction(cancellationToken, fileEncryption, file, destinationPath, fileInfo, queue, maxBlockSize); for (int i = 0; i < numThreads; i++) { encryptTasks.Add(Task.Factory.StartNew((action), cancellationToken)); } } try { Task.WaitAll(encryptTasks.ToArray()); } finally { fileEncryption.Dispose(); } } }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file a needs should be downloaded. If blob has private read permissions then an appropriate SAS url need to be passed.</param> /// <param name="localFile">The full path where file will be saved.</param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used.</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="client">The azure client to access a blob.</param> /// <param name="cancellationToken">The cancellation token to cancel the download operation.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <param name="start">Start pos to download</param> /// <param name="length">Number of bytes to download, -1 to download all.</param> /// <returns>A task that downloads the specified blob.</returns> public virtual Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature = null, long start = 0, long length = -1 ) { if (client != null && getSharedAccessSignature != null) { throw new InvalidOperationException("The arguments client and sharedAccessSignature cannot both be non-null"); } if (start < 0) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Initial offset {0} to download a file must not be negative", start)); } // To download the asset file as a whole or byte-range, the localFile should not be empty if (string.IsNullOrEmpty(localFile)) { throw new ArgumentException("Parameter localFile must be set with a full path"); } if (length < -1) { throw new ArgumentException("Parameter length must be equals or greater than -1"); } if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path Thread.Sleep(_forceSharedAccessSignatureRetry); } BlobDownloader blobDownloader = new BlobDownloader(new MemoryManagerFactory()); blobDownloader.TransferCompleted += (sender, args) => { if (TransferCompleted != null) { TransferCompleted(sender, args); } else if (args.Error != null) { throw args.Error; } }; blobDownloader.TransferProgressChanged += (sender, args) => { if (TransferProgressChanged != null) { TransferProgressChanged(sender, args); } }; return blobDownloader.DownloadBlob( uri, localFile, fileEncryption, initializationVector, client, cancellationToken, retryPolicy, getSharedAccessSignature, start, length, parallelTransferThreadCount: ParallelTransferThreadCount, numberOfConcurrentTransfers: NumberOfConcurrentTransfers); }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded. If blob has private write permissions then /// appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="contentType">Content type of the blob</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, string contentType, FileEncryption fileEncryption, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature = null) { return UploadBlob( url, localFile, fileEncryption, cancellationToken, null, retryPolicy, contentType, getSharedAccessSignature: getSharedAccessSignature) ; }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path where file will be saved </param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="getSharedAccessSignature">A callback function which returns Sas signature for the file to be downloaded</param> /// <returns></returns> public virtual Task DownloadBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature = null ) { return DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy, getSharedAccessSignature); }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path where file will be saved </param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns></returns> public virtual Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { return DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy); }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string name, Stream stream, string contentType, string subDirectory, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature, int parallelTransferThreadCount, bool shouldDoFileIO = true) { BlobTransferContext transferContext = new BlobTransferContext(); transferContext.Exceptions = new ConcurrentBag <Exception>(); try { ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, name, contentType, getSharedAccessSignature); transferContext.Length = stream.Length; transferContext.LocalFilePath = name; transferContext.OnComplete = () => uploadCompletedSignal.Set(); transferContext.Blob = blob; transferContext.FileEncryption = fileEncryption; if (stream.Length == 0) { blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions); } else if (stream.Length < cloudBlockBlobUploadDownloadSizeLimit) { AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition(); OperationContext operationContext = new OperationContext(); operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (var memoryStream = new MemoryStream()) { stream.CopyTo(memoryStream); byte[] fileContent = memoryStream.ToArray(); ApplyEncryptionTransform( transferContext.FileEncryption, Path.GetFileName(transferContext.LocalFilePath), 0, fileContent, Convert.ToInt32(stream.Length)); using (var uploadMemoryStream = new MemoryStream(fileContent)) { blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext); } } InvokeProgressCallback(transferContext, stream.Length, stream.Length); transferContext.OnComplete(); } else { int numThreads = parallelTransferThreadCount; int blockSize = GetBlockSize(stream.Length); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(stream.Length, blockSize, ref numThreads); transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); transferContext.ContentType = contentType; transferContext.BlobSubFolder = subDirectory; transferContext.NextFileIOBlock = 0; transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>(); RunUploadLoop(transferContext, stream, numThreads); } } catch (Exception e) { //Add the exception to the exception list. transferContext.Exceptions.Add(e); } finally { // We should to be able to releaseunusedbuffers if memorymanager was initialized by then if (transferContext.MemoryManager != null) { transferContext.MemoryManager.ReleaseUnusedBuffers(); } //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user. TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Upload, name, uri); } }
/// <summary> /// Uploads file to a blob storage. /// </summary> /// <param name="url">The URL where file needs to be uploaded.If blob has private write permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path of local file.</param> /// <param name="fileEncryption">The file encryption if file needs to be stored encrypted. Pass null if no encryption required</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="client">The client which will be used to upload file. Use client if request need to be signed with client credentials. When upload performed using Sas url, /// then client can be null</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <param name="contentType">Content type of the blob</param> /// <param name="subDirectory">Virtual subdirectory for this file in the blog container.</param> /// <returns></returns> public virtual Task UploadBlob( Uri url, string localFile, FileEncryption fileEncryption, CancellationToken cancellationToken, CloudBlobClient client, IRetryPolicy retryPolicy, string contentType = null, string subDirectory = "") { SetMaxConnectionLimit(url); return Task.Factory.StartNew( () => UploadFileToBlob(cancellationToken, url, localFile, contentType, subDirectory, fileEncryption, client, retryPolicy), cancellationToken); }
public override Task <IAssetFile> CreateAsync(string name, CancellationToken cancelation) { if (_parentAsset == null) { throw new InvalidOperationException(StringTable.AssetFileCreateParentAssetIsNull); } if (string.IsNullOrWhiteSpace(name)) { throw new ArgumentException(String.Format(CultureInfo.InvariantCulture, StringTable.ErrorCreatingAssetFileEmptyFileName)); } cancelation.ThrowIfCancellationRequested(); IMediaDataServiceContext dataContext = null; AssetFileData assetFile = null; return(Task.Factory.StartNew(() => { cancelation.ThrowIfCancellationRequested(); dataContext = MediaContext.MediaServicesClassFactory.CreateDataServiceContext(); FileEncryption fileEncryption = null; // Set a MIME type based on the extension of the file name string mimeType = AssetFileData.GetMimeType(name); assetFile = new AssetFileData { Name = name, ParentAssetId = _parentAsset.Id, MimeType = mimeType, }; try { // Update the files associated with the asset with the encryption related metadata. if (_parentAsset.Options.HasFlag(AssetCreationOptions.StorageEncrypted)) { IContentKey storageEncryptionKey = _parentAsset.ContentKeys.Where(c => c.ContentKeyType == ContentKeyType.StorageEncryption).FirstOrDefault(); cancelation.ThrowIfCancellationRequested(); if (storageEncryptionKey == null) { throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, StringTable.StorageEncryptionContentKeyIsMissing, _parentAsset.Id)); } fileEncryption = new FileEncryption(storageEncryptionKey.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(storageEncryptionKey.Id)); AssetBaseCollection.AddEncryptionMetadataToAssetFile(assetFile, fileEncryption); } else if (_parentAsset.Options.HasFlag(AssetCreationOptions.CommonEncryptionProtected)) { AssetBaseCollection.SetAssetFileForCommonEncryption(assetFile); } else if (_parentAsset.Options.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected)) { AssetBaseCollection.SetAssetFileForEnvelopeEncryption(assetFile); } } finally { if (fileEncryption != null) { fileEncryption.Dispose(); } } dataContext.AddObject(FileSet, assetFile); cancelation.ThrowIfCancellationRequested(); cancelation.ThrowIfCancellationRequested(); MediaRetryPolicy retryPolicy = this.MediaContext.MediaServicesClassFactory.GetSaveChangesRetryPolicy(dataContext as IRetryPolicyAdapter); return retryPolicy.ExecuteAsync <IMediaDataServiceResponse>(() => { cancelation.ThrowIfCancellationRequested(); return dataContext.SaveChangesAsync(assetFile); }, cancelation).Result; }, cancelation) .ContinueWith <IAssetFile>(t => { t.ThrowIfFaulted(); AssetFileData data = (AssetFileData)t.Result.AsyncState; return data; }, cancelation)); }
private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response, KeyValuePair<long, int> blockOffsetAndLength, ref long bytesDownloaded) { using (Stream stream = response.GetResponseStream()) { int offsetInChunk = 0; int remaining = blockOffsetAndLength.Value; while (remaining > 0) { int read = stream.Read(buffer, offsetInChunk, remaining); lock (lockobject) { fs.Position = blockOffsetAndLength.Key + offsetInChunk; if (fileEncryption != null) { lock (fileEncryption) { using ( FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk) ) { encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk); } } } fs.Write(buffer, offsetInChunk, read); } offsetInChunk += read; remaining -= read; Interlocked.Add(ref bytesDownloaded, read); } } return bytesDownloaded; }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subDirectory, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature, int parallelTransferThreadCount, bool shouldDoFileIO = true) { BlobTransferContext transferContext = new BlobTransferContext(); transferContext.Exceptions = new ConcurrentBag<Exception>(); try { ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; //attempt to open the file first so that we throw an exception before getting into the async work using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { } CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature); BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions)); FileInfo file = new FileInfo(localFile); long fileSize = file.Length; transferContext.Length = fileSize; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => uploadCompletedSignal.Set(); transferContext.Blob = blob; transferContext.FileEncryption = fileEncryption; if (fileSize == 0) { blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions); } else if (fileSize < cloudBlockBlobUploadDownloadSizeLimit) { AccessCondition accessCondition = AccessCondition.GenerateEmptyCondition(); OperationContext operationContext = new OperationContext(); operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); using (FileStream fileStream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { using (var memoryStream = new MemoryStream()) { fileStream.CopyTo(memoryStream); byte[] fileContent = memoryStream.ToArray(); ApplyEncryptionTransform( transferContext.FileEncryption, Path.GetFileName(transferContext.LocalFilePath), 0, fileContent, Convert.ToInt32(fileStream.Length)); using (var uploadMemoryStream = new MemoryStream(fileContent)) { blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext); } } } InvokeProgressCallback(transferContext, fileSize, fileSize); transferContext.OnComplete(); } else { int numThreads = parallelTransferThreadCount; int blockSize = GetBlockSize(fileSize); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads); transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); transferContext.ContentType = contentType; transferContext.BlobSubFolder = subDirectory; transferContext.NextFileIOBlock = 0; transferContext.PartialFileIOState = new ConcurrentDictionary<long, int>(); using ( FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { RunUploadLoop(transferContext, stream, numThreads); } } } catch (Exception e) { //Add the exception to the exception list. transferContext.Exceptions.Add(e); } finally { // We should to be able to releaseunusedbuffers if memorymanager was initialized by then if (transferContext.MemoryManager != null) { transferContext.MemoryManager.ReleaseUnusedBuffers(); } //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user. TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Upload, localFile, uri); } }
/// <summary> /// Downloads the specified blob to the specified location. /// </summary> /// <param name="uri">The blob url from which file needs to be downloaded.If blob has private read permissions then appropriate sas url need to be passed</param> /// <param name="localFile">The full path where file will be saved </param> /// <param name="fileEncryption">The file encryption if file has been encrypted. Pass null if no encryption has been used</param> /// <param name="initializationVector">The initialization vector if encryption has been used.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <param name="retryPolicy">The RetryPolicy delegate returns a ShouldRetry delegate, which can be used to implement a custom retry policy.RetryPolicies class can bee used to get default policies</param> /// <returns></returns> public Task DownloadBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CancellationToken cancellationToken, IRetryPolicy retryPolicy) { return(DownloadBlob(uri, localFile, fileEncryption, initializationVector, null, cancellationToken, retryPolicy)); }
private static Action GetEncryptionAction( CancellationToken cancellationToken, FileEncryption fileEncryption, IngestManifestFileData file, string destinationPath, FileInfo fileInfo, ConcurrentQueue<Tuple<int, int>> queue, int maxBlockSize) { Action action = () => { cancellationToken.ThrowIfCancellationRequested(); if (queue.Count > 0) { using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read)) { Tuple<int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); var buffer = new byte[blockIdAndLength.Item2]; var binaryReader = new BinaryReader(fs); // Move the file system reader to the proper position. fs.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2); while (readSize != blockIdAndLength.Item2) { readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize); } bool lockWasTakenForEncode = false; try { Monitor.Enter(fileEncryption, ref lockWasTakenForEncode); using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1*(long) maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } finally { if (lockWasTakenForEncode) Monitor.Exit(fileEncryption); } bool lockWasTakenForWrite = false; try { Monitor.Enter(file, ref lockWasTakenForWrite); using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write)) { writeStream.Seek(blockIdAndLength.Item1*(long) maxBlockSize, SeekOrigin.Begin); writeStream.Write(buffer, 0, readSize); } } finally { if (lockWasTakenForWrite) Monitor.Exit(file); } } } } }; return action; }
private void DownloadFileFromBlob(Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature) { int exceptionCount = 0; int numThreads = ParallelTransferThreadCount; Exception lastException = null; long bytesDownloaded = 0; CloudBlockBlob blob = InitializeCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature); long blobLength = blob.Properties.Length; int bufferLength = GetBlockSize(blobLength); var queue = PrepareDownloadQueue(blobLength, bufferLength, ref numThreads); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Download, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } using (var fs = new FileStream(localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read)) { var tasks = new List <Task>(); Action action = () => { KeyValuePair <long, int> blockOffsetAndLength; int exceptionPerThread = 0; int sasRetry = 0; // A buffer to fill per read request. var buffer = new byte[bufferLength]; if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { // The following sleep is for unit test purpose and we will force the shared access signature to expire and hit retry code path Thread.Sleep(_forceSharedAccessSignatureRetry); } while (queue.TryDequeue(out blockOffsetAndLength)) { if (cancellationToken.IsCancellationRequested) { break; } try { var blobGetRequest = BlobGetRequest(blockOffsetAndLength, blob); using (var response = blobGetRequest.GetResponse() as HttpWebResponse) { if (response != null) { ReadResponseStream(fileEncryption, initializationVector, fs, buffer, response, blockOffsetAndLength, ref bytesDownloaded); var progress = (int)((double)bytesDownloaded / blob.Properties.Length * 100); // raise the progress changed event var eArgs = new BlobTransferProgressChangedEventArgs(bytesDownloaded, blockOffsetAndLength.Value, blob.Properties.Length, progress, _downloadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesDownloaded), uri, localFile, null); OnTaskProgressChanged(eArgs); } } } catch (Exception ex) { var webEx = ex as WebException; bool ok = (webEx != null) || ex is ObjectDisposedException; bool isSasException = false; if (!ok) { throw; } if (webEx != null && getSharedAccessSignature != null) { if (webEx.Response is HttpWebResponse) { var httpex = (HttpWebResponse)webEx.Response; if (httpex.StatusCode == HttpStatusCode.Forbidden) { sasRetry++; if (sasRetry > MaxSasSignatureRetry) { throw; } isSasException = true; } } } if (isSasException) { blob = InitializeCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature); } else { TimeSpan tm; exceptionCount++; exceptionPerThread++; if (!retryPolicy.ShouldRetry(exceptionPerThread, 0, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while downloading. Canceling download.", exceptionCount), ex); break; } Thread.Sleep(tm); } // Add block back to queue queue.Enqueue(blockOffsetAndLength); } } }; // Launch threads to download chunks. for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew(action)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, lastException, BlobTransferType.Download, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.WaitAll(tasks.ToArray(), cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Download, localFile, uri); } }
private void AssetEncryptAction(string outputPath, bool overwriteExistingEncryptedFiles, CancellationToken cancellationToken, ConcurrentDictionary<string, IContentKey> keys, IIngestManifestAsset asset) { cancellationToken.ThrowIfCancellationRequested(); List<Task> encryptTasks = new List<Task>(); AssetCreationOptions assetCreationOptions = asset.Asset.Options; if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { IContentKey contentKeyData = keys[asset.Id]; var fileEncryption = new FileEncryption(contentKeyData.GetClearKeyValue(), EncryptionUtils.GetKeyIdAsGuid(contentKeyData.Id)); foreach (IngestManifestFileData file in asset.IngestManifestFiles) { ulong iv = Convert.ToUInt64(file.InitializationVector, CultureInfo.InvariantCulture); fileEncryption.SetInitializationVectorForFile(file.Name, iv); FileInfo fileInfo = null; fileInfo = TrackedFilesPaths.ContainsKey(file.Id) ? new FileInfo(TrackedFilesPaths[file.Id]) : new FileInfo(file.Name); string destinationPath = Path.Combine(outputPath, fileInfo.Name); if (File.Exists(destinationPath)) { if (overwriteExistingEncryptedFiles) { File.Delete(destinationPath); } else { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, StringTable.BulkIngestFileExists, destinationPath)); } } long fileSize = fileInfo.Length; int maxBlockSize = GetBlockSize(fileSize); int numThreads = MaxNumberOfEncryptionThreadsForFilePerCore*Environment.ProcessorCount; ConcurrentQueue<Tuple<int, int>> queue = PrepareUploadQueue(maxBlockSize, fileSize); if (queue.Count < numThreads) { numThreads = queue.Count; } File.Create(destinationPath).Dispose(); Action action = GetEncryptionAction(cancellationToken, fileEncryption, file, destinationPath, fileInfo, queue, maxBlockSize); for (int i = 0; i < numThreads; i++) { encryptTasks.Add(Task.Factory.StartNew((action), cancellationToken)); } } try { Task.WaitAll(encryptTasks.ToArray()); } finally { fileEncryption.Dispose(); } } }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subFolder, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read)) { } Exception lastException = null; CloudBlockBlob blob = null; // stats from azurescope show 10 to be an optimal number of transfer threads int numThreads = ParallelTransferThreadCount; var file = new FileInfo(localFile); long fileSize = file.Length; int maxBlockSize = GetBlockSize(fileSize); // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where // the 'key' is block id and 'value' is the block length. List <string> blockList; var queue = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList); int exceptionCount = 0; blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); blob.DeleteIfExists(options: new BlobRequestOptions() { RetryPolicy = retryPolicy }); if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } var options = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; // Launch threads to upload blocks. var tasks = new List <Task>(); long bytesSent = 0; Action action = () => { List <Exception> exceptions = new List <Exception>(); int sasRetry = 0; if (_forceSharedAccessSignatureRetry != TimeSpan.Zero) { Thread.Sleep(_forceSharedAccessSignatureRetry); } if (queue.Count > 0) { FileStream fs = null; try { fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); KeyValuePair <int, int> blockIdAndLength; while (queue.TryDequeue(out blockIdAndLength)) { cancellationToken.ThrowIfCancellationRequested(); try { var buffer = new byte[blockIdAndLength.Value]; var binaryReader = new BinaryReader(fs); // move the file system reader to the proper position fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin); int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value); if (fileEncryption != null) { lock (fileEncryption) { using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize)) { encryptor.TransformBlock(buffer, 0, readSize, buffer, 0); } } } using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value)) { string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture)))); string blockHash = GetMd5HashFromStream(buffer); if (blob != null) { blob.PutBlock(blockIdString, ms, blockHash, options: options); } } Interlocked.Add(ref bytesSent, blockIdAndLength.Value); var progress = (int)((double)bytesSent / file.Length * 100); var eArgs = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null); OnTaskProgressChanged(eArgs); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null) { sasRetry++; if (sasRetry > MaxSasSignatureRetry) { throw; } blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); } else { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions); throw lastException; } Thread.Sleep(tm); } queue.Enqueue(blockIdAndLength); } catch (IOException ex) { TimeSpan tm; exceptionCount++; exceptions.Add(ex); if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext())) { lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.", exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions); throw lastException; } // dispose existing file stream if (fs != null) { fs.Close(); } Thread.Sleep(tm); // try to reopen the file stream again fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read); queue.Enqueue(blockIdAndLength); } } } finally { if (fs != null) { fs.Close(); } } } }; for (int idxThread = 0; idxThread < numThreads; idxThread++) { tasks.Add(Task.Factory.StartNew( action, cancellationToken, TaskCreationOptions.AttachedToParent, TaskScheduler.Current)); } if (cancellationToken.IsCancellationRequested) { TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri); cancellationToken.ThrowIfCancellationRequested(); } Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) => { if (result.Any(t => t.IsFaulted)) { return; } try { blob.PutBlockList(blockList, options: options); } catch (StorageException ex) { if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null) { blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature); blob.PutBlockList(blockList, options: options); } else { throw; } } }, TaskContinuationOptions.None).Wait(cancellationToken); TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri); }
private void PostUploadAction(Task task,string path, FileEncryption fileEncryption, AssetCreationOptions assetCreationOptions, CancellationToken token) { try { task.ThrowIfFaulted(); token.ThrowIfCancellationRequested(); FileInfo fileInfo = new FileInfo(path); //Updating Name based on file name to avoid exceptions trying to download file.Mapping to storage account is through file name this.Name = fileInfo.Name; // Set the ContentFileSize base on the local file size this.ContentFileSize = fileInfo.Length; // Update the files associated with the asset with the encryption related metadata. if (assetCreationOptions.HasFlag(AssetCreationOptions.StorageEncrypted)) { AssetBaseCollection.AddEncryptionMetadataToAssetFile(this, fileEncryption); } else if (assetCreationOptions.HasFlag(AssetCreationOptions.CommonEncryptionProtected)) { AssetBaseCollection.SetAssetFileForCommonEncryption(this); } else if (assetCreationOptions.HasFlag(AssetCreationOptions.EnvelopeEncryptionProtected)) { AssetBaseCollection.SetAssetFileForEnvelopeEncryption(this); } this.Update(); } finally { if (fileEncryption != null) { fileEncryption.Dispose(); } } }
private void DownloadFileFromBlob( Uri uri, string localFile, FileEncryption fileEncryption, ulong initializationVector, CloudBlobClient client, CancellationToken cancellationToken, IRetryPolicy retryPolicy, Func<string> getSharedAccessSignature, bool shouldDoFileIO = true, long start = 0, long length = -1, int parallelTransferThreadCount = 10) { int numThreads = Environment.ProcessorCount * parallelTransferThreadCount; ManualResetEvent downloadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy }; CloudBlockBlob blob = null; BlobTransferContext transferContext = new BlobTransferContext(); transferContext.Exceptions = new ConcurrentBag<Exception>(); try { blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature); long initialOffset = start; long sizeToDownload = blob.Properties.Length; if (length != -1) { if (length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length)); } if (start + length > blob.Properties.Length) { throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length)); } sizeToDownload = length; } if (sizeToDownload == 0) { using (FileStream stream = new FileStream( localFile, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { } } else { int blockSize = GetBlockSize(blob.Properties.Length); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset); transferContext.BlocksForFileIO = new ConcurrentDictionary<int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.Blob = blob; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.Length = sizeToDownload; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => downloadCompletedSignal.Set(); transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary<byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); transferContext.FileEncryption = fileEncryption; transferContext.InitializationVector = initializationVector; transferContext.InitialOffset = start; using (FileStream stream = new FileStream( transferContext.LocalFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read )) { stream.SetLength(sizeToDownload); RunDownloadLoop(transferContext, stream, numThreads); } } } catch(Exception e) { //Add the exception to the exception list. transferContext.Exceptions.Add(e); } finally { // We should to be able to releaseunusedbuffers if memorymanager was initialized by then if (transferContext.MemoryManager != null) { transferContext.MemoryManager.ReleaseUnusedBuffers(); } //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user. TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Download, localFile, uri); } }
private void UploadFileToBlob( CancellationToken cancellationToken, Uri uri, string localFile, string contentType, string subDirectory, FileEncryption fileEncryption, CloudBlobClient client, IRetryPolicy retryPolicy, Func <string> getSharedAccessSignature, bool shouldDoFileIO = true) { //attempt to open the file first so that we throw an exception before getting into the async work using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { } SetConnectionLimits(uri); ManualResetEvent uploadCompletedSignal = new ManualResetEvent(false); BlobRequestOptions blobRequestOptions = new BlobRequestOptions { RetryPolicy = retryPolicy, ServerTimeout = TimeSpan.FromSeconds(90) }; CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature); BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions)); FileInfo file = new FileInfo(localFile); long fileSize = file.Length; if (fileSize == 0) { blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions); TaskCompletedCallback( cancellationToken.IsCancellationRequested, null, BlobTransferType.Upload, localFile, uri); } else { int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier; int blockSize = GetBlockSize(fileSize); BlobTransferContext transferContext = new BlobTransferContext(); transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads); transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>(); for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++) { transferContext.BlocksForFileIO[i] = null; } transferContext.BlockSize = blockSize; transferContext.CancellationToken = cancellationToken; transferContext.Blob = blob; transferContext.BlobRequestOptions = blobRequestOptions; transferContext.Length = fileSize; transferContext.LocalFilePath = localFile; transferContext.OnComplete = () => uploadCompletedSignal.Set(); transferContext.MemoryManager = MemoryManagerFactory.GetMemoryManager(blockSize); transferContext.Client = client; transferContext.RetryPolicy = retryPolicy; transferContext.GetSharedAccessSignature = getSharedAccessSignature; transferContext.ShouldDoFileIO = shouldDoFileIO; transferContext.BufferStreams = new ConcurrentDictionary <byte[], MemoryStream>(); transferContext.ClientRequestId = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture); transferContext.Exceptions = new ConcurrentBag <Exception>(); transferContext.FileEncryption = fileEncryption; transferContext.ContentType = contentType; transferContext.BlobSubFolder = subDirectory; transferContext.NextFileIOBlock = 0; transferContext.PartialFileIOState = new ConcurrentDictionary <long, int>(); using ( FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { RunUploadLoop(transferContext, stream, numThreads); } transferContext.MemoryManager.ReleaseUnusedBuffers(); TaskCompletedCallback( cancellationToken.IsCancellationRequested, transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null, BlobTransferType.Upload, localFile, uri); } }