Beispiel #1
0
 protected BlobTransferBase(MemoryManagerFactory memoryManagerFactory)
 {
     if (memoryManagerFactory == null)
     {
         throw new ArgumentNullException("memoryManagerFactory");
     }
     MemoryManagerFactory = memoryManagerFactory;
 }
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subDirectory,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            bool shouldDoFileIO = true)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
            {
            }

            SetConnectionLimits(uri);

            ManualResetEvent   uploadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions    = new BlobRequestOptions
            {
                RetryPolicy   = retryPolicy,
                ServerTimeout = TimeSpan.FromSeconds(90)
            };

            CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, localFile, contentType, getSharedAccessSignature);

            BlobPolicyActivationWait(() => blob.DeleteIfExists(options: blobRequestOptions));

            FileInfo file     = new FileInfo(localFile);
            long     fileSize = file.Length;

            if (fileSize == 0)
            {
                blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
            else
            {
                int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
                int blockSize  = GetBlockSize(fileSize);

                BlobTransferContext transferContext = new BlobTransferContext();
                transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(fileSize, blockSize, ref numThreads);

                transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                {
                    transferContext.BlocksForFileIO[i] = null;
                }
                transferContext.BlockSize         = blockSize;
                transferContext.CancellationToken = cancellationToken;
                transferContext.Blob = blob;
                transferContext.BlobRequestOptions       = blobRequestOptions;
                transferContext.Length                   = fileSize;
                transferContext.LocalFilePath            = localFile;
                transferContext.OnComplete               = () => uploadCompletedSignal.Set();
                transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                transferContext.Client                   = client;
                transferContext.RetryPolicy              = retryPolicy;
                transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                transferContext.ShouldDoFileIO           = shouldDoFileIO;
                transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                transferContext.Exceptions               = new ConcurrentBag <Exception>();
                transferContext.FileEncryption           = fileEncryption;
                transferContext.ContentType              = contentType;
                transferContext.BlobSubFolder            = subDirectory;
                transferContext.NextFileIOBlock          = 0;
                transferContext.PartialFileIOState       = new ConcurrentDictionary <long, int>();

                using (
                    FileStream stream = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                {
                    RunUploadLoop(transferContext, stream, numThreads);
                }

                transferContext.MemoryManager.ReleaseUnusedBuffers();

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
                    BlobTransferType.Upload,
                    localFile,
                    uri);
            }
        }
 public BlobUploader(MemoryManagerFactory memoryManagerFactory) : base(memoryManagerFactory)
 {
 }
 public BlobUploader(MemoryManagerFactory memoryManagerFactory)
     : base(memoryManagerFactory)
 {
 }
Beispiel #5
0
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string name,
            Stream stream,
            string contentType,
            string subDirectory,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            int parallelTransferThreadCount,
            bool shouldDoFileIO = true)
        {
            BlobTransferContext transferContext = new BlobTransferContext();

            transferContext.Exceptions = new ConcurrentBag <Exception>();
            try
            {
                ManualResetEvent   uploadCompletedSignal = new ManualResetEvent(false);
                BlobRequestOptions blobRequestOptions    = new BlobRequestOptions
                {
                    RetryPolicy   = retryPolicy,
                    ServerTimeout = TimeSpan.FromSeconds(90)
                };

                CloudBlockBlob blob = GetCloudBlockBlob(uri, client, subDirectory, name, contentType, getSharedAccessSignature);

                transferContext.Length         = stream.Length;
                transferContext.LocalFilePath  = name;
                transferContext.OnComplete     = () => uploadCompletedSignal.Set();
                transferContext.Blob           = blob;
                transferContext.FileEncryption = fileEncryption;
                if (stream.Length == 0)
                {
                    blob.UploadFromByteArray(new byte[1], 0, 0, options: blobRequestOptions);
                }
                else if (stream.Length < cloudBlockBlobUploadDownloadSizeLimit)
                {
                    AccessCondition  accessCondition  = AccessCondition.GenerateEmptyCondition();
                    OperationContext operationContext = new OperationContext();
                    operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);

                    using (var memoryStream = new MemoryStream())
                    {
                        stream.CopyTo(memoryStream);
                        byte[] fileContent = memoryStream.ToArray();
                        ApplyEncryptionTransform(
                            transferContext.FileEncryption,
                            Path.GetFileName(transferContext.LocalFilePath),
                            0,
                            fileContent,
                            Convert.ToInt32(stream.Length));

                        using (var uploadMemoryStream = new MemoryStream(fileContent))
                        {
                            blob.UploadFromStream(uploadMemoryStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
                        }
                    }
                    InvokeProgressCallback(transferContext, stream.Length, stream.Length);
                    transferContext.OnComplete();
                }
                else
                {
                    int numThreads = parallelTransferThreadCount;
                    int blockSize  = GetBlockSize(stream.Length);

                    transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(stream.Length, blockSize, ref numThreads);

                    transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                    for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                    {
                        transferContext.BlocksForFileIO[i] = null;
                    }
                    transferContext.BlockSize                = blockSize;
                    transferContext.CancellationToken        = cancellationToken;
                    transferContext.BlobRequestOptions       = blobRequestOptions;
                    transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                    transferContext.Client                   = client;
                    transferContext.RetryPolicy              = retryPolicy;
                    transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                    transferContext.ShouldDoFileIO           = shouldDoFileIO;
                    transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                    transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                    transferContext.ContentType              = contentType;
                    transferContext.BlobSubFolder            = subDirectory;
                    transferContext.NextFileIOBlock          = 0;
                    transferContext.PartialFileIOState       = new ConcurrentDictionary <long, int>();

                    RunUploadLoop(transferContext, stream, numThreads);
                }
            }
            catch (Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();
                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                        ? new AggregateException(transferContext.Exceptions)
                        : null,
                    BlobTransferType.Upload,
                    name,
                    uri);
            }
        }
Beispiel #6
0
        private void DownloadFileFromBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            bool shouldDoFileIO = true,
            long start          = 0,
            long length         = -1)
        {
            int numThreads = Environment.ProcessorCount * ParallelUploadDownloadThreadCountMultiplier;
            ManualResetEvent   downloadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions      = new BlobRequestOptions {
                RetryPolicy = retryPolicy
            };

            CloudBlockBlob blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);

            long initialOffset  = start;
            long sizeToDownload = blob.Properties.Length;

            if (length != -1)
            {
                if (length > blob.Properties.Length)
                {
                    throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
                }

                if (start + length > blob.Properties.Length)
                {
                    throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start, blob.Properties.Length));
                }

                sizeToDownload = length;
            }

            if (sizeToDownload == 0)
            {
                using (FileStream stream =
                           new FileStream(
                               localFile,
                               FileMode.OpenOrCreate,
                               FileAccess.Write,
                               FileShare.Read
                               ))
                {
                }

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    null,
                    BlobTransferType.Download,
                    localFile,
                    uri);
            }
            else
            {
                int blockSize = GetBlockSize(blob.Properties.Length);

                BlobTransferContext transferContext = new BlobTransferContext();

                transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize, ref numThreads, initialOffset);

                transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                {
                    transferContext.BlocksForFileIO[i] = null;
                }
                transferContext.BlockSize         = blockSize;
                transferContext.CancellationToken = cancellationToken;
                transferContext.Blob = blob;
                transferContext.BlobRequestOptions = blobRequestOptions;
                transferContext.Length             = sizeToDownload;

                transferContext.LocalFilePath            = localFile;
                transferContext.OnComplete               = () => downloadCompletedSignal.Set();
                transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                transferContext.Client                   = client;
                transferContext.RetryPolicy              = retryPolicy;
                transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                transferContext.ShouldDoFileIO           = shouldDoFileIO;
                transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                transferContext.Exceptions               = new ConcurrentBag <Exception>();
                transferContext.FileEncryption           = fileEncryption;
                transferContext.InitializationVector     = initializationVector;
                transferContext.InitialOffset            = start;

                using (FileStream stream = new FileStream(
                           transferContext.LocalFilePath,
                           FileMode.OpenOrCreate,
                           FileAccess.Write,
                           FileShare.Read
                           ))
                {
                    stream.SetLength(sizeToDownload);
                    RunDownloadLoop(transferContext, stream, numThreads);
                }

                transferContext.MemoryManager.ReleaseUnusedBuffers();

                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0 ? new AggregateException(transferContext.Exceptions) : null,
                    BlobTransferType.Download,
                    localFile,
                    uri);
            }
        }
        private void DownloadFileFromBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            bool shouldDoFileIO             = true,
            long start                      = 0,
            long length                     = -1,
            int parallelTransferThreadCount = 10)
        {
            ManualResetEvent   downloadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions      = new BlobRequestOptions {
                RetryPolicy = retryPolicy
            };
            CloudBlockBlob      blob            = null;
            BlobTransferContext transferContext = new BlobTransferContext();

            transferContext.Exceptions = new ConcurrentBag <Exception>();

            try
            {
                blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);

                long initialOffset  = start;
                long sizeToDownload = blob.Properties.Length;

                if (length != -1)
                {
                    if (length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                                                                  "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
                    }

                    if (start + length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                                                                  "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
                                                                  blob.Properties.Length));
                    }

                    sizeToDownload = length;
                }
                transferContext.Length               = sizeToDownload;
                transferContext.LocalFilePath        = localFile;
                transferContext.OnComplete           = () => downloadCompletedSignal.Set();
                transferContext.Blob                 = blob;
                transferContext.FileEncryption       = fileEncryption;
                transferContext.InitializationVector = initializationVector;
                transferContext.InitialOffset        = start;

                if (sizeToDownload == 0)
                {
                    using (FileStream stream =
                               new FileStream(
                                   localFile,
                                   FileMode.OpenOrCreate,
                                   FileAccess.Write,
                                   FileShare.Read
                                   ))
                    {
                    }
                }
                else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit)
                {
                    AccessCondition  accessCondition  = AccessCondition.GenerateEmptyCondition();
                    OperationContext operationContext = new OperationContext();
                    operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                    using (FileStream fileStream = new FileStream(
                               transferContext.LocalFilePath,
                               FileMode.OpenOrCreate,
                               FileAccess.ReadWrite,
                               FileShare.Read
                               ))
                    {
                        blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
                        if (fileEncryption != null)
                        {
                            using (MemoryStream msDecrypt = new MemoryStream())
                            {
                                //Using CryptoTransform APIs per Quintin's suggestion.
                                using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0))
                                {
                                    fileStream.Position = 0;
                                    fileStream.CopyTo(msDecrypt);
                                    msDecrypt.Position  = 0;
                                    fileStream.Position = 0;
                                    using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read))
                                    {
                                        csEncrypt.CopyTo(fileStream);
                                    }
                                }
                            }
                        }
                    }
                    InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload);
                    transferContext.OnComplete();
                }
                else
                {
                    int numThreads = parallelTransferThreadCount;
                    int blockSize  = GetBlockSize(blob.Properties.Length);

                    transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
                                                                                  ref numThreads, initialOffset);

                    transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                    for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                    {
                        transferContext.BlocksForFileIO[i] = null;
                    }
                    transferContext.BlockSize                = blockSize;
                    transferContext.CancellationToken        = cancellationToken;
                    transferContext.BlobRequestOptions       = blobRequestOptions;
                    transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                    transferContext.Client                   = client;
                    transferContext.RetryPolicy              = retryPolicy;
                    transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                    transferContext.ShouldDoFileIO           = shouldDoFileIO;
                    transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                    transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);

                    using (FileStream stream = new FileStream(
                               transferContext.LocalFilePath,
                               FileMode.OpenOrCreate,
                               FileAccess.Write,
                               FileShare.Read
                               ))
                    {
                        stream.SetLength(sizeToDownload);
                        RunDownloadLoop(transferContext, stream, numThreads);
                    }
                }
            }
            catch (Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();
                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                            ? new AggregateException(transferContext.Exceptions)
                            : null,
                    BlobTransferType.Download,
                    localFile,
                    uri);
            }
        }