private long ReadResponseStream(FileEncryption fileEncryption, ulong initializationVector, FileStream fs, byte[] buffer, HttpWebResponse response,
                                 KeyValuePair <long, int> blockOffsetAndLength, ref long bytesDownloaded)
 {
     using (Stream stream = response.GetResponseStream())
     {
         int offsetInChunk = 0;
         int remaining     = blockOffsetAndLength.Value;
         while (remaining > 0)
         {
             int read = stream.Read(buffer, offsetInChunk, remaining);
             lock (lockobject)
             {
                 fs.Position = blockOffsetAndLength.Key + offsetInChunk;
                 if (fileEncryption != null)
                 {
                     lock (fileEncryption)
                     {
                         using (
                             FileEncryptionTransform encryptor = fileEncryption.GetTransform(initializationVector, blockOffsetAndLength.Key + offsetInChunk)
                             )
                         {
                             encryptor.TransformBlock(inputBuffer: buffer, inputOffset: offsetInChunk, inputCount: read, outputBuffer: buffer, outputOffset: offsetInChunk);
                         }
                     }
                 }
                 fs.Write(buffer, offsetInChunk, read);
             }
             offsetInChunk += read;
             remaining     -= read;
             Interlocked.Add(ref bytesDownloaded, read);
         }
     }
     return(bytesDownloaded);
 }
Esempio n. 2
0
 protected void ApplyEncryptionTransform(FileEncryption fileEncryption, string fileName, long beginFilePosition, byte[] buffer, int bytesToWrite)
 {
     if (fileEncryption != null)
     {
         lock (fileEncryption)
         {
             using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(fileName, beginFilePosition))
             {
                 encryptor.TransformBlock(inputBuffer: buffer, inputOffset: 0, inputCount: bytesToWrite, outputBuffer: buffer, outputOffset: 0);
             }
         }
     }
 }
        private void UploadFileToBlob(
            CancellationToken cancellationToken,
            Uri uri,
            string localFile,
            string contentType,
            string subFolder,
            FileEncryption fileEncryption,
            CloudBlobClient client,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature)
        {
            //attempt to open the file first so that we throw an exception before getting into the async work
            using (new FileStream(localFile, FileMode.Open, FileAccess.Read))
            {
            }

            Exception      lastException = null;
            CloudBlockBlob blob          = null;
            // stats from azurescope show 10 to be an optimal number of transfer threads
            int  numThreads = ParallelTransferThreadCount;
            var  file       = new FileInfo(localFile);
            long fileSize   = file.Length;

            int maxBlockSize = GetBlockSize(fileSize);

            // Prepare a queue of blocks to be uploaded. Each queue item is a key-value pair where
            // the 'key' is block id and 'value' is the block length.
            List <string> blockList;
            var           queue          = PreapreUploadQueue(maxBlockSize, fileSize, ref numThreads, out blockList);
            int           exceptionCount = 0;

            blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
            blob.DeleteIfExists(options: new BlobRequestOptions()
            {
                RetryPolicy = retryPolicy
            });

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, null, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            var options = new BlobRequestOptions
            {
                RetryPolicy   = retryPolicy,
                ServerTimeout = TimeSpan.FromSeconds(90)
            };

            // Launch threads to upload blocks.
            var    tasks     = new List <Task>();
            long   bytesSent = 0;
            Action action    = () =>
            {
                List <Exception> exceptions = new List <Exception>();
                int sasRetry = 0;

                if (_forceSharedAccessSignatureRetry != TimeSpan.Zero)
                {
                    Thread.Sleep(_forceSharedAccessSignatureRetry);
                }

                if (queue.Count > 0)
                {
                    FileStream fs = null;

                    try
                    {
                        fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);

                        KeyValuePair <int, int> blockIdAndLength;
                        while (queue.TryDequeue(out blockIdAndLength))
                        {
                            cancellationToken.ThrowIfCancellationRequested();

                            try
                            {
                                var buffer       = new byte[blockIdAndLength.Value];
                                var binaryReader = new BinaryReader(fs);

                                // move the file system reader to the proper position
                                fs.Seek(blockIdAndLength.Key * (long)maxBlockSize, SeekOrigin.Begin);
                                int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Value);

                                if (fileEncryption != null)
                                {
                                    lock (fileEncryption)
                                    {
                                        using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Key * (long)maxBlockSize))
                                        {
                                            encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                        }
                                    }
                                }

                                using (var ms = new MemoryStream(buffer, 0, blockIdAndLength.Value))
                                {
                                    string blockIdString = Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format(CultureInfo.InvariantCulture, "BlockId{0}", blockIdAndLength.Key.ToString("0000000", CultureInfo.InvariantCulture))));
                                    string blockHash     = GetMd5HashFromStream(buffer);
                                    if (blob != null)
                                    {
                                        blob.PutBlock(blockIdString, ms, blockHash, options: options);
                                    }
                                }

                                Interlocked.Add(ref bytesSent, blockIdAndLength.Value);
                                var progress = (int)((double)bytesSent / file.Length * 100);
                                var eArgs    = new BlobTransferProgressChangedEventArgs(bytesSent, blockIdAndLength.Value, file.Length, progress, _uploadSpeedCalculator.UpdateCountersAndCalculateSpeed(bytesSent), uri, localFile, null);
                                OnTaskProgressChanged(eArgs);
                            }
                            catch (StorageException ex)
                            {
                                if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
                                {
                                    sasRetry++;
                                    if (sasRetry > MaxSasSignatureRetry)
                                    {
                                        throw;
                                    }
                                    blob = GetCloudBlockBlob(uri, client, subFolder, localFile,
                                                             contentType, getSharedAccessSignature);
                                }
                                else
                                {
                                    TimeSpan tm;
                                    exceptionCount++;
                                    exceptions.Add(ex);
                                    if (!retryPolicy.ShouldRetry(exceptions.Count, ex.RequestInformation.HttpStatusCode, ex, out tm, new OperationContext()))
                                    {
                                        lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while uploading. Canceling upload.", exceptions.Count), exceptions);
                                        throw lastException;
                                    }
                                    Thread.Sleep(tm);
                                }
                                queue.Enqueue(blockIdAndLength);
                            }
                            catch (IOException ex)
                            {
                                TimeSpan tm;
                                exceptionCount++;
                                exceptions.Add(ex);
                                if (!retryPolicy.ShouldRetry(exceptions.Count, 0, ex, out tm, new OperationContext()))
                                {
                                    lastException = new AggregateException(String.Format(CultureInfo.InvariantCulture, "Received {0} exceptions while reading file {1} @ location {2} to be uploaded. Canceling upload.",
                                                                                         exceptions.Count, file.Name, blockIdAndLength.Key * (long)maxBlockSize), exceptions);
                                    throw lastException;
                                }

                                // dispose existing file stream
                                if (fs != null)
                                {
                                    fs.Close();
                                }

                                Thread.Sleep(tm);

                                // try to reopen the file stream again
                                fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read);
                                queue.Enqueue(blockIdAndLength);
                            }
                        }
                    }
                    finally
                    {
                        if (fs != null)
                        {
                            fs.Close();
                        }
                    }
                }
            };

            for (int idxThread = 0; idxThread < numThreads; idxThread++)
            {
                tasks.Add(Task.Factory.StartNew(
                              action,
                              cancellationToken,
                              TaskCreationOptions.AttachedToParent,
                              TaskScheduler.Current));
            }

            if (cancellationToken.IsCancellationRequested)
            {
                TaskCompletedCallback(true, lastException, BlobTransferType.Upload, localFile, uri);
                cancellationToken.ThrowIfCancellationRequested();
            }

            Task.Factory.ContinueWhenAll(tasks.ToArray(), (Task[] result) =>
            {
                if (result.Any(t => t.IsFaulted))
                {
                    return;
                }
                try
                {
                    blob.PutBlockList(blockList, options: options);
                }
                catch (StorageException ex)
                {
                    if (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Forbidden && getSharedAccessSignature != null)
                    {
                        blob = GetCloudBlockBlob(uri, client, subFolder, localFile, contentType, getSharedAccessSignature);
                        blob.PutBlockList(blockList, options: options);
                    }
                    else
                    {
                        throw;
                    }
                }
            }, TaskContinuationOptions.None).Wait(cancellationToken);

            TaskCompletedCallback(cancellationToken.IsCancellationRequested, lastException, BlobTransferType.Upload, localFile, uri);
        }
        private static Action GetEncryptionAction(
            CancellationToken cancellationToken,
            FileEncryption fileEncryption,
            IngestManifestFileData file,
            string destinationPath,
            FileInfo fileInfo,
            ConcurrentQueue <Tuple <int, int> > queue,
            int maxBlockSize)
        {
            Action action = () =>
            {
                cancellationToken.ThrowIfCancellationRequested();

                if (queue.Count > 0)
                {
                    using (var fs = new FileStream(fileInfo.FullName, FileMode.Open, FileAccess.Read))
                    {
                        Tuple <int, int> blockIdAndLength;
                        while (queue.TryDequeue(out blockIdAndLength))
                        {
                            cancellationToken.ThrowIfCancellationRequested();
                            var buffer       = new byte[blockIdAndLength.Item2];
                            var binaryReader = new BinaryReader(fs);

                            // Move the file system reader to the proper position.
                            fs.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
                            int readSize = binaryReader.Read(buffer, 0, blockIdAndLength.Item2);

                            while (readSize != blockIdAndLength.Item2)
                            {
                                readSize += binaryReader.Read(buffer, readSize, blockIdAndLength.Item2 - readSize);
                            }

                            bool lockWasTakenForEncode = false;
                            try
                            {
                                Monitor.Enter(fileEncryption, ref lockWasTakenForEncode);
                                using (FileEncryptionTransform encryptor = fileEncryption.GetTransform(file.Name, blockIdAndLength.Item1 * (long)maxBlockSize))
                                {
                                    encryptor.TransformBlock(buffer, 0, readSize, buffer, 0);
                                }
                            }
                            finally
                            {
                                if (lockWasTakenForEncode)
                                {
                                    Monitor.Exit(fileEncryption);
                                }
                            }

                            bool lockWasTakenForWrite = false;
                            try
                            {
                                Monitor.Enter(file, ref lockWasTakenForWrite);
                                using (var writeStream = new FileStream(destinationPath, FileMode.Open, FileAccess.Write))
                                {
                                    writeStream.Seek(blockIdAndLength.Item1 * (long)maxBlockSize, SeekOrigin.Begin);
                                    writeStream.Write(buffer, 0, readSize);
                                }
                            }
                            finally
                            {
                                if (lockWasTakenForWrite)
                                {
                                    Monitor.Exit(file);
                                }
                            }
                        }
                    }
                }
            };

            return(action);
        }
        private void DownloadFileFromBlob(
            Uri uri,
            string localFile,
            FileEncryption fileEncryption,
            ulong initializationVector,
            CloudBlobClient client,
            CancellationToken cancellationToken,
            IRetryPolicy retryPolicy,
            Func <string> getSharedAccessSignature,
            bool shouldDoFileIO             = true,
            long start                      = 0,
            long length                     = -1,
            int parallelTransferThreadCount = 10)
        {
            ManualResetEvent   downloadCompletedSignal = new ManualResetEvent(false);
            BlobRequestOptions blobRequestOptions      = new BlobRequestOptions {
                RetryPolicy = retryPolicy
            };
            CloudBlockBlob      blob            = null;
            BlobTransferContext transferContext = new BlobTransferContext();

            transferContext.Exceptions = new ConcurrentBag <Exception>();

            try
            {
                blob = GetCloudBlockBlob(uri, client, retryPolicy, getSharedAccessSignature);

                long initialOffset  = start;
                long sizeToDownload = blob.Properties.Length;

                if (length != -1)
                {
                    if (length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                                                                  "Size {0} is beyond the Length of Blob {1}", length, blob.Properties.Length));
                    }

                    if (start + length > blob.Properties.Length)
                    {
                        throw new ArgumentException(string.Format(CultureInfo.CurrentCulture,
                                                                  "Size {0} plus offset {1} is beyond the Length of Blob {2}", length, start,
                                                                  blob.Properties.Length));
                    }

                    sizeToDownload = length;
                }
                transferContext.Length               = sizeToDownload;
                transferContext.LocalFilePath        = localFile;
                transferContext.OnComplete           = () => downloadCompletedSignal.Set();
                transferContext.Blob                 = blob;
                transferContext.FileEncryption       = fileEncryption;
                transferContext.InitializationVector = initializationVector;
                transferContext.InitialOffset        = start;

                if (sizeToDownload == 0)
                {
                    using (FileStream stream =
                               new FileStream(
                                   localFile,
                                   FileMode.OpenOrCreate,
                                   FileAccess.Write,
                                   FileShare.Read
                                   ))
                    {
                    }
                }
                else if (sizeToDownload < cloudBlockBlobUploadDownloadSizeLimit)
                {
                    AccessCondition  accessCondition  = AccessCondition.GenerateEmptyCondition();
                    OperationContext operationContext = new OperationContext();
                    operationContext.ClientRequestID = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
                    using (FileStream fileStream = new FileStream(
                               transferContext.LocalFilePath,
                               FileMode.OpenOrCreate,
                               FileAccess.ReadWrite,
                               FileShare.Read
                               ))
                    {
                        blob.DownloadToStream(fileStream, accessCondition: accessCondition, options: blobRequestOptions, operationContext: operationContext);
                        if (fileEncryption != null)
                        {
                            using (MemoryStream msDecrypt = new MemoryStream())
                            {
                                //Using CryptoTransform APIs per Quintin's suggestion.
                                using (FileEncryptionTransform fileEncryptionTransform = fileEncryption.GetTransform(initializationVector, 0))
                                {
                                    fileStream.Position = 0;
                                    fileStream.CopyTo(msDecrypt);
                                    msDecrypt.Position  = 0;
                                    fileStream.Position = 0;
                                    using (CryptoStream csEncrypt = new CryptoStream(msDecrypt, fileEncryptionTransform, CryptoStreamMode.Read))
                                    {
                                        csEncrypt.CopyTo(fileStream);
                                    }
                                }
                            }
                        }
                    }
                    InvokeProgressCallback(transferContext, sizeToDownload, sizeToDownload);
                    transferContext.OnComplete();
                }
                else
                {
                    int numThreads = parallelTransferThreadCount;
                    int blockSize  = GetBlockSize(blob.Properties.Length);

                    transferContext.BlocksToTransfer = PrepareUploadDownloadQueue(sizeToDownload, blockSize,
                                                                                  ref numThreads, initialOffset);

                    transferContext.BlocksForFileIO = new ConcurrentDictionary <int, byte[]>();
                    for (int i = 0; i < transferContext.BlocksToTransfer.Count(); i++)
                    {
                        transferContext.BlocksForFileIO[i] = null;
                    }
                    transferContext.BlockSize                = blockSize;
                    transferContext.CancellationToken        = cancellationToken;
                    transferContext.BlobRequestOptions       = blobRequestOptions;
                    transferContext.MemoryManager            = MemoryManagerFactory.GetMemoryManager(blockSize);
                    transferContext.Client                   = client;
                    transferContext.RetryPolicy              = retryPolicy;
                    transferContext.GetSharedAccessSignature = getSharedAccessSignature;
                    transferContext.ShouldDoFileIO           = shouldDoFileIO;
                    transferContext.BufferStreams            = new ConcurrentDictionary <byte[], MemoryStream>();
                    transferContext.ClientRequestId          = DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);

                    using (FileStream stream = new FileStream(
                               transferContext.LocalFilePath,
                               FileMode.OpenOrCreate,
                               FileAccess.Write,
                               FileShare.Read
                               ))
                    {
                        stream.SetLength(sizeToDownload);
                        RunDownloadLoop(transferContext, stream, numThreads);
                    }
                }
            }
            catch (Exception e)
            {
                //Add the exception to the exception list.
                transferContext.Exceptions.Add(e);
            }
            finally
            {
                // We should to be able to releaseunusedbuffers if memorymanager was initialized by then
                if (transferContext.MemoryManager != null)
                {
                    transferContext.MemoryManager.ReleaseUnusedBuffers();
                }
                //TaskCompletedCallback should be called to populate exceptions if relevant and other eventargs for the user.
                TaskCompletedCallback(
                    cancellationToken.IsCancellationRequested,
                    transferContext.Exceptions != null && transferContext.Exceptions.Count > 0
                            ? new AggregateException(transferContext.Exceptions)
                            : null,
                    BlobTransferType.Download,
                    localFile,
                    uri);
            }
        }