Exemplo n.º 1
0
        public async Task<bool> UploadMultipartFileAsync(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo,
            CancellationTokenSource cts, CancellationToken token, IProgress<Tuple<string, long>> progress)
        {
            token.ThrowIfCancellationRequested();

            _log.Debug("Called UploadMultipartFileAsync with ArchiveFileInfo properties: KeyName = \"" + fileInfo.KeyName +
                "\", FilePath = \"" + fileInfo.FilePath + "\".");

            bool hasException = false;
            this.IsUploading = true;

            List<PartDetail> uploadedParts = new List<PartDetail>();
            Dictionary<int, long> multipartUploadProgress = new Dictionary<int, long>();

            var uploadPartTasks = new List<Task<UploadPartResponse>>();
            Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>();
            List<Task<UploadPartResponse>> runningTasks = new List<Task<UploadPartResponse>>();

            try
            {
                token.ThrowIfCancellationRequested();

                // if this isn't a new file upload (resuming a past upload)
                // then we have to get the uploaded parts so the upload continues
                // where it's stopped.
                if (!isNewFileUpload)
                    uploadedParts = await this.ListPartsAsync(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, token).ConfigureAwait(false);

                token.ThrowIfCancellationRequested();

                // create all part requests.
                partRequests = this.PreparePartRequests(isNewFileUpload, existingBucketName, fileInfo, uploadedParts, multipartUploadProgress);

                token.ThrowIfCancellationRequested();

                var parallelLimit = (MAX_PARALLEL_ALLOWED > 1) ? MAX_PARALLEL_ALLOWED : 2;

                // initialize first tasks to run.
                while (runningTasks.Count < parallelLimit && partRequests.Count > 0)
                {
                    var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress);
                    runningTasks.Add(uploadTask);
                }

                while (runningTasks.Count > 0)
                {
                    token.ThrowIfCancellationRequested();

                    var finishedTask = await Task<UploadPartResponse>.WhenAny(runningTasks).ConfigureAwait(false);

                    runningTasks.Remove(finishedTask);

                    if (finishedTask.Status == TaskStatus.Faulted)
                        throw finishedTask.Exception;

                    if (partRequests.Count > 0)
                    {
                        token.ThrowIfCancellationRequested();

                        var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress);
                        runningTasks.Add(uploadTask);
                    }

                    finishedTask = null;
                }

                token.ThrowIfCancellationRequested();

                // if all goes well, return true
                return true;
            }
            catch (Exception)
            {
                hasException = true;

                partRequests.Clear();
                runningTasks.Clear();
                multipartUploadProgress.Clear();
                uploadedParts.Clear();
                uploadPartTasks.Clear();

                throw;
            }
            finally
            {
                if (hasException && cts != null && !cts.IsCancellationRequested)
                {
                    cts.Cancel();
                }

                uploadedParts.Clear();
                multipartUploadProgress.Clear();
                uploadPartTasks.Clear();
                partRequests.Clear();
                runningTasks.Clear();

                this.IsUploading = false;
            }
        }
Exemplo n.º 2
0
        /// <summary>
        /// Create UploadPartRequest objects for a multipart upload.
        /// </summary>
        /// <param name="isNewFileUpload"></param>
        /// <param name="existingBucketName"></param>
        /// <param name="fileInfo"></param>
        /// <param name="uploadedParts"></param>
        /// <param name="partsProgress"></param>
        /// <param name="token"></param>
        /// <returns></returns>
        public Queue<UploadPartRequest> PreparePartRequests(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo, 
            List<PartDetail> uploadedParts, Dictionary<int, long> partsProgress)
        {
            long filePosition = 0;
            long partSize = PART_SIZE; // 5 MB
            Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>();

            if (fileInfo.Size < partSize)
                partSize = fileInfo.Size;

            for (int i = 1; filePosition < fileInfo.Size; i++)
            {
                var uploadedPart = uploadedParts.Where(x => x.PartNumber == i).FirstOrDefault();
                if (uploadedPart != null) // uploadedParts.Select(x => x.PartNumber).Contains(i))
                {
                    // for each already uploaded part, add total part size as transferred bytes.
                    partsProgress.Add(i, uploadedPart.Size); 
                    // move the file position
                    filePosition += uploadedPart.Size;
                    continue;
                }

                // for each NOT uploaded part, add a pair in the progress dictionary with value = 0
                partsProgress.Add(i, 0);

                bool isLastPart = false;

                // check remaining size and if it's smalled than partSize
                // then set partSize = remainingSize and this as the last part in its request.
                long remainingSize = fileInfo.Size - filePosition;
                if (remainingSize <= partSize)
                {
                    isLastPart = true;
                    partSize = remainingSize;
                }

                // Create request to upload a part.
                var uploadPartRequest =
                    CreateUploadPartRequest(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, i, partSize, filePosition, fileInfo.FilePath);

                // Now check if this is the last part and mark the request.
                if (isLastPart)
                    uploadPartRequest.IsLastPart = true;

                partRequests.Enqueue(uploadPartRequest);

                // increment position by partSize
                filePosition += partSize;
            }

            return partRequests;
        }