public async Task<bool> UploadMultipartFileAsync(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo, CancellationTokenSource cts, CancellationToken token, IProgress<Tuple<string, long>> progress) { token.ThrowIfCancellationRequested(); _log.Debug("Called UploadMultipartFileAsync with ArchiveFileInfo properties: KeyName = \"" + fileInfo.KeyName + "\", FilePath = \"" + fileInfo.FilePath + "\"."); bool hasException = false; this.IsUploading = true; List<PartDetail> uploadedParts = new List<PartDetail>(); Dictionary<int, long> multipartUploadProgress = new Dictionary<int, long>(); var uploadPartTasks = new List<Task<UploadPartResponse>>(); Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>(); List<Task<UploadPartResponse>> runningTasks = new List<Task<UploadPartResponse>>(); try { token.ThrowIfCancellationRequested(); // if this isn't a new file upload (resuming a past upload) // then we have to get the uploaded parts so the upload continues // where it's stopped. if (!isNewFileUpload) uploadedParts = await this.ListPartsAsync(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, token).ConfigureAwait(false); token.ThrowIfCancellationRequested(); // create all part requests. partRequests = this.PreparePartRequests(isNewFileUpload, existingBucketName, fileInfo, uploadedParts, multipartUploadProgress); token.ThrowIfCancellationRequested(); var parallelLimit = (MAX_PARALLEL_ALLOWED > 1) ? MAX_PARALLEL_ALLOWED : 2; // initialize first tasks to run. while (runningTasks.Count < parallelLimit && partRequests.Count > 0) { var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress); runningTasks.Add(uploadTask); } while (runningTasks.Count > 0) { token.ThrowIfCancellationRequested(); var finishedTask = await Task<UploadPartResponse>.WhenAny(runningTasks).ConfigureAwait(false); runningTasks.Remove(finishedTask); if (finishedTask.Status == TaskStatus.Faulted) throw finishedTask.Exception; if (partRequests.Count > 0) { token.ThrowIfCancellationRequested(); var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress); runningTasks.Add(uploadTask); } finishedTask = null; } token.ThrowIfCancellationRequested(); // if all goes well, return true return true; } catch (Exception) { hasException = true; partRequests.Clear(); runningTasks.Clear(); multipartUploadProgress.Clear(); uploadedParts.Clear(); uploadPartTasks.Clear(); throw; } finally { if (hasException && cts != null && !cts.IsCancellationRequested) { cts.Cancel(); } uploadedParts.Clear(); multipartUploadProgress.Clear(); uploadPartTasks.Clear(); partRequests.Clear(); runningTasks.Clear(); this.IsUploading = false; } }
/// <summary> /// Create UploadPartRequest objects for a multipart upload. /// </summary> /// <param name="isNewFileUpload"></param> /// <param name="existingBucketName"></param> /// <param name="fileInfo"></param> /// <param name="uploadedParts"></param> /// <param name="partsProgress"></param> /// <param name="token"></param> /// <returns></returns> public Queue<UploadPartRequest> PreparePartRequests(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo, List<PartDetail> uploadedParts, Dictionary<int, long> partsProgress) { long filePosition = 0; long partSize = PART_SIZE; // 5 MB Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>(); if (fileInfo.Size < partSize) partSize = fileInfo.Size; for (int i = 1; filePosition < fileInfo.Size; i++) { var uploadedPart = uploadedParts.Where(x => x.PartNumber == i).FirstOrDefault(); if (uploadedPart != null) // uploadedParts.Select(x => x.PartNumber).Contains(i)) { // for each already uploaded part, add total part size as transferred bytes. partsProgress.Add(i, uploadedPart.Size); // move the file position filePosition += uploadedPart.Size; continue; } // for each NOT uploaded part, add a pair in the progress dictionary with value = 0 partsProgress.Add(i, 0); bool isLastPart = false; // check remaining size and if it's smalled than partSize // then set partSize = remainingSize and this as the last part in its request. long remainingSize = fileInfo.Size - filePosition; if (remainingSize <= partSize) { isLastPart = true; partSize = remainingSize; } // Create request to upload a part. var uploadPartRequest = CreateUploadPartRequest(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, i, partSize, filePosition, fileInfo.FilePath); // Now check if this is the last part and mark the request. if (isLastPart) uploadPartRequest.IsLastPart = true; partRequests.Enqueue(uploadPartRequest); // increment position by partSize filePosition += partSize; } return partRequests; }