public async Task<bool> UploadMultipartFileAsync(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo,
CancellationTokenSource cts, CancellationToken token, IProgress<Tuple<string, long>> progress)
{
token.ThrowIfCancellationRequested();
_log.Debug("Called UploadMultipartFileAsync with ArchiveFileInfo properties: KeyName = \"" + fileInfo.KeyName +
"\", FilePath = \"" + fileInfo.FilePath + "\".");
bool hasException = false;
this.IsUploading = true;
List<PartDetail> uploadedParts = new List<PartDetail>();
Dictionary<int, long> multipartUploadProgress = new Dictionary<int, long>();
var uploadPartTasks = new List<Task<UploadPartResponse>>();
Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>();
List<Task<UploadPartResponse>> runningTasks = new List<Task<UploadPartResponse>>();
try
{
token.ThrowIfCancellationRequested();
// if this isn't a new file upload (resuming a past upload)
// then we have to get the uploaded parts so the upload continues
// where it's stopped.
if (!isNewFileUpload)
uploadedParts = await this.ListPartsAsync(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, token).ConfigureAwait(false);
token.ThrowIfCancellationRequested();
// create all part requests.
partRequests = this.PreparePartRequests(isNewFileUpload, existingBucketName, fileInfo, uploadedParts, multipartUploadProgress);
token.ThrowIfCancellationRequested();
var parallelLimit = (MAX_PARALLEL_ALLOWED > 1) ? MAX_PARALLEL_ALLOWED : 2;
// initialize first tasks to run.
while (runningTasks.Count < parallelLimit && partRequests.Count > 0)
{
var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress);
runningTasks.Add(uploadTask);
}
while (runningTasks.Count > 0)
{
token.ThrowIfCancellationRequested();
var finishedTask = await Task<UploadPartResponse>.WhenAny(runningTasks).ConfigureAwait(false);
runningTasks.Remove(finishedTask);
if (finishedTask.Status == TaskStatus.Faulted)
throw finishedTask.Exception;
if (partRequests.Count > 0)
{
token.ThrowIfCancellationRequested();
var uploadTask = this.UploadPartAsync(partRequests.Dequeue(), multipartUploadProgress, token, progress);
runningTasks.Add(uploadTask);
}
finishedTask = null;
}
token.ThrowIfCancellationRequested();
// if all goes well, return true
return true;
}
catch (Exception)
{
hasException = true;
partRequests.Clear();
runningTasks.Clear();
multipartUploadProgress.Clear();
uploadedParts.Clear();
uploadPartTasks.Clear();
throw;
}
finally
{
if (hasException && cts != null && !cts.IsCancellationRequested)
{
cts.Cancel();
}
uploadedParts.Clear();
multipartUploadProgress.Clear();
uploadPartTasks.Clear();
partRequests.Clear();
runningTasks.Clear();
this.IsUploading = false;
}
}
/// <summary>
/// Create UploadPartRequest objects for a multipart upload.
/// </summary>
/// <param name="isNewFileUpload"></param>
/// <param name="existingBucketName"></param>
/// <param name="fileInfo"></param>
/// <param name="uploadedParts"></param>
/// <param name="partsProgress"></param>
/// <param name="token"></param>
/// <returns></returns>
public Queue<UploadPartRequest> PreparePartRequests(bool isNewFileUpload, string existingBucketName, ArchiveFileInfo fileInfo,
List<PartDetail> uploadedParts, Dictionary<int, long> partsProgress)
{
long filePosition = 0;
long partSize = PART_SIZE; // 5 MB
Queue<UploadPartRequest> partRequests = new Queue<UploadPartRequest>();
if (fileInfo.Size < partSize)
partSize = fileInfo.Size;
for (int i = 1; filePosition < fileInfo.Size; i++)
{
var uploadedPart = uploadedParts.Where(x => x.PartNumber == i).FirstOrDefault();
if (uploadedPart != null) // uploadedParts.Select(x => x.PartNumber).Contains(i))
{
// for each already uploaded part, add total part size as transferred bytes.
partsProgress.Add(i, uploadedPart.Size);
// move the file position
filePosition += uploadedPart.Size;
continue;
}
// for each NOT uploaded part, add a pair in the progress dictionary with value = 0
partsProgress.Add(i, 0);
bool isLastPart = false;
// check remaining size and if it's smalled than partSize
// then set partSize = remainingSize and this as the last part in its request.
long remainingSize = fileInfo.Size - filePosition;
if (remainingSize <= partSize)
{
isLastPart = true;
partSize = remainingSize;
}
// Create request to upload a part.
var uploadPartRequest =
CreateUploadPartRequest(existingBucketName, fileInfo.KeyName, fileInfo.UploadId, i, partSize, filePosition, fileInfo.FilePath);
// Now check if this is the last part and mark the request.
if (isLastPart)
uploadPartRequest.IsLastPart = true;
partRequests.Enqueue(uploadPartRequest);
// increment position by partSize
filePosition += partSize;
}
return partRequests;
}