private async Task <List <Task> > Dispatch(FilePartSource partSource, string chunkUploadUrl, string fileName, CancellationToken cancellationToken = default(CancellationToken)) { var incrementLock = new object(); long currentPartSize = partConfig.InitialPartSize; var partSizeCalc = new PartSizeCalculator(concurrentWorkers, partConfig); Func <FilePart, Task> attemptPartUpload = async part => { IStopwatch timer = Stopwatch.StartNew(); try { await UploadPart(chunkUploadUrl, part, fileName, cancellationToken).ConfigureAwait(false); } finally { timer.Stop(); } lock (incrementLock) { currentPartSize = partSizeCalc.NextPartSize(currentPartSize, part.Bytes.Length, timer.Elapsed); } }; var workerTasks = new List <Task>(); try { var activeWorkers = new AsyncSemaphore(concurrentWorkers); bool giveUp = false; while (!giveUp && partSource.HasMore) { if (cancellationToken.IsCancellationRequested) { throw new UploadException( "Upload was cancelled", UploadStatusCode.Cancelled, new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded)); } await activeWorkers.WaitAsync().ConfigureAwait(false); if (giveUp) { return(workerTasks); } var part = await partSource.GetNextPart(Interlocked.Read(ref currentPartSize)).ConfigureAwait(false); var task = Task.Run(async() => { try { await AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount).ConfigureAwait(false); } catch { giveUp = true; throw; } finally { activeWorkers.Release(); part.Bytes.Dispose(); } }); workerTasks.Add(task); } return(workerTasks); } catch { ObserveExceptions(workerTasks); throw; } }
private async Task UploadChunks(MemoryMappedFile memoryMappedFile, PartSizeCalculator partSizeCalc, int workerCount, CancellationToken cancellationToken) { var hashWorkers = new List <Task <Chunk> >(workerCount); var chunkQueue = new Queue <Chunk>(workerCount); var uploadWorkers = new List <Task <ChunkUploadTime> >(workerCount); int chunkIndex = 0; long nextChunkSize = Config.PartConfig.InitialPartSize; long positionInFile = initialPosition; long bytesRemaining = UploadSpecificationRequest.FileSize - initialPosition; try { while ((bytesRemaining > 0 || hashWorkers.Count > 0 || chunkQueue.Count > 0 || uploadWorkers.Count > 0) && !cancellationToken.IsCancellationRequested) { await TryPauseAsync(cancellationToken).ConfigureAwait(false); if (bytesRemaining > 0 && hashWorkers.Count < workerCount) { long chunkSize = Math.Min(nextChunkSize, bytesRemaining); hashWorkers.Add(HashWorker(memoryMappedFile, chunkIndex, positionInFile, chunkSize, cancellationToken)); positionInFile += chunkSize; bytesRemaining -= chunkSize; chunkIndex += 1; } else if (chunkQueue.Count > 0 && uploadWorkers.Count < workerCount) { uploadWorkers.Add(UploadWorker(chunkQueue.Dequeue(), progressReporter.ChunkProgressReporter(), cancellationToken)); } else if (uploadWorkers.Count > 0 || hashWorkers.Count > 0) { IEnumerable <Task> workersToAwait = uploadWorkers; if (chunkQueue.Count < workerCount) { workersToAwait = workersToAwait.Concat(hashWorkers); } Task finished = await Task.WhenAny(workersToAwait).ConfigureAwait(false); if (finished is Task <Chunk> ) { var finishedHashWorker = (Task <Chunk>)finished; hashWorkers.Remove(finishedHashWorker); chunkQueue.Enqueue(await finishedHashWorker.ConfigureAwait(false)); } else { var finishedUploadWorker = (Task <ChunkUploadTime>)finished; uploadWorkers.Remove(finishedUploadWorker); var chunkTime = await finishedUploadWorker.ConfigureAwait(false); nextChunkSize = partSizeCalc.NextPartSize(nextChunkSize, chunkTime.Size, chunkTime.Elapsed); } } else { throw new Exception("Impossible"); } } } catch (TaskCanceledException) when(cancellationToken.IsCancellationRequested) { throw CancellationException(); } catch (Exception ex) { throw FailureException(ex); } }