private IEnumerable <Task <PartUploadResult> > Dispatch(FilePartSource partSource, string chunkUploadUrl) { int currentPartSize = partConfig.InitialPartSize; //do not make this a long, needs to be atomic or have a lock Func <FilePart, Task> attemptPartUpload = part => { var timer = ShareFile.Api.Client.Logging.Stopwatch.StartNew(); return(UploadPart(chunkUploadUrl, part).ContinueWith(workerTask => { timer.Stop(); workerTask.Rethrow(); int partSizeIncrement = CalculatePartSizeIncrement(part.Bytes.Length, TimeSpan.FromMilliseconds(timer.ElapsedMilliseconds)); //this increment isn't thread-safe, but nothing horrible should happen if it gets clobbered currentPartSize = (currentPartSize + partSizeIncrement).Bound(partConfig.MaxPartSize, partConfig.MinPartSize); })); }; var workers = new AsyncSemaphore(concurrentWorkers); bool giveUp = false; while (!giveUp && partSource.HasMore) { workers.WaitAsync().Wait(); var part = partSource.GetNextPart(currentPartSize); if (part == null || giveUp) { yield return(TaskFromResult(PartUploadResult.Error)); break; //stream is busted } var task = AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount) .ContinueWith(partUploadTask => { var partResult = partUploadTask.Result; if (!partResult.IsSuccess) { giveUp = true; } workers.Release(); return(partResult); }); yield return(task); } yield break; }
public async Task Upload( Stream fileStream, string fileName, IMD5HashProvider hashProvider, string chunkUploadUrl, bool raw, long offset = 0, CancellationToken cancellationToken = default(CancellationToken)) { this.raw = raw; // We block this after setting 'started', so make sure this statement is first NumberOfThreads = (int)Math.Min(NumberOfThreads, (fileStream.Length / partConfig.MinFileSizeForMultithreaded) + 1); started = true; if (offset != 0) { updateProgress(offset); completedBytes.Add(0, offset); } try { var filePartSource = new FilePartSource(fileStream, hashProvider, partConfig.BufferAllocator, offset); var workers = await Dispatch(filePartSource, chunkUploadUrl, fileName, cancellationToken).ConfigureAwait(false); await Task.WhenAll(workers).ConfigureAwait(false); logger.Info("[Scaling Uploader] All upload parts succeeded"); } catch (Exception innerException) { logger.Info("[Scaling Uploader] Upload failed. Bytes uploaded: " + LastConsecutiveByteUploaded); var uploadException = innerException as UploadException; UploadStatusCode statusCode = uploadException == null ? UploadStatusCode.Unknown : uploadException.StatusCode; throw new UploadException( "FilePart upload failed", statusCode, new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded), innerException); } }
private async Task <List <Task> > Dispatch(FilePartSource partSource, string chunkUploadUrl, string fileName, CancellationToken cancellationToken = default(CancellationToken)) { var incrementLock = new object(); long currentPartSize = partConfig.InitialPartSize; var partSizeCalc = new PartSizeCalculator(concurrentWorkers, partConfig); Func <FilePart, Task> attemptPartUpload = async part => { IStopwatch timer = Stopwatch.StartNew(); try { await UploadPart(chunkUploadUrl, part, fileName, cancellationToken).ConfigureAwait(false); } finally { timer.Stop(); } lock (incrementLock) { currentPartSize = partSizeCalc.NextPartSize(currentPartSize, part.Bytes.Length, timer.Elapsed); } }; var workerTasks = new List <Task>(); try { var activeWorkers = new AsyncSemaphore(concurrentWorkers); bool giveUp = false; while (!giveUp && partSource.HasMore) { if (cancellationToken.IsCancellationRequested) { throw new UploadException( "Upload was cancelled", UploadStatusCode.Cancelled, new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded)); } await activeWorkers.WaitAsync().ConfigureAwait(false); if (giveUp) { return(workerTasks); } var part = await partSource.GetNextPart(Interlocked.Read(ref currentPartSize)).ConfigureAwait(false); var task = Task.Run(async() => { try { await AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount).ConfigureAwait(false); } catch { giveUp = true; throw; } finally { activeWorkers.Release(); part.Bytes.Dispose(); } }); workerTasks.Add(task); } return(workerTasks); } catch { ObserveExceptions(workerTasks); throw; } }