private IEnumerable <Task <PartUploadResult> > Dispatch(FilePartSource partSource, string chunkUploadUrl)
        {
            int currentPartSize = partConfig.InitialPartSize; //do not make this a long, needs to be atomic or have a lock

            Func <FilePart, Task> attemptPartUpload = part =>
            {
                var timer = ShareFile.Api.Client.Logging.Stopwatch.StartNew();
                return(UploadPart(chunkUploadUrl, part).ContinueWith(workerTask =>
                {
                    timer.Stop();
                    workerTask.Rethrow();
                    int partSizeIncrement = CalculatePartSizeIncrement(part.Bytes.Length, TimeSpan.FromMilliseconds(timer.ElapsedMilliseconds));
                    //this increment isn't thread-safe, but nothing horrible should happen if it gets clobbered
                    currentPartSize = (currentPartSize + partSizeIncrement).Bound(partConfig.MaxPartSize, partConfig.MinPartSize);
                }));
            };

            var  workers = new AsyncSemaphore(concurrentWorkers);
            bool giveUp  = false;

            while (!giveUp && partSource.HasMore)
            {
                workers.WaitAsync().Wait();
                var part = partSource.GetNextPart(currentPartSize);
                if (part == null || giveUp)
                {
                    yield return(TaskFromResult(PartUploadResult.Error));

                    break; //stream is busted
                }

                var task = AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount)
                           .ContinueWith(partUploadTask =>
                {
                    var partResult = partUploadTask.Result;
                    if (!partResult.IsSuccess)
                    {
                        giveUp = true;
                    }
                    workers.Release();
                    return(partResult);
                });
                yield return(task);
            }
            yield break;
        }
Ejemplo n.º 2
0
        public async Task Upload(
            Stream fileStream,
            string fileName,
            IMD5HashProvider hashProvider,
            string chunkUploadUrl,
            bool raw,
            long offset = 0,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            this.raw = raw;
            // We block this after setting 'started', so make sure this statement is first
            NumberOfThreads = (int)Math.Min(NumberOfThreads, (fileStream.Length / partConfig.MinFileSizeForMultithreaded) + 1);
            started         = true;
            if (offset != 0)
            {
                updateProgress(offset);
                completedBytes.Add(0, offset);
            }
            try
            {
                var filePartSource = new FilePartSource(fileStream, hashProvider, partConfig.BufferAllocator, offset);
                var workers        = await Dispatch(filePartSource, chunkUploadUrl, fileName, cancellationToken).ConfigureAwait(false);

                await Task.WhenAll(workers).ConfigureAwait(false);

                logger.Info("[Scaling Uploader] All upload parts succeeded");
            }
            catch (Exception innerException)
            {
                logger.Info("[Scaling Uploader] Upload failed. Bytes uploaded: " + LastConsecutiveByteUploaded);
                var uploadException         = innerException as UploadException;
                UploadStatusCode statusCode = uploadException == null ? UploadStatusCode.Unknown : uploadException.StatusCode;
                throw new UploadException(
                          "FilePart upload failed",
                          statusCode,
                          new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded),
                          innerException);
            }
        }
Ejemplo n.º 3
0
        private async Task <List <Task> > Dispatch(FilePartSource partSource, string chunkUploadUrl, string fileName, CancellationToken cancellationToken = default(CancellationToken))
        {
            var  incrementLock   = new object();
            long currentPartSize = partConfig.InitialPartSize;
            var  partSizeCalc    = new PartSizeCalculator(concurrentWorkers, partConfig);

            Func <FilePart, Task> attemptPartUpload = async part =>
            {
                IStopwatch timer = Stopwatch.StartNew();
                try
                {
                    await UploadPart(chunkUploadUrl, part, fileName, cancellationToken).ConfigureAwait(false);
                }
                finally
                {
                    timer.Stop();
                }
                lock (incrementLock)
                {
                    currentPartSize = partSizeCalc.NextPartSize(currentPartSize, part.Bytes.Length, timer.Elapsed);
                }
            };

            var workerTasks = new List <Task>();

            try
            {
                var  activeWorkers = new AsyncSemaphore(concurrentWorkers);
                bool giveUp        = false;
                while (!giveUp && partSource.HasMore)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        throw new UploadException(
                                  "Upload was cancelled",
                                  UploadStatusCode.Cancelled,
                                  new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded));
                    }
                    await activeWorkers.WaitAsync().ConfigureAwait(false);

                    if (giveUp)
                    {
                        return(workerTasks);
                    }

                    var part = await partSource.GetNextPart(Interlocked.Read(ref currentPartSize)).ConfigureAwait(false);

                    var task = Task.Run(async() =>
                    {
                        try
                        {
                            await AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount).ConfigureAwait(false);
                        }
                        catch
                        {
                            giveUp = true;
                            throw;
                        }
                        finally
                        {
                            activeWorkers.Release();
                            part.Bytes.Dispose();
                        }
                    });
                    workerTasks.Add(task);
                }
                return(workerTasks);
            }
            catch
            {
                ObserveExceptions(workerTasks);
                throw;
            }
        }