private IEnumerable <Task <PartUploadResult> > Dispatch(FilePartSource partSource, string chunkUploadUrl)
{
int currentPartSize = partConfig.InitialPartSize; //do not make this a long, needs to be atomic or have a lock
Func <FilePart, Task> attemptPartUpload = part =>
{
var timer = ShareFile.Api.Client.Logging.Stopwatch.StartNew();
return(UploadPart(chunkUploadUrl, part).ContinueWith(workerTask =>
{
timer.Stop();
workerTask.Rethrow();
int partSizeIncrement = CalculatePartSizeIncrement(part.Bytes.Length, TimeSpan.FromMilliseconds(timer.ElapsedMilliseconds));
//this increment isn't thread-safe, but nothing horrible should happen if it gets clobbered
currentPartSize = (currentPartSize + partSizeIncrement).Bound(partConfig.MaxPartSize, partConfig.MinPartSize);
}));
};
var workers = new AsyncSemaphore(concurrentWorkers);
bool giveUp = false;
while (!giveUp && partSource.HasMore)
{
workers.WaitAsync().Wait();
var part = partSource.GetNextPart(currentPartSize);
if (part == null || giveUp)
{
yield return(TaskFromResult(PartUploadResult.Error));
break; //stream is busted
}
var task = AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount)
.ContinueWith(partUploadTask =>
{
var partResult = partUploadTask.Result;
if (!partResult.IsSuccess)
{
giveUp = true;
}
workers.Release();
return(partResult);
});
yield return(task);
}
yield break;
}
public async Task Upload(
Stream fileStream,
string fileName,
IMD5HashProvider hashProvider,
string chunkUploadUrl,
bool raw,
long offset = 0,
CancellationToken cancellationToken = default(CancellationToken))
{
this.raw = raw;
// We block this after setting 'started', so make sure this statement is first
NumberOfThreads = (int)Math.Min(NumberOfThreads, (fileStream.Length / partConfig.MinFileSizeForMultithreaded) + 1);
started = true;
if (offset != 0)
{
updateProgress(offset);
completedBytes.Add(0, offset);
}
try
{
var filePartSource = new FilePartSource(fileStream, hashProvider, partConfig.BufferAllocator, offset);
var workers = await Dispatch(filePartSource, chunkUploadUrl, fileName, cancellationToken).ConfigureAwait(false);
await Task.WhenAll(workers).ConfigureAwait(false);
logger.Info("[Scaling Uploader] All upload parts succeeded");
}
catch (Exception innerException)
{
logger.Info("[Scaling Uploader] Upload failed. Bytes uploaded: " + LastConsecutiveByteUploaded);
var uploadException = innerException as UploadException;
UploadStatusCode statusCode = uploadException == null ? UploadStatusCode.Unknown : uploadException.StatusCode;
throw new UploadException(
"FilePart upload failed",
statusCode,
new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded),
innerException);
}
}
private async Task <List <Task> > Dispatch(FilePartSource partSource, string chunkUploadUrl, string fileName, CancellationToken cancellationToken = default(CancellationToken))
{
var incrementLock = new object();
long currentPartSize = partConfig.InitialPartSize;
var partSizeCalc = new PartSizeCalculator(concurrentWorkers, partConfig);
Func <FilePart, Task> attemptPartUpload = async part =>
{
IStopwatch timer = Stopwatch.StartNew();
try
{
await UploadPart(chunkUploadUrl, part, fileName, cancellationToken).ConfigureAwait(false);
}
finally
{
timer.Stop();
}
lock (incrementLock)
{
currentPartSize = partSizeCalc.NextPartSize(currentPartSize, part.Bytes.Length, timer.Elapsed);
}
};
var workerTasks = new List <Task>();
try
{
var activeWorkers = new AsyncSemaphore(concurrentWorkers);
bool giveUp = false;
while (!giveUp && partSource.HasMore)
{
if (cancellationToken.IsCancellationRequested)
{
throw new UploadException(
"Upload was cancelled",
UploadStatusCode.Cancelled,
new ActiveUploadState(UploadSpecification, LastConsecutiveByteUploaded));
}
await activeWorkers.WaitAsync().ConfigureAwait(false);
if (giveUp)
{
return(workerTasks);
}
var part = await partSource.GetNextPart(Interlocked.Read(ref currentPartSize)).ConfigureAwait(false);
var task = Task.Run(async() =>
{
try
{
await AttemptPartUploadWithRetry(attemptPartUpload, part, partConfig.PartRetryCount).ConfigureAwait(false);
}
catch
{
giveUp = true;
throw;
}
finally
{
activeWorkers.Release();
part.Bytes.Dispose();
}
});
workerTasks.Add(task);
}
return(workerTasks);
}
catch
{
ObserveExceptions(workerTasks);
throw;
}
}