private void ProcessCallbackInternal(object sender, StreamTransferProgressArgs e)
{
Interlocked.Add(ref _uploadedBytes, e.IncrementTransferred);
Interlocked.Add(ref _incrementalUploadedBytes, e.IncrementTransferred);
if (_uploadProgressCallback != null && _incrementalUploadedBytes >= _conf.ProgressUpdateInterval)
{
lock (this._callbackLock)
{
if (_incrementalUploadedBytes >= _conf.ProgressUpdateInterval)
{
long incrementalUploadedBytes = _incrementalUploadedBytes;
StreamTransferProgressArgs progress = new StreamTransferProgressArgs(incrementalUploadedBytes, _uploadedBytes, _totalBytes);
_uploadProgressCallback.Invoke(this, progress);
Interlocked.Add(ref _incrementalUploadedBytes, -incrementalUploadedBytes);
}
}
}
}
/// <summary>
/// Do the resumable upload with multithread from non file stream
/// </summary>
/// <param name="bucketName">Bucket name.</param>
/// <param name="key">Key.</param>
/// <param name="resumableContext">Resumable context.</param>
/// <param name="fs">Fs.</param>
/// <param name="uploadProgressCallback">Upload progress callback.</param>
private void DoResumableUploadPreReadMultiThread(string bucketName, string key, ResumableContext resumableContext, Stream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
_uploadProgressCallback = uploadProgressCallback;
_uploadedBytes = resumableContext.GetUploadedBytes();
_totalBytes = fs.Length;
_incrementalUploadedBytes = 0;
Exception e = null;
int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count);
int preReadPartCount = Math.Min(parallel, _conf.PreReadBufferCount) + parallel;
ManualResetEvent[] taskFinishEvents = new ManualResetEvent[parallel];
UploadTask[] runningTasks = new UploadTask[parallel];
fs.Seek(0, SeekOrigin.Begin);
// init the buffer pool
PreReadThreadParam preReadParam = new PreReadThreadParam();
preReadParam.Fs = fs;
preReadParam.ResumableContext = resumableContext;
for (int i = 0; i < preReadPartCount && i < resumableContext.PartContextList.Count; i++)
{
var part = resumableContext.PartContextList[i];
preReadParam.ReturnBuffer(new MemoryStream((int)part.Length));
}
Thread thread = new Thread(new ParameterizedThreadStart(StartPreRead));
thread.Start(preReadParam);
bool allTaskDone = false;
for (int i = 0; i < parallel; i++)
{
UploadTask task = preReadParam.TakeTask();
if (task == null)
{
continue;
}
taskFinishEvents[i] = task.Finished;
runningTasks[i] = task;
StartUploadPartTask(task);
}
int nextPart = parallel;
try
{
int waitingCount = 0;
const int MaxWaitingCount = 100;
while (nextPart < resumableContext.PartContextList.Count && waitingCount < MaxWaitingCount)
{
int index = ManualResetEvent.WaitAny(taskFinishEvents);
if (runningTasks[index].Error == null)
{
resumableContext.Dump();
}
else
{
e = runningTasks[index].Error;
}
runningTasks[index].Finished.Close();
preReadParam.ReturnBuffer(runningTasks[index].InputStream as MemoryStream);
UploadTask task = preReadParam.TakeTask();
if (task == null)
{
waitingCount++;
if (preReadParam.PreReadError != null) // no more task will be created;
{
break;
}
continue;
}
StartUploadPartTask(task);
runningTasks[index] = task;
taskFinishEvents[index] = runningTasks[index].Finished;
nextPart++;
}
if (waitingCount >= MaxWaitingCount)
{
e = new ClientException("Fail to read the data from local stream");
}
WaitHandle.WaitAll(taskFinishEvents);
allTaskDone = true;
}
finally
{
preReadParam.RequestStopPreRead = true;
if (!allTaskDone)
{
WaitHandle.WaitAll(taskFinishEvents);
}
if (uploadProgressCallback != null)
{
long latestUploadedBytes = resumableContext.GetUploadedBytes();
long lastIncrementalUploadedBytes = latestUploadedBytes - _uploadedBytes + _incrementalUploadedBytes;
if (lastIncrementalUploadedBytes > 0)
{
StreamTransferProgressArgs progress = new StreamTransferProgressArgs(lastIncrementalUploadedBytes, latestUploadedBytes, fs.Length);
uploadProgressCallback.Invoke(this, progress);
}
_uploadedBytes = latestUploadedBytes;
}
for (int i = 0; i < parallel; i++)
{
if (runningTasks[i].Error != null)
{
e = runningTasks[i].Error;
}
runningTasks[i].Finished.Close();
}
if (preReadParam.PreReadError != null)
{
e = preReadParam.PreReadError;
}
MemoryStream buffer = null;
while (preReadParam.GetBufferLength() != 0 && (buffer = preReadParam.TakeBuffer()) != null)
{
buffer.Dispose();
}
resumableContext.Dump();
if (e != null)
{
throw e;
}
}
}
/// <summary>
/// Do the resumable upload with multithread from file stream.
/// </summary>
/// <param name="bucketName">Bucket name.</param>
/// <param name="key">Key.</param>
/// <param name="resumableContext">Resumable context.</param>
/// <param name="fs">Fs.</param>
/// <param name="uploadProgressCallback">Upload progress callback.</param>
private void DoResumableUploadFileMultiThread(string bucketName, string key, ResumableContext resumableContext, FileStream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
_uploadProgressCallback = uploadProgressCallback;
_uploadedBytes = resumableContext.GetUploadedBytes();
_totalBytes = fs.Length;
_incrementalUploadedBytes = 0;
Exception e = null;
int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count);
ManualResetEvent[] taskFinishEvents = new ManualResetEvent[parallel];
UploadTask[] runningTasks = new UploadTask[parallel];
fs.Seek(0, SeekOrigin.Begin);
bool allTaskDone = false;
for (int i = 0; i < parallel; i++)
{
UploadTask param = CreateTask(i, resumableContext, fs);
taskFinishEvents[i] = param.Finished;
runningTasks[i] = param;
StartUploadPartTask(param);
}
int nextPart = parallel;
try
{
while (nextPart < resumableContext.PartContextList.Count)
{
int index = ManualResetEvent.WaitAny(taskFinishEvents);
if (runningTasks[index].Error == null)
{
resumableContext.Dump();
}
else
{
e = runningTasks[index].Error;
}
runningTasks[index].Finished.Close();
runningTasks[index].InputStream.Dispose();
UploadTask task = CreateTask(nextPart, resumableContext, fs);
StartUploadPartTask(task);
runningTasks[index] = task;
taskFinishEvents[index] = runningTasks[index].Finished;
nextPart++;
}
WaitHandle.WaitAll(taskFinishEvents);
allTaskDone = true;
}
finally
{
if (!allTaskDone)
{
WaitHandle.WaitAll(taskFinishEvents);
}
if (uploadProgressCallback != null)
{
long latestUploadedBytes = resumableContext.GetUploadedBytes();
long lastIncrementalUploadedBytes = latestUploadedBytes - _uploadedBytes + _incrementalUploadedBytes;
if (lastIncrementalUploadedBytes > 0)
{
StreamTransferProgressArgs progress = new StreamTransferProgressArgs(lastIncrementalUploadedBytes, latestUploadedBytes, fs.Length);
uploadProgressCallback.Invoke(this, progress);
}
_uploadedBytes = latestUploadedBytes;
}
for (int i = 0; i < parallel; i++)
{
taskFinishEvents[i].Close();
if (runningTasks[i].Error != null)
{
e = runningTasks[i].Error;
}
runningTasks[i].InputStream.Dispose();
}
resumableContext.Dump();
if (e != null)
{
throw e;
}
}
}
private void DoResumableDownloadMultiThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext,
EventHandler <StreamTransferProgressArgs> downloadProgressCallback)
{
_downloadedBytes = resumableContext.GetDownloadedBytes();
if (!request.MatchingETagConstraints.Contains(resumableContext.ETag))
{
request.MatchingETagConstraints.Add(resumableContext.ETag);
}
Exception e = null;
int parallel = Math.Min(Math.Min(request.ParallelThreadCount, resumableContext.PartContextList.Count), Environment.ProcessorCount);
ManualResetEvent[] taskFinishedEvents = new ManualResetEvent[parallel];
DownloadTaskParam[] taskParams = new DownloadTaskParam[parallel];
int nextPart = 0;
for (nextPart = 0; nextPart < parallel; nextPart++)
{
var part = resumableContext.PartContextList[nextPart];
taskParams[nextPart] = StartDownloadPartTask(request, part, downloadProgressCallback);
taskFinishedEvents[nextPart] = taskParams[nextPart].DownloadFinished;
}
bool allTasksDone = false;
try
{
long totalBytes = resumableContext.GetTotalBytes();
long lastDownloadedBytes = _downloadedBytes;
while (nextPart < resumableContext.PartContextList.Count)
{
int index = ManualResetEvent.WaitAny(taskFinishedEvents);
if (taskParams[index].Error == null)
{
if (request.StreamTransferProgress != null)
{
lastDownloadedBytes = _downloadedBytes;
StreamTransferProgressArgs args = new StreamTransferProgressArgs(resumableContext.PartContextList[index].Length, lastDownloadedBytes, totalBytes);
request.StreamTransferProgress.Invoke(this, args);
}
resumableContext.Dump();
}
else
{
e = taskParams[index].Error;
}
taskFinishedEvents[index].Close();
taskParams[index] = StartDownloadPartTask(request, resumableContext.PartContextList[nextPart++], downloadProgressCallback);
taskFinishedEvents[index] = taskParams[index].DownloadFinished;
}
ManualResetEvent.WaitAll(taskFinishedEvents);
allTasksDone = true;
if (request.StreamTransferProgress != null)
{
StreamTransferProgressArgs args = new StreamTransferProgressArgs(_downloadedBytes - lastDownloadedBytes, _downloadedBytes, totalBytes);
request.StreamTransferProgress.Invoke(this, args);
}
if (e == null)
{
Validate(request, resumableContext);
}
}
finally
{
if (!allTasksDone)
{
ManualResetEvent.WaitAll(taskFinishedEvents);
}
for (int i = 0; i < parallel; i++)
{
taskFinishedEvents[i].Close();
if (taskParams[i].Error != null)
{
e = taskParams[i].Error;
}
}
if (e != null)
{
throw e;
}
}
}