public void ResumableUploadWithRetry(UploadObjectRequest request, ResumableContext resumableContext)
{
_request = request;
using (Stream fs = request.UploadStream ?? new FileStream(request.UploadFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
for (int i = 0; i < _maxRetryTimes; i++)
{
try
{
DoResumableUpload(request.BucketName, request.Key, resumableContext, fs, request.StreamTransferProgress);
break;
}
catch (Exception ex)
{
if (i != _maxRetryTimes - 1)
{
Thread.Sleep(1000);
continue;
}
else
{
throw ex;
}
}
}
}
}
private UploadTask CreateTask(int i, ResumableContext resumableContext, FileStream fs)
{
UploadTask param = new UploadTask();
param.UploadFileStream = fs;
param.InputStream = new FileStream(fs.Name, FileMode.Open, FileAccess.Read, FileShare.Read);
param.ResumableUploadContext = resumableContext;
param.ResumableUploadPartContext = resumableContext.PartContextList[i];
param.UploadProgressCallback = _uploadProgressCallback;
param.ProgressUpdateInterval = _conf.ProgressUpdateInterval;
param.Finished = new ManualResetEvent(false);
return(param);
}
private void DoResumableUploadSingleThread(string bucketName, string key, ResumableContext resumableContext, Stream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
var uploadedBytes = resumableContext.GetUploadedBytes();
foreach (var part in resumableContext.PartContextList)
{
if (part.IsCompleted)
{
continue;
}
fs.Seek(part.Position, SeekOrigin.Begin);
var originalStream = fs;
if (uploadProgressCallback != null)
{
originalStream = _ossClient.SetupProgressListeners(originalStream,
fs.Length,
uploadedBytes,
_conf.ProgressUpdateInterval,
uploadProgressCallback);
}
var request = new UploadPartRequest(bucketName, key, resumableContext.UploadId)
{
InputStream = originalStream,
PartSize = part.Length,
PartNumber = part.PartId,
RequestPayer = _request.RequestPayer,
TrafficLimit = _request.TrafficLimit
};
var partResult = _ossClient.UploadPart(request);
part.PartETag = partResult.PartETag;
part.IsCompleted = true;
if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma))
{
part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]);
}
resumableContext.Dump();
uploadedBytes += part.Length;
}
}
private void DoResumableUpload(string bucketName, string key, ResumableContext resumableContext, Stream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
bool isFileStream = fs is FileStream;
// use single thread if MaxResumableUploadThreads is no bigger than 1
// or when the stream is not file stream and the part size is bigger than the conf.MaxPartCachingSize
if (_request.ParallelThreadCount <= 1 || (!isFileStream || _conf.UseSingleThreadReadInResumableUpload) && resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize)
{
DoResumableUploadSingleThread(bucketName, key, resumableContext, fs, uploadProgressCallback);
}
else if (isFileStream && !_conf.UseSingleThreadReadInResumableUpload)
{
// multi-threaded read file and send the data
DoResumableUploadFileMultiThread(bucketName, key, resumableContext, fs as FileStream, uploadProgressCallback);
}
else
{
// single thread pre-read the data and multi-thread send the data.
DoResumableUploadPreReadMultiThread(bucketName, key, resumableContext, fs, uploadProgressCallback);
}
}
/// <summary>
/// Do the resumable upload with multithread from non file stream
/// </summary>
/// <param name="bucketName">Bucket name.</param>
/// <param name="key">Key.</param>
/// <param name="resumableContext">Resumable context.</param>
/// <param name="fs">Fs.</param>
/// <param name="uploadProgressCallback">Upload progress callback.</param>
private void DoResumableUploadPreReadMultiThread(string bucketName, string key, ResumableContext resumableContext, Stream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
_uploadProgressCallback = uploadProgressCallback;
_uploadedBytes = resumableContext.GetUploadedBytes();
_totalBytes = fs.Length;
_incrementalUploadedBytes = 0;
Exception e = null;
int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count);
int preReadPartCount = Math.Min(parallel, _conf.PreReadBufferCount) + parallel;
ManualResetEvent[] taskFinishEvents = new ManualResetEvent[parallel];
UploadTask[] runningTasks = new UploadTask[parallel];
fs.Seek(0, SeekOrigin.Begin);
// init the buffer pool
PreReadThreadParam preReadParam = new PreReadThreadParam();
preReadParam.Fs = fs;
preReadParam.ResumableContext = resumableContext;
for (int i = 0; i < preReadPartCount && i < resumableContext.PartContextList.Count; i++)
{
var part = resumableContext.PartContextList[i];
preReadParam.ReturnBuffer(new MemoryStream((int)part.Length));
}
Thread thread = new Thread(new ParameterizedThreadStart(StartPreRead));
thread.Start(preReadParam);
bool allTaskDone = false;
for (int i = 0; i < parallel; i++)
{
UploadTask task = preReadParam.TakeTask();
if (task == null)
{
continue;
}
taskFinishEvents[i] = task.Finished;
runningTasks[i] = task;
StartUploadPartTask(task);
}
int nextPart = parallel;
try
{
int waitingCount = 0;
const int MaxWaitingCount = 100;
while (nextPart < resumableContext.PartContextList.Count && waitingCount < MaxWaitingCount)
{
int index = ManualResetEvent.WaitAny(taskFinishEvents);
if (runningTasks[index].Error == null)
{
resumableContext.Dump();
}
else
{
e = runningTasks[index].Error;
}
runningTasks[index].Finished.Close();
preReadParam.ReturnBuffer(runningTasks[index].InputStream as MemoryStream);
UploadTask task = preReadParam.TakeTask();
if (task == null)
{
waitingCount++;
if (preReadParam.PreReadError != null) // no more task will be created;
{
break;
}
continue;
}
StartUploadPartTask(task);
runningTasks[index] = task;
taskFinishEvents[index] = runningTasks[index].Finished;
nextPart++;
}
if (waitingCount >= MaxWaitingCount)
{
e = new ClientException("Fail to read the data from local stream");
}
WaitHandle.WaitAll(taskFinishEvents);
allTaskDone = true;
}
finally
{
preReadParam.RequestStopPreRead = true;
if (!allTaskDone)
{
WaitHandle.WaitAll(taskFinishEvents);
}
if (uploadProgressCallback != null)
{
long latestUploadedBytes = resumableContext.GetUploadedBytes();
long lastIncrementalUploadedBytes = latestUploadedBytes - _uploadedBytes + _incrementalUploadedBytes;
if (lastIncrementalUploadedBytes > 0)
{
StreamTransferProgressArgs progress = new StreamTransferProgressArgs(lastIncrementalUploadedBytes, latestUploadedBytes, fs.Length);
uploadProgressCallback.Invoke(this, progress);
}
_uploadedBytes = latestUploadedBytes;
}
for (int i = 0; i < parallel; i++)
{
if (runningTasks[i].Error != null)
{
e = runningTasks[i].Error;
}
runningTasks[i].Finished.Close();
}
if (preReadParam.PreReadError != null)
{
e = preReadParam.PreReadError;
}
MemoryStream buffer = null;
while (preReadParam.GetBufferLength() != 0 && (buffer = preReadParam.TakeBuffer()) != null)
{
buffer.Dispose();
}
resumableContext.Dump();
if (e != null)
{
throw e;
}
}
}
/// <summary>
/// Do the resumable upload with multithread from file stream.
/// </summary>
/// <param name="bucketName">Bucket name.</param>
/// <param name="key">Key.</param>
/// <param name="resumableContext">Resumable context.</param>
/// <param name="fs">Fs.</param>
/// <param name="uploadProgressCallback">Upload progress callback.</param>
private void DoResumableUploadFileMultiThread(string bucketName, string key, ResumableContext resumableContext, FileStream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
_uploadProgressCallback = uploadProgressCallback;
_uploadedBytes = resumableContext.GetUploadedBytes();
_totalBytes = fs.Length;
_incrementalUploadedBytes = 0;
Exception e = null;
int parallel = Math.Min(_request.ParallelThreadCount, resumableContext.PartContextList.Count);
ManualResetEvent[] taskFinishEvents = new ManualResetEvent[parallel];
UploadTask[] runningTasks = new UploadTask[parallel];
fs.Seek(0, SeekOrigin.Begin);
bool allTaskDone = false;
for (int i = 0; i < parallel; i++)
{
UploadTask param = CreateTask(i, resumableContext, fs);
taskFinishEvents[i] = param.Finished;
runningTasks[i] = param;
StartUploadPartTask(param);
}
int nextPart = parallel;
try
{
while (nextPart < resumableContext.PartContextList.Count)
{
int index = ManualResetEvent.WaitAny(taskFinishEvents);
if (runningTasks[index].Error == null)
{
resumableContext.Dump();
}
else
{
e = runningTasks[index].Error;
}
runningTasks[index].Finished.Close();
runningTasks[index].InputStream.Dispose();
UploadTask task = CreateTask(nextPart, resumableContext, fs);
StartUploadPartTask(task);
runningTasks[index] = task;
taskFinishEvents[index] = runningTasks[index].Finished;
nextPart++;
}
WaitHandle.WaitAll(taskFinishEvents);
allTaskDone = true;
}
finally
{
if (!allTaskDone)
{
WaitHandle.WaitAll(taskFinishEvents);
}
if (uploadProgressCallback != null)
{
long latestUploadedBytes = resumableContext.GetUploadedBytes();
long lastIncrementalUploadedBytes = latestUploadedBytes - _uploadedBytes + _incrementalUploadedBytes;
if (lastIncrementalUploadedBytes > 0)
{
StreamTransferProgressArgs progress = new StreamTransferProgressArgs(lastIncrementalUploadedBytes, latestUploadedBytes, fs.Length);
uploadProgressCallback.Invoke(this, progress);
}
_uploadedBytes = latestUploadedBytes;
}
for (int i = 0; i < parallel; i++)
{
taskFinishEvents[i].Close();
if (runningTasks[i].Error != null)
{
e = runningTasks[i].Error;
}
runningTasks[i].InputStream.Dispose();
}
resumableContext.Dump();
if (e != null)
{
throw e;
}
}
}
