private void processMakeBlocks(Dictionary <long, byte[]> blockDataDict, string upToken,
PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
Dictionary <string, long> uploadedBytesDict, long fileSize)
{
int taskMax = blockDataDict.Count;
ManualResetEvent[] doneEvents = new ManualResetEvent[taskMax];
int eventIndex = 0;
object progressLock = new object();
foreach (long blockIndex in blockDataDict.Keys)
{
//signal task
ManualResetEvent doneEvent = new ManualResetEvent(false);
doneEvents[eventIndex] = doneEvent;
eventIndex += 1;
//queue task
byte[] blockData = blockDataDict[blockIndex];
ResumeBlocker resumeBlocker = new ResumeBlocker(doneEvent, blockData, blockIndex, upToken, putExtra,
resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize);
ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlock), resumeBlocker);
}
try
{
WaitHandle.WaitAll(doneEvents);
}
catch (Exception ex)
{
Console.WriteLine("wait all exceptions:" + ex.StackTrace);
//pass
}
}
/// <summary>
/// 尝试从从文件载入断点信息
/// </summary>
/// <param name="recordFile">断点记录文件</param>
/// <returns>断点信息</returns>
public static ResumeInfo Load(string recordFile)
{
ResumeInfo resumeInfo = null;
try
{
resumeInfo = JsonConvert.DeserializeObject <ResumeInfo>(IOUtils.Api.ReadFile(recordFile));
}
catch (Exception)
{
resumeInfo = null;
}
return(resumeInfo);
}
public ResumeBlocker(ManualResetEvent doneEvent, byte[] blockBuffer, long blockIndex, string uploadToken,
PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
object progressLock, Dictionary <string, long> uploadedBytesDict, long fileSize)
{
this.DoneEvent = doneEvent;
this.BlockBuffer = blockBuffer;
this.BlockIndex = blockIndex;
this.UploadToken = uploadToken;
this.PutExtra = putExtra;
this.ResumeInfo = resumeInfo;
this.BlockMakeResults = blockMakeResults;
this.ProgressLock = progressLock;
this.UploadedBytesDict = uploadedBytesDict;
this.FileSize = fileSize;
}
/// <summary>
/// 分片上传/断点续上传，带有自定义进度处理和上传控制，检查CRC32，可自动重试
/// </summary>
/// <param name="stream">待上传文件流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">可选配置参数</param>
/// <returns>上传文件后返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
{
HttpResult result = new HttpResult();
//check put extra
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64))
{
putExtra.BlockUploadThreads = 1;
}
using (stream)
{
//start to upload
try
{
long uploadedBytes = 0;
long fileSize = stream.Length;
long blockCount = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE;
//check resume record file
ResumeInfo resumeInfo = null;
if (IOUtils.Api.FileExist(putExtra.ResumeRecordFile))
{
resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
if (resumeInfo != null && fileSize == resumeInfo.FileSize)
{
//check whether ctx expired
if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
{
resumeInfo = null;
}
}
}
if (resumeInfo == null)
{
resumeInfo = new ResumeInfo()
{
FileSize = fileSize,
BlockCount = blockCount,
Contexts = new string[blockCount],
ExpiredAt = 0,
};
}
//calc upload progress
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (!string.IsNullOrEmpty(context))
{
uploadedBytes += BLOCK_SIZE;
}
}
//set upload progress
putExtra.ProgressHandler(uploadedBytes, fileSize);
//init block upload error
//check not finished blocks to upload
UploadControllerAction upCtrl = putExtra.UploadController();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
Dictionary <long, byte[]> blockDataDict = new Dictionary <long, byte[]>();
Dictionary <long, HttpResult> blockMakeResults = new Dictionary <long, HttpResult>();
Dictionary <string, long> uploadedBytesDict = new Dictionary <string, long>();
uploadedBytesDict.Add("UploadProgress", uploadedBytes);
byte[] blockBuffer = new byte[BLOCK_SIZE];
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (string.IsNullOrEmpty(context))
{
//check upload controller action before each chunk
while (true)
{
upCtrl = putExtra.UploadController();
if (upCtrl == UploadControllerAction.Aborted)
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.Set();
return(result);
}
else if (upCtrl == UploadControllerAction.Suspended)
{
result.RefCode = (int)HttpCode.USER_PAUSED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.WaitOne(1000);
}
else if (upCtrl == UploadControllerAction.Activated)
{
break;
}
}
long offset = blockIndex * BLOCK_SIZE;
stream.Seek(offset, SeekOrigin.Begin);
int blockLen = stream.Read(blockBuffer, 0, BLOCK_SIZE);
byte[] blockData = new byte[blockLen];
Array.Copy(blockBuffer, blockData, blockLen);
blockDataDict.Add(blockIndex, blockData);
if (blockDataDict.Count == putExtra.BlockUploadThreads)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
}
}
if (blockDataDict.Count > 0)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
if (upCtrl == UploadControllerAction.Activated)
{
HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
if (hr.Code != (int)HttpCode.OK)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
}
if (IOUtils.Api.FileExist(putExtra.ResumeRecordFile))
{
IOUtils.Api.FileDelete(putExtra.ResumeRecordFile);
}
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
}
else
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
manualResetEvent.Set();
return(result);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
/// <summary>
/// 创建块(携带首片数据),同时检查CRC32
/// </summary>
/// <param name="resumeBlockerObj">创建分片上次的块请求</param>
private void MakeBlock(object resumeBlockerObj)
{
ResumeBlocker resumeBlocker = (ResumeBlocker)resumeBlockerObj;
ManualResetEvent doneEvent = resumeBlocker.DoneEvent;
Dictionary <long, HttpResult> blockMakeResults = resumeBlocker.BlockMakeResults;
PutExtra putExtra = resumeBlocker.PutExtra;
long blockIndex = resumeBlocker.BlockIndex;
HttpResult result = new HttpResult();
//check whether to cancel
while (true)
{
UploadControllerAction upCtl = resumeBlocker.PutExtra.UploadController();
if (upCtl == UploadControllerAction.Suspended)
{
doneEvent.WaitOne(1000);
continue;
}
else if (upCtl == UploadControllerAction.Aborted)
{
doneEvent.Set();
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkblk {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), blockIndex);
blockMakeResults.Add(blockIndex, result);
return;
}
else
{
break;
}
}
byte[] blockBuffer = resumeBlocker.BlockBuffer;
int blockSize = blockBuffer.Length;
string upToken = resumeBlocker.UploadToken;
Dictionary <string, long> uploadedBytesDict = resumeBlocker.UploadedBytesDict;
long fileSize = resumeBlocker.FileSize;
object progressLock = resumeBlocker.ProgressLock;
ResumeInfo resumeInfo = resumeBlocker.ResumeInfo;
try
{
//get upload host
string ak = UpToken.GetAccessKeyFromUpToken(upToken);
string bucket = UpToken.GetBucketFromUpToken(upToken);
if (ak == null || bucket == null)
{
result = HttpResult.InvalidToken;
doneEvent.Set();
return;
}
string uploadHost = this.config.UpHost(ak, bucket);
string url = string.Format("{0}/mkblk/{1}", uploadHost, blockSize);
string upTokenStr = string.Format("UpToken {0}", upToken);
using (MemoryStream ms = new MemoryStream(blockBuffer, 0, blockSize))
{
byte[] data = ms.ToArray();
result = httpManager.PostData(url, data, upTokenStr);
if (result.Code == (int)HttpCode.OK)
{
ResumeContext rc = JsonConvert.DeserializeObject <ResumeContext>(result.Text);
if (rc.Crc32 > 0)
{
uint crc_1 = rc.Crc32;
uint crc_2 = CRC32.CheckSumSlice(blockBuffer, 0, blockSize);
if (crc_1 != crc_2)
{
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
result.RefText += string.Format(" CRC32: remote={0}, local={1}\n", crc_1, crc_2);
}
else
{
//write the mkblk context
resumeInfo.Contexts[blockIndex] = rc.Ctx;
resumeInfo.ExpiredAt = rc.ExpiredAt;
lock (progressLock)
{
uploadedBytesDict["UploadProgress"] += blockSize;
}
putExtra.ProgressHandler(uploadedBytesDict["UploadProgress"], fileSize);
}
}
else
{
result.RefText += string.Format("[{0}] JSON Decode Error: text = {1}",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Text);
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
}
}
else
{
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
}
}
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] mkblk Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException)
{
QiniuException qex = (QiniuException)ex;
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
//return the http result
blockMakeResults.Add(blockIndex, result);
doneEvent.Set();
}
/// <summary>
/// 保存断点信息到文件
/// </summary>
/// <param name="resumeInfo">断点信息</param>
/// <param name="recordFile">断点记录文件</param>
public static void Save(ResumeInfo resumeInfo, string recordFile)
{
var jsonStr = resumeInfo.ToJsonStr();
IOUtils.Api.WriteFile(recordFile, jsonStr);
}