/// <summary>
/// 分片上传/断点续上传，带有自定义进度处理和上传控制，检查CRC32，可自动重试
/// </summary>
/// <param name="stream">待上传文件流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">可选配置参数</param>
/// <returns>上传文件后返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
{
HttpResult result = new HttpResult();
//check put extra
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
if (putExtra.MaxRetryTimes == 0)
{
putExtra.MaxRetryTimes = DEFAULT_MAX_RETRY_TIMES;
}
//start to upload
try
{
long fileSize = stream.Length;
long chunkSize = CHUNK_SIZE;
long blockSize = BLOCK_SIZE;
byte[] chunkBuffer = new byte[chunkSize];
int blockCount = (int)((fileSize + blockSize - 1) / blockSize);
int index = 0; // zero block
//check resume record file
ResumeInfo resumeInfo = null;
if (File.Exists(putExtra.ResumeRecordFile))
{
bool useLastRecord = false;
resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
if (resumeInfo != null && fileSize == resumeInfo.FileSize)
{
//check whether ctx expired
if (!UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
{
useLastRecord = true;
}
}
if (useLastRecord)
{
index = resumeInfo.BlockIndex;
}
}
if (resumeInfo == null)
{
resumeInfo = new ResumeInfo()
{
FileSize = fileSize,
BlockIndex = 0,
BlockCount = blockCount,
Contexts = new string[blockCount],
ExpiredAt = 0,
};
}
//read from offset
long offset = index * blockSize;
string context = null;
long expiredAt = 0;
long leftBytes = fileSize - offset;
long blockLeft = 0;
long blockOffset = 0;
HttpResult hr = null;
ResumeContext rc = null;
stream.Seek(offset, SeekOrigin.Begin);
var upts = UploadControllerAction.Activated;
bool bres = true;
var manualResetEvent = new ManualResetEvent(true);
int iTry = 0;
while (leftBytes > 0)
{
// 每上传一个BLOCK之前，都要检查一下UPTS
upts = putExtra.UploadController();
if (upts == UploadControllerAction.Aborted)
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
return(result);
}
else if (upts == UploadControllerAction.Suspended)
{
if (bres)
{
bres = false;
manualResetEvent.Reset();
result.RefCode = (int)HttpCode.USER_PAUSED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
manualResetEvent.WaitOne(1000);
}
else
{
if (!bres)
{
bres = true;
manualResetEvent.Set();
result.RefCode = (int)HttpCode.USER_RESUMED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is resumed\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
#region one-block
#region mkblk
if (leftBytes < BLOCK_SIZE)
{
blockSize = leftBytes;
}
else
{
blockSize = BLOCK_SIZE;
}
if (leftBytes < CHUNK_SIZE)
{
chunkSize = leftBytes;
}
else
{
chunkSize = CHUNK_SIZE;
}
//read data buffer
stream.Read(chunkBuffer, 0, (int)chunkSize);
iTry = 0;
while (++iTry <= putExtra.MaxRetryTimes)
{
result.RefText += string.Format("[{0}] [ResumableUpload] try mkblk#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);
hr = MakeBlock(chunkBuffer, blockSize, chunkSize, upToken);
if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
{
break;
}
}
if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkblk: code = {1}, text = {2}, offset = {3}, blockSize = {4}, chunkSize = {5}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockSize, chunkSize);
return(result);
}
if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
{
result.Shadow(hr);
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += string.Format("[{0}] [ResumableUpload] mkblk Error: JSON Decode Error: text = {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);
return(result);
}
context = rc.Ctx;
offset += chunkSize;
leftBytes -= chunkSize;
#endregion mkblk
putExtra.ProgressHandler(offset, fileSize);
if (leftBytes > 0)
{
blockLeft = blockSize - chunkSize;
blockOffset = chunkSize;
while (blockLeft > 0)
{
#region bput-loop
if (blockLeft < CHUNK_SIZE)
{
chunkSize = blockLeft;
}
else
{
chunkSize = CHUNK_SIZE;
}
stream.Read(chunkBuffer, 0, (int)chunkSize);
iTry = 0;
while (++iTry <= putExtra.MaxRetryTimes)
{
result.RefText += string.Format("[{0}] [ResumableUpload] try bput#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);
hr = BputChunk(chunkBuffer, blockOffset, chunkSize, context, upToken);
if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
{
break;
}
}
if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: bput: code = {1}, text = {2}, offset = {3}, blockOffset = {4}, chunkSize = {5}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockOffset, chunkSize);
return(result);
}
if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
{
result.Shadow(hr);
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += string.Format("[{0}] [ResumableUpload] bput Error: JSON Decode Error: text = {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);
return(result);
}
context = rc.Ctx;
if (expiredAt == 0)
{
expiredAt = rc.Expired_At;
}
offset += chunkSize;
leftBytes -= chunkSize;
blockOffset += chunkSize;
blockLeft -= chunkSize;
#endregion bput-loop
putExtra.ProgressHandler(offset, fileSize);
}
}
#endregion one-block
resumeInfo.BlockIndex = index;
resumeInfo.Contexts[index] = context;
resumeInfo.ExpiredAt = expiredAt;
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
++index;
}
}
hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
if (hr.Code != (int)HttpCode.OK)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
return(result);
}
if (File.Exists(putExtra.ResumeRecordFile))
{
File.Delete(putExtra.ResumeRecordFile);
}
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
finally
{
if (stream != null)
{
stream.Close();
stream.Dispose();
}
}
return(result);
}
/// <summary>
/// 分片上传/断点续上传，带有自定义进度处理和上传控制，检查CRC32，可自动重试
/// </summary>
/// <param name="stream">待上传文件流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">可选配置参数</param>
/// <returns>上传文件后返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
{
HttpResult result = new HttpResult();
//check put extra
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64))
{
putExtra.BlockUploadThreads = 1;
}
using (stream)
{
//start to upload
try
{
long uploadedBytes = 0;
long fileSize = stream.Length;
long blockCount = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE;
//check resume record file
ResumeInfo resumeInfo = null;
if (File.Exists(putExtra.ResumeRecordFile))
{
resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
if (resumeInfo != null && fileSize == resumeInfo.FileSize)
{
//check whether ctx expired
if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
{
resumeInfo = null;
}
}
}
if (resumeInfo == null)
{
resumeInfo = new ResumeInfo()
{
FileSize = fileSize,
BlockCount = blockCount,
Contexts = new string[blockCount],
ExpiredAt = 0,
};
}
//calc upload progress
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (!string.IsNullOrEmpty(context))
{
uploadedBytes += BLOCK_SIZE;
}
}
//set upload progress
putExtra.ProgressHandler(uploadedBytes, fileSize);
//init block upload error
//check not finished blocks to upload
UploadControllerAction upCtrl = putExtra.UploadController();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
Dictionary <long, byte[]> blockDataDict = new Dictionary <long, byte[]>();
Dictionary <long, HttpResult> blockMakeResults = new Dictionary <long, HttpResult>();
Dictionary <string, long> uploadedBytesDict = new Dictionary <string, long>();
uploadedBytesDict.Add("UploadProgress", uploadedBytes);
byte[] blockBuffer = new byte[BLOCK_SIZE];
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (string.IsNullOrEmpty(context))
{
//check upload controller action before each chunk
while (true)
{
upCtrl = putExtra.UploadController();
if (upCtrl == UploadControllerAction.Aborted)
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.Set();
return(result);
}
else if (upCtrl == UploadControllerAction.Suspended)
{
result.RefCode = (int)HttpCode.USER_PAUSED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.WaitOne(1000);
}
else if (upCtrl == UploadControllerAction.Activated)
{
break;
}
}
long offset = blockIndex * BLOCK_SIZE;
stream.Seek(offset, SeekOrigin.Begin);
int blockLen = stream.Read(blockBuffer, 0, BLOCK_SIZE);
byte[] blockData = new byte[blockLen];
Array.Copy(blockBuffer, blockData, blockLen);
blockDataDict.Add(blockIndex, blockData);
if (blockDataDict.Count == putExtra.BlockUploadThreads)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
}
}
if (blockDataDict.Count > 0)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
if (upCtrl == UploadControllerAction.Activated)
{
HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
if (hr.Code != (int)HttpCode.OK)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
}
if (File.Exists(putExtra.ResumeRecordFile))
{
File.Delete(putExtra.ResumeRecordFile);
}
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
}
else
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
manualResetEvent.Set();
return(result);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
