/// <summary> /// 分片上传/断点续上传,带有自定义进度处理和上传控制,检查CRC32,可自动重试 /// </summary> /// <param name="stream">待上传文件流</param> /// <param name="key">要保存的文件名称</param> /// <param name="upToken">上传凭证</param> /// <param name="putExtra">可选配置参数</param> /// <returns>上传文件后返回结果</returns> public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra) { HttpResult result = new HttpResult(); //check put extra if (putExtra == null) { putExtra = new PutExtra(); } if (putExtra.ProgressHandler == null) { putExtra.ProgressHandler = DefaultUploadProgressHandler; } if (putExtra.UploadController == null) { putExtra.UploadController = DefaultUploadController; } if (putExtra.MaxRetryTimes == 0) { putExtra.MaxRetryTimes = DEFAULT_MAX_RETRY_TIMES; } //start to upload try { long fileSize = stream.Length; long chunkSize = CHUNK_SIZE; long blockSize = BLOCK_SIZE; byte[] chunkBuffer = new byte[chunkSize]; int blockCount = (int)((fileSize + blockSize - 1) / blockSize); int index = 0; // zero block //check resume record file ResumeInfo resumeInfo = null; if (File.Exists(putExtra.ResumeRecordFile)) { bool useLastRecord = false; resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile); if (resumeInfo != null && fileSize == resumeInfo.FileSize) { //check whether ctx expired if (!UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt)) { useLastRecord = true; } } if (useLastRecord) { index = resumeInfo.BlockIndex; } } if (resumeInfo == null) { resumeInfo = new ResumeInfo() { FileSize = fileSize, BlockIndex = 0, BlockCount = blockCount, Contexts = new string[blockCount], ExpiredAt = 0, }; } //read from offset long offset = index * blockSize; string context = null; long expiredAt = 0; long leftBytes = fileSize - offset; long blockLeft = 0; long blockOffset = 0; HttpResult hr = null; ResumeContext rc = null; stream.Seek(offset, SeekOrigin.Begin); var upts = UploadControllerAction.Activated; bool bres = true; var manualResetEvent = new ManualResetEvent(true); int iTry = 0; while (leftBytes > 0) { // 每上传一个BLOCK之前,都要检查一下UPTS upts = putExtra.UploadController(); if (upts == UploadControllerAction.Aborted) { result.Code = (int)HttpCode.USER_CANCELED; result.RefCode = (int)HttpCode.USER_CANCELED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); return(result); } else if (upts == UploadControllerAction.Suspended) { if (bres) { bres = false; manualResetEvent.Reset(); result.RefCode = (int)HttpCode.USER_PAUSED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); } manualResetEvent.WaitOne(1000); } else { if (!bres) { bres = true; manualResetEvent.Set(); result.RefCode = (int)HttpCode.USER_RESUMED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is resumed\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); } #region one-block #region mkblk if (leftBytes < BLOCK_SIZE) { blockSize = leftBytes; } else { blockSize = BLOCK_SIZE; } if (leftBytes < CHUNK_SIZE) { chunkSize = leftBytes; } else { chunkSize = CHUNK_SIZE; } //read data buffer stream.Read(chunkBuffer, 0, (int)chunkSize); iTry = 0; while (++iTry <= putExtra.MaxRetryTimes) { result.RefText += string.Format("[{0}] [ResumableUpload] try mkblk#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry); hr = MakeBlock(chunkBuffer, blockSize, chunkSize, upToken); if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY) { break; } } if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY) { result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkblk: code = {1}, text = {2}, offset = {3}, blockSize = {4}, chunkSize = {5}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockSize, chunkSize); return(result); } if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null) { result.Shadow(hr); result.RefCode = (int)HttpCode.USER_UNDEF; result.RefText += string.Format("[{0}] [ResumableUpload] mkblk Error: JSON Decode Error: text = {1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text); return(result); } context = rc.Ctx; offset += chunkSize; leftBytes -= chunkSize; #endregion mkblk putExtra.ProgressHandler(offset, fileSize); if (leftBytes > 0) { blockLeft = blockSize - chunkSize; blockOffset = chunkSize; while (blockLeft > 0) { #region bput-loop if (blockLeft < CHUNK_SIZE) { chunkSize = blockLeft; } else { chunkSize = CHUNK_SIZE; } stream.Read(chunkBuffer, 0, (int)chunkSize); iTry = 0; while (++iTry <= putExtra.MaxRetryTimes) { result.RefText += string.Format("[{0}] [ResumableUpload] try bput#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry); hr = BputChunk(chunkBuffer, blockOffset, chunkSize, context, upToken); if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY) { break; } } if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY) { result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Error: bput: code = {1}, text = {2}, offset = {3}, blockOffset = {4}, chunkSize = {5}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockOffset, chunkSize); return(result); } if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null) { result.Shadow(hr); result.RefCode = (int)HttpCode.USER_UNDEF; result.RefText += string.Format("[{0}] [ResumableUpload] bput Error: JSON Decode Error: text = {1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text); return(result); } context = rc.Ctx; if (expiredAt == 0) { expiredAt = rc.Expired_At; } offset += chunkSize; leftBytes -= chunkSize; blockOffset += chunkSize; blockLeft -= chunkSize; #endregion bput-loop putExtra.ProgressHandler(offset, fileSize); } } #endregion one-block resumeInfo.BlockIndex = index; resumeInfo.Contexts[index] = context; resumeInfo.ExpiredAt = expiredAt; if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile)) { ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile); } ++index; } } hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts); if (hr.Code != (int)HttpCode.OK) { result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text); return(result); } if (File.Exists(putExtra.ResumeRecordFile)) { File.Delete(putExtra.ResumeRecordFile); } result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key); } catch (Exception ex) { Console.WriteLine(ex.StackTrace); StringBuilder sb = new StringBuilder(); sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); Exception e = ex; while (e != null) { sb.Append(e.Message + " "); e = e.InnerException; } sb.AppendLine(); result.RefCode = (int)HttpCode.USER_UNDEF; result.RefText += sb.ToString(); } finally { if (stream != null) { stream.Close(); stream.Dispose(); } } return(result); }
/// <summary> /// 分片上传/断点续上传,带有自定义进度处理和上传控制,检查CRC32,可自动重试 /// </summary> /// <param name="stream">待上传文件流</param> /// <param name="key">要保存的文件名称</param> /// <param name="upToken">上传凭证</param> /// <param name="putExtra">可选配置参数</param> /// <returns>上传文件后返回结果</returns> public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra) { HttpResult result = new HttpResult(); //check put extra if (putExtra == null) { putExtra = new PutExtra(); } if (putExtra.ProgressHandler == null) { putExtra.ProgressHandler = DefaultUploadProgressHandler; } if (putExtra.UploadController == null) { putExtra.UploadController = DefaultUploadController; } if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64)) { putExtra.BlockUploadThreads = 1; } using (stream) { //start to upload try { long uploadedBytes = 0; long fileSize = stream.Length; long blockCount = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE; //check resume record file ResumeInfo resumeInfo = null; if (File.Exists(putExtra.ResumeRecordFile)) { resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile); if (resumeInfo != null && fileSize == resumeInfo.FileSize) { //check whether ctx expired if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt)) { resumeInfo = null; } } } if (resumeInfo == null) { resumeInfo = new ResumeInfo() { FileSize = fileSize, BlockCount = blockCount, Contexts = new string[blockCount], ExpiredAt = 0, }; } //calc upload progress for (long blockIndex = 0; blockIndex < blockCount; blockIndex++) { string context = resumeInfo.Contexts[blockIndex]; if (!string.IsNullOrEmpty(context)) { uploadedBytes += BLOCK_SIZE; } } //set upload progress putExtra.ProgressHandler(uploadedBytes, fileSize); //init block upload error //check not finished blocks to upload UploadControllerAction upCtrl = putExtra.UploadController(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); Dictionary <long, byte[]> blockDataDict = new Dictionary <long, byte[]>(); Dictionary <long, HttpResult> blockMakeResults = new Dictionary <long, HttpResult>(); Dictionary <string, long> uploadedBytesDict = new Dictionary <string, long>(); uploadedBytesDict.Add("UploadProgress", uploadedBytes); byte[] blockBuffer = new byte[BLOCK_SIZE]; for (long blockIndex = 0; blockIndex < blockCount; blockIndex++) { string context = resumeInfo.Contexts[blockIndex]; if (string.IsNullOrEmpty(context)) { //check upload controller action before each chunk while (true) { upCtrl = putExtra.UploadController(); if (upCtrl == UploadControllerAction.Aborted) { result.Code = (int)HttpCode.USER_CANCELED; result.RefCode = (int)HttpCode.USER_CANCELED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); manualResetEvent.Set(); return(result); } else if (upCtrl == UploadControllerAction.Suspended) { result.RefCode = (int)HttpCode.USER_PAUSED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); manualResetEvent.WaitOne(1000); } else if (upCtrl == UploadControllerAction.Activated) { break; } } long offset = blockIndex * BLOCK_SIZE; stream.Seek(offset, SeekOrigin.Begin); int blockLen = stream.Read(blockBuffer, 0, BLOCK_SIZE); byte[] blockData = new byte[blockLen]; Array.Copy(blockBuffer, blockData, blockLen); blockDataDict.Add(blockIndex, blockData); if (blockDataDict.Count == putExtra.BlockUploadThreads) { processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize); //check mkblk results foreach (int blkIndex in blockMakeResults.Keys) { HttpResult mkblkRet = blockMakeResults[blkIndex]; if (mkblkRet.Code != 200) { result = mkblkRet; manualResetEvent.Set(); return(result); } } blockDataDict.Clear(); blockMakeResults.Clear(); if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile)) { ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile); } } } } if (blockDataDict.Count > 0) { processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize); //check mkblk results foreach (int blkIndex in blockMakeResults.Keys) { HttpResult mkblkRet = blockMakeResults[blkIndex]; if (mkblkRet.Code != 200) { result = mkblkRet; manualResetEvent.Set(); return(result); } } blockDataDict.Clear(); blockMakeResults.Clear(); if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile)) { ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile); } } if (upCtrl == UploadControllerAction.Activated) { HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts); if (hr.Code != (int)HttpCode.OK) { result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text); } if (File.Exists(putExtra.ResumeRecordFile)) { File.Delete(putExtra.ResumeRecordFile); } result.Shadow(hr); result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key); } else { result.Code = (int)HttpCode.USER_CANCELED; result.RefCode = (int)HttpCode.USER_CANCELED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); } manualResetEvent.Set(); return(result); } catch (Exception ex) { Console.WriteLine(ex.StackTrace); StringBuilder sb = new StringBuilder(); sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); Exception e = ex; while (e != null) { sb.Append(e.Message + " "); e = e.InnerException; } sb.AppendLine(); result.RefCode = (int)HttpCode.USER_UNDEF; result.RefText += sb.ToString(); } } return(result); }