private void processMakeBlocks(Dictionary <long, byte[]> blockDataDict, string upToken, PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults, Dictionary <string, long> uploadedBytesDict, long fileSize) { int taskMax = blockDataDict.Count; ManualResetEvent[] doneEvents = new ManualResetEvent[taskMax]; int eventIndex = 0; object progressLock = new object(); foreach (long blockIndex in blockDataDict.Keys) { //signal task ManualResetEvent doneEvent = new ManualResetEvent(false); doneEvents[eventIndex] = doneEvent; eventIndex += 1; //queue task byte[] blockData = blockDataDict[blockIndex]; ResumeBlocker resumeBlocker = new ResumeBlocker(doneEvent, blockData, blockIndex, upToken, putExtra, resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize); ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlock), resumeBlocker); } try { WaitHandle.WaitAll(doneEvents); } catch (Exception ex) { Console.WriteLine("wait all exceptions:" + ex.StackTrace); //pass } }
private void ProcessMakeBlocks( Dictionary <long, byte[]> blockDataDict, string upToken, PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults, Dictionary <string, long> uploadedBytesDict, long fileSize) { var taskMax = blockDataDict.Count; var doneEvents = new ManualResetEvent[taskMax]; var eventIndex = 0; var progressLock = new object(); var makeBlockTasks = blockDataDict.Keys.Select( blockIndex => { //signal task var doneEvent = new ManualResetEvent(false); doneEvents[eventIndex] = doneEvent; eventIndex += 1; //queue task var blockData = blockDataDict[blockIndex]; var resumeBlocker = new ResumeBlocker( doneEvent, blockData, blockIndex, upToken, putExtra, resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize); return(MakeBlock(resumeBlocker)); }).ToArray(); try { Task.WaitAll(makeBlockTasks); // ReSharper disable once CoVariantArrayConversion WaitHandle.WaitAll(doneEvents); } catch (Exception ex) { Console.WriteLine("wait all exceptions:" + ex.StackTrace); //pass } }
/// <summary> /// 创建块(携带首片数据),同时检查CRC32 /// </summary> /// <param name="resumeBlockerObj">创建分片上次的块请求</param> private void MakeBlock(object resumeBlockerObj) { ResumeBlocker resumeBlocker = (ResumeBlocker)resumeBlockerObj; ManualResetEvent doneEvent = resumeBlocker.DoneEvent; Dictionary <long, HttpResult> blockMakeResults = resumeBlocker.BlockMakeResults; PutExtra putExtra = resumeBlocker.PutExtra; long blockIndex = resumeBlocker.BlockIndex; HttpResult result = new HttpResult(); //check whether to cancel while (true) { UploadControllerAction upCtl = resumeBlocker.PutExtra.UploadController(); if (upCtl == UploadControllerAction.Suspended) { doneEvent.WaitOne(1000); continue; } else if (upCtl == UploadControllerAction.Aborted) { doneEvent.Set(); result.Code = (int)HttpCode.USER_CANCELED; result.RefCode = (int)HttpCode.USER_CANCELED; result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkblk {1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), blockIndex); blockMakeResults.Add(blockIndex, result); return; } else { break; } } byte[] blockBuffer = resumeBlocker.BlockBuffer; int blockSize = blockBuffer.Length; string upToken = resumeBlocker.UploadToken; Dictionary <string, long> uploadedBytesDict = resumeBlocker.UploadedBytesDict; long fileSize = resumeBlocker.FileSize; object progressLock = resumeBlocker.ProgressLock; ResumeInfo resumeInfo = resumeBlocker.ResumeInfo; try { //get upload host string ak = UpToken.GetAccessKeyFromUpToken(upToken); string bucket = UpToken.GetBucketFromUpToken(upToken); if (ak == null || bucket == null) { result = HttpResult.InvalidToken; doneEvent.Set(); return; } string uploadHost = this.config.UpHost(ak, bucket); string url = string.Format("{0}/mkblk/{1}", uploadHost, blockSize); string upTokenStr = string.Format("UpToken {0}", upToken); using (MemoryStream ms = new MemoryStream(blockBuffer, 0, blockSize)) { byte[] data = ms.ToArray(); result = httpManager.PostData(url, data, upTokenStr); if (result.Code == (int)HttpCode.OK) { ResumeContext rc = JsonConvert.DeserializeObject <ResumeContext>(result.Text); if (rc.Crc32 > 0) { uint crc_1 = rc.Crc32; uint crc_2 = CRC32.CheckSumSlice(blockBuffer, 0, blockSize); if (crc_1 != crc_2) { result.RefCode = (int)HttpCode.USER_NEED_RETRY; result.RefText += string.Format(" CRC32: remote={0}, local={1}\n", crc_1, crc_2); } else { //write the mkblk context resumeInfo.Contexts[blockIndex] = rc.Ctx; resumeInfo.ExpiredAt = rc.ExpiredAt; lock (progressLock) { uploadedBytesDict["UploadProgress"] += blockSize; } putExtra.ProgressHandler(uploadedBytesDict["UploadProgress"], fileSize); } } else { result.RefText += string.Format("[{0}] JSON Decode Error: text = {1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Text); result.RefCode = (int)HttpCode.USER_NEED_RETRY; } } else { result.RefCode = (int)HttpCode.USER_NEED_RETRY; } } } catch (Exception ex) { StringBuilder sb = new StringBuilder(); sb.AppendFormat("[{0}] mkblk Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff")); Exception e = ex; while (e != null) { sb.Append(e.Message + " "); e = e.InnerException; } sb.AppendLine(); if (ex is QiniuException) { QiniuException qex = (QiniuException)ex; result.Code = qex.HttpResult.Code; result.RefCode = qex.HttpResult.Code; result.Text = qex.HttpResult.Text; result.RefText += sb.ToString(); } else { result.RefCode = (int)HttpCode.USER_UNDEF; result.RefText += sb.ToString(); } } //return the http result blockMakeResults.Add(blockIndex, result); doneEvent.Set(); }