예제 #1
0
        private void processMakeBlocks(Dictionary <long, byte[]> blockDataDict, string upToken,
                                       PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
                                       Dictionary <string, long> uploadedBytesDict, long fileSize)
        {
            int taskMax = blockDataDict.Count;

            ManualResetEvent[] doneEvents = new ManualResetEvent[taskMax];
            int    eventIndex             = 0;
            object progressLock           = new object();

            foreach (long blockIndex in blockDataDict.Keys)
            {
                //signal task
                ManualResetEvent doneEvent = new ManualResetEvent(false);
                doneEvents[eventIndex] = doneEvent;
                eventIndex            += 1;

                //queue task
                byte[]        blockData     = blockDataDict[blockIndex];
                ResumeBlocker resumeBlocker = new ResumeBlocker(doneEvent, blockData, blockIndex, upToken, putExtra,
                                                                resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize);
                ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlock), resumeBlocker);
            }

            try
            {
                WaitHandle.WaitAll(doneEvents);
            }
            catch (Exception ex)
            {
                Console.WriteLine("wait all exceptions:" + ex.StackTrace);
                //pass
            }
        }
예제 #2
0
        /// <summary>
        /// 尝试从从文件载入断点信息
        /// </summary>
        /// <param name="recordFile">断点记录文件</param>
        /// <returns>断点信息</returns>
        public static ResumeInfo Load(string recordFile)
        {
            ResumeInfo resumeInfo = null;

            try
            {
                resumeInfo = JsonConvert.DeserializeObject <ResumeInfo>(IOUtils.Api.ReadFile(recordFile));
            }
            catch (Exception)
            {
                resumeInfo = null;
            }

            return(resumeInfo);
        }
예제 #3
0
 public ResumeBlocker(ManualResetEvent doneEvent, byte[] blockBuffer, long blockIndex, string uploadToken,
                      PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
                      object progressLock, Dictionary <string, long> uploadedBytesDict, long fileSize)
 {
     this.DoneEvent         = doneEvent;
     this.BlockBuffer       = blockBuffer;
     this.BlockIndex        = blockIndex;
     this.UploadToken       = uploadToken;
     this.PutExtra          = putExtra;
     this.ResumeInfo        = resumeInfo;
     this.BlockMakeResults  = blockMakeResults;
     this.ProgressLock      = progressLock;
     this.UploadedBytesDict = uploadedBytesDict;
     this.FileSize          = fileSize;
 }
예제 #4
0
        /// <summary>
        /// 分片上传/断点续上传,带有自定义进度处理和上传控制,检查CRC32,可自动重试
        /// </summary>
        /// <param name="stream">待上传文件流</param>
        /// <param name="key">要保存的文件名称</param>
        /// <param name="upToken">上传凭证</param>
        /// <param name="putExtra">可选配置参数</param>
        /// <returns>上传文件后返回结果</returns>
        public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
        {
            HttpResult result = new HttpResult();

            //check put extra
            if (putExtra == null)
            {
                putExtra = new PutExtra();
            }
            if (putExtra.ProgressHandler == null)
            {
                putExtra.ProgressHandler = DefaultUploadProgressHandler;
            }
            if (putExtra.UploadController == null)
            {
                putExtra.UploadController = DefaultUploadController;
            }

            if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64))
            {
                putExtra.BlockUploadThreads = 1;
            }

            using (stream)
            {
                //start to upload
                try
                {
                    long uploadedBytes = 0;
                    long fileSize      = stream.Length;
                    long blockCount    = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE;

                    //check resume record file
                    ResumeInfo resumeInfo = null;
                    if (IOUtils.Api.FileExist(putExtra.ResumeRecordFile))
                    {
                        resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
                        if (resumeInfo != null && fileSize == resumeInfo.FileSize)
                        {
                            //check whether ctx expired
                            if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
                            {
                                resumeInfo = null;
                            }
                        }
                    }
                    if (resumeInfo == null)
                    {
                        resumeInfo = new ResumeInfo()
                        {
                            FileSize   = fileSize,
                            BlockCount = blockCount,
                            Contexts   = new string[blockCount],
                            ExpiredAt  = 0,
                        };
                    }

                    //calc upload progress
                    for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
                    {
                        string context = resumeInfo.Contexts[blockIndex];
                        if (!string.IsNullOrEmpty(context))
                        {
                            uploadedBytes += BLOCK_SIZE;
                        }
                    }

                    //set upload progress
                    putExtra.ProgressHandler(uploadedBytes, fileSize);

                    //init block upload error
                    //check not finished blocks to upload
                    UploadControllerAction        upCtrl            = putExtra.UploadController();
                    ManualResetEvent              manualResetEvent  = new ManualResetEvent(false);
                    Dictionary <long, byte[]>     blockDataDict     = new Dictionary <long, byte[]>();
                    Dictionary <long, HttpResult> blockMakeResults  = new Dictionary <long, HttpResult>();
                    Dictionary <string, long>     uploadedBytesDict = new Dictionary <string, long>();
                    uploadedBytesDict.Add("UploadProgress", uploadedBytes);
                    byte[] blockBuffer = new byte[BLOCK_SIZE];
                    for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
                    {
                        string context = resumeInfo.Contexts[blockIndex];
                        if (string.IsNullOrEmpty(context))
                        {
                            //check upload controller action before each chunk
                            while (true)
                            {
                                upCtrl = putExtra.UploadController();

                                if (upCtrl == UploadControllerAction.Aborted)
                                {
                                    result.Code     = (int)HttpCode.USER_CANCELED;
                                    result.RefCode  = (int)HttpCode.USER_CANCELED;
                                    result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                                    manualResetEvent.Set();
                                    return(result);
                                }
                                else if (upCtrl == UploadControllerAction.Suspended)
                                {
                                    result.RefCode  = (int)HttpCode.USER_PAUSED;
                                    result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                                    manualResetEvent.WaitOne(1000);
                                }
                                else if (upCtrl == UploadControllerAction.Activated)
                                {
                                    break;
                                }
                            }

                            long offset = blockIndex * BLOCK_SIZE;
                            stream.Seek(offset, SeekOrigin.Begin);
                            int    blockLen  = stream.Read(blockBuffer, 0, BLOCK_SIZE);
                            byte[] blockData = new byte[blockLen];
                            Array.Copy(blockBuffer, blockData, blockLen);
                            blockDataDict.Add(blockIndex, blockData);

                            if (blockDataDict.Count == putExtra.BlockUploadThreads)
                            {
                                processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
                                //check mkblk results
                                foreach (int blkIndex in blockMakeResults.Keys)
                                {
                                    HttpResult mkblkRet = blockMakeResults[blkIndex];
                                    if (mkblkRet.Code != 200)
                                    {
                                        result = mkblkRet;
                                        manualResetEvent.Set();
                                        return(result);
                                    }
                                }
                                blockDataDict.Clear();
                                blockMakeResults.Clear();
                                if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
                                {
                                    ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
                                }
                            }
                        }
                    }

                    if (blockDataDict.Count > 0)
                    {
                        processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
                        //check mkblk results
                        foreach (int blkIndex in blockMakeResults.Keys)
                        {
                            HttpResult mkblkRet = blockMakeResults[blkIndex];
                            if (mkblkRet.Code != 200)
                            {
                                result = mkblkRet;
                                manualResetEvent.Set();
                                return(result);
                            }
                        }
                        blockDataDict.Clear();
                        blockMakeResults.Clear();
                        if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
                        {
                            ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
                        }
                    }

                    if (upCtrl == UploadControllerAction.Activated)
                    {
                        HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
                        if (hr.Code != (int)HttpCode.OK)
                        {
                            result.Shadow(hr);
                            result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
                        }

                        if (IOUtils.Api.FileExist(putExtra.ResumeRecordFile))
                        {
                            IOUtils.Api.FileDelete(putExtra.ResumeRecordFile);
                        }
                        result.Shadow(hr);
                        result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
                                                        DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
                    }
                    else
                    {
                        result.Code     = (int)HttpCode.USER_CANCELED;
                        result.RefCode  = (int)HttpCode.USER_CANCELED;
                        result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n",
                                                        DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                    }

                    manualResetEvent.Set();
                    return(result);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.StackTrace);
                    StringBuilder sb = new StringBuilder();
                    sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                    Exception e = ex;
                    while (e != null)
                    {
                        sb.Append(e.Message + " ");
                        e = e.InnerException;
                    }
                    sb.AppendLine();

                    result.RefCode  = (int)HttpCode.USER_UNDEF;
                    result.RefText += sb.ToString();
                }
            }

            return(result);
        }
예제 #5
0
        /// <summary>
        /// 创建块(携带首片数据),同时检查CRC32
        /// </summary>
        /// <param name="resumeBlockerObj">创建分片上次的块请求</param>
        private void MakeBlock(object resumeBlockerObj)
        {
            ResumeBlocker    resumeBlocker = (ResumeBlocker)resumeBlockerObj;
            ManualResetEvent doneEvent     = resumeBlocker.DoneEvent;
            Dictionary <long, HttpResult> blockMakeResults = resumeBlocker.BlockMakeResults;
            PutExtra   putExtra   = resumeBlocker.PutExtra;
            long       blockIndex = resumeBlocker.BlockIndex;
            HttpResult result     = new HttpResult();

            //check whether to cancel
            while (true)
            {
                UploadControllerAction upCtl = resumeBlocker.PutExtra.UploadController();
                if (upCtl == UploadControllerAction.Suspended)
                {
                    doneEvent.WaitOne(1000);
                    continue;
                }
                else if (upCtl == UploadControllerAction.Aborted)
                {
                    doneEvent.Set();

                    result.Code     = (int)HttpCode.USER_CANCELED;
                    result.RefCode  = (int)HttpCode.USER_CANCELED;
                    result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkblk {1}\n",
                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), blockIndex);
                    blockMakeResults.Add(blockIndex, result);
                    return;
                }
                else
                {
                    break;
                }
            }

            byte[] blockBuffer = resumeBlocker.BlockBuffer;
            int    blockSize   = blockBuffer.Length;

            string upToken = resumeBlocker.UploadToken;
            Dictionary <string, long> uploadedBytesDict = resumeBlocker.UploadedBytesDict;
            long       fileSize     = resumeBlocker.FileSize;
            object     progressLock = resumeBlocker.ProgressLock;
            ResumeInfo resumeInfo   = resumeBlocker.ResumeInfo;

            try
            {
                //get upload host
                string ak     = UpToken.GetAccessKeyFromUpToken(upToken);
                string bucket = UpToken.GetBucketFromUpToken(upToken);
                if (ak == null || bucket == null)
                {
                    result = HttpResult.InvalidToken;
                    doneEvent.Set();
                    return;
                }

                string uploadHost = this.config.UpHost(ak, bucket);

                string url        = string.Format("{0}/mkblk/{1}", uploadHost, blockSize);
                string upTokenStr = string.Format("UpToken {0}", upToken);
                using (MemoryStream ms = new MemoryStream(blockBuffer, 0, blockSize))
                {
                    byte[] data = ms.ToArray();

                    result = httpManager.PostData(url, data, upTokenStr);

                    if (result.Code == (int)HttpCode.OK)
                    {
                        ResumeContext rc = JsonConvert.DeserializeObject <ResumeContext>(result.Text);

                        if (rc.Crc32 > 0)
                        {
                            uint crc_1 = rc.Crc32;
                            uint crc_2 = CRC32.CheckSumSlice(blockBuffer, 0, blockSize);
                            if (crc_1 != crc_2)
                            {
                                result.RefCode  = (int)HttpCode.USER_NEED_RETRY;
                                result.RefText += string.Format(" CRC32: remote={0}, local={1}\n", crc_1, crc_2);
                            }
                            else
                            {
                                //write the mkblk context
                                resumeInfo.Contexts[blockIndex] = rc.Ctx;
                                resumeInfo.ExpiredAt            = rc.ExpiredAt;
                                lock (progressLock)
                                {
                                    uploadedBytesDict["UploadProgress"] += blockSize;
                                }
                                putExtra.ProgressHandler(uploadedBytesDict["UploadProgress"], fileSize);
                            }
                        }
                        else
                        {
                            result.RefText += string.Format("[{0}] JSON Decode Error: text = {1}",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Text);
                            result.RefCode = (int)HttpCode.USER_NEED_RETRY;
                        }
                    }
                    else
                    {
                        result.RefCode = (int)HttpCode.USER_NEED_RETRY;
                    }
                }
            }
            catch (Exception ex)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("[{0}] mkblk Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                Exception e = ex;
                while (e != null)
                {
                    sb.Append(e.Message + " ");
                    e = e.InnerException;
                }
                sb.AppendLine();

                if (ex is QiniuException)
                {
                    QiniuException qex = (QiniuException)ex;
                    result.Code     = qex.HttpResult.Code;
                    result.RefCode  = qex.HttpResult.Code;
                    result.Text     = qex.HttpResult.Text;
                    result.RefText += sb.ToString();
                }
                else
                {
                    result.RefCode  = (int)HttpCode.USER_UNDEF;
                    result.RefText += sb.ToString();
                }
            }

            //return the http result
            blockMakeResults.Add(blockIndex, result);
            doneEvent.Set();
        }
예제 #6
0
        /// <summary>
        /// 保存断点信息到文件
        /// </summary>
        /// <param name="resumeInfo">断点信息</param>
        /// <param name="recordFile">断点记录文件</param>
        public static void Save(ResumeInfo resumeInfo, string recordFile)
        {
            var jsonStr = resumeInfo.ToJsonStr();

            IOUtils.Api.WriteFile(recordFile, jsonStr);
        }