/// <summary>
        /// 分片上传/断点续上传,带有自定义进度处理和上传控制,检查CRC32,可自动重试
        /// </summary>
        /// <param name="stream">待上传文件流</param>
        /// <param name="key">要保存的文件名称</param>
        /// <param name="upToken">上传凭证</param>
        /// <param name="putExtra">可选配置参数</param>
        /// <returns>上传文件后返回结果</returns>
        public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
        {
            HttpResult result = new HttpResult();

            //check put extra
            if (putExtra == null)
            {
                putExtra = new PutExtra();
            }
            if (putExtra.ProgressHandler == null)
            {
                putExtra.ProgressHandler = DefaultUploadProgressHandler;
            }
            if (putExtra.UploadController == null)
            {
                putExtra.UploadController = DefaultUploadController;
            }
            if (putExtra.MaxRetryTimes == 0)
            {
                putExtra.MaxRetryTimes = DEFAULT_MAX_RETRY_TIMES;
            }

            //start to upload
            try
            {
                long   fileSize    = stream.Length;
                long   chunkSize   = CHUNK_SIZE;
                long   blockSize   = BLOCK_SIZE;
                byte[] chunkBuffer = new byte[chunkSize];
                int    blockCount  = (int)((fileSize + blockSize - 1) / blockSize);
                int    index       = 0; // zero block

                //check resume record file
                ResumeInfo resumeInfo = null;
                if (File.Exists(putExtra.ResumeRecordFile))
                {
                    bool useLastRecord = false;
                    resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
                    if (resumeInfo != null && fileSize == resumeInfo.FileSize)
                    {
                        //check whether ctx expired
                        if (!UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
                        {
                            useLastRecord = true;
                        }
                    }

                    if (useLastRecord)
                    {
                        index = resumeInfo.BlockIndex;
                    }
                }
                if (resumeInfo == null)
                {
                    resumeInfo = new ResumeInfo()
                    {
                        FileSize   = fileSize,
                        BlockIndex = 0,
                        BlockCount = blockCount,
                        Contexts   = new string[blockCount],
                        ExpiredAt  = 0,
                    };
                }

                //read from offset
                long          offset      = index * blockSize;
                string        context     = null;
                long          expiredAt   = 0;
                long          leftBytes   = fileSize - offset;
                long          blockLeft   = 0;
                long          blockOffset = 0;
                HttpResult    hr          = null;
                ResumeContext rc          = null;

                stream.Seek(offset, SeekOrigin.Begin);

                var  upts             = UploadControllerAction.Activated;
                bool bres             = true;
                var  manualResetEvent = new ManualResetEvent(true);
                int  iTry             = 0;

                while (leftBytes > 0)
                {
                    // 每上传一个BLOCK之前,都要检查一下UPTS
                    upts = putExtra.UploadController();

                    if (upts == UploadControllerAction.Aborted)
                    {
                        result.Code     = (int)HttpCode.USER_CANCELED;
                        result.RefCode  = (int)HttpCode.USER_CANCELED;
                        result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
                                                        DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));

                        return(result);
                    }
                    else if (upts == UploadControllerAction.Suspended)
                    {
                        if (bres)
                        {
                            bres = false;
                            manualResetEvent.Reset();

                            result.RefCode  = (int)HttpCode.USER_PAUSED;
                            result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                        }
                        manualResetEvent.WaitOne(1000);
                    }
                    else
                    {
                        if (!bres)
                        {
                            bres = true;
                            manualResetEvent.Set();

                            result.RefCode  = (int)HttpCode.USER_RESUMED;
                            result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is resumed\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                        }

                        #region one-block

                        #region mkblk
                        if (leftBytes < BLOCK_SIZE)
                        {
                            blockSize = leftBytes;
                        }
                        else
                        {
                            blockSize = BLOCK_SIZE;
                        }

                        if (leftBytes < CHUNK_SIZE)
                        {
                            chunkSize = leftBytes;
                        }
                        else
                        {
                            chunkSize = CHUNK_SIZE;
                        }

                        //read data buffer
                        stream.Read(chunkBuffer, 0, (int)chunkSize);

                        iTry = 0;
                        while (++iTry <= putExtra.MaxRetryTimes)
                        {
                            result.RefText += string.Format("[{0}] [ResumableUpload] try mkblk#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);

                            hr = MakeBlock(chunkBuffer, blockSize, chunkSize, upToken);
                            if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
                            {
                                break;
                            }
                        }

                        if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
                        {
                            result.Shadow(hr);
                            result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkblk: code = {1}, text = {2}, offset = {3}, blockSize = {4}, chunkSize = {5}\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockSize, chunkSize);

                            return(result);
                        }

                        if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
                        {
                            result.Shadow(hr);
                            result.RefCode  = (int)HttpCode.USER_UNDEF;
                            result.RefText += string.Format("[{0}] [ResumableUpload] mkblk Error: JSON Decode Error: text = {1}\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);

                            return(result);
                        }

                        context    = rc.Ctx;
                        offset    += chunkSize;
                        leftBytes -= chunkSize;

                        #endregion mkblk
                        putExtra.ProgressHandler(offset, fileSize);

                        if (leftBytes > 0)
                        {
                            blockLeft   = blockSize - chunkSize;
                            blockOffset = chunkSize;
                            while (blockLeft > 0)
                            {
                                #region bput-loop

                                if (blockLeft < CHUNK_SIZE)
                                {
                                    chunkSize = blockLeft;
                                }
                                else
                                {
                                    chunkSize = CHUNK_SIZE;
                                }

                                stream.Read(chunkBuffer, 0, (int)chunkSize);

                                iTry = 0;
                                while (++iTry <= putExtra.MaxRetryTimes)
                                {
                                    result.RefText += string.Format("[{0}] [ResumableUpload] try bput#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);

                                    hr = BputChunk(chunkBuffer, blockOffset, chunkSize, context, upToken);

                                    if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
                                    {
                                        break;
                                    }
                                }
                                if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
                                {
                                    result.Shadow(hr);
                                    result.RefText += string.Format("[{0}] [ResumableUpload] Error: bput: code = {1}, text = {2}, offset = {3}, blockOffset = {4}, chunkSize = {5}\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockOffset, chunkSize);

                                    return(result);
                                }

                                if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
                                {
                                    result.Shadow(hr);
                                    result.RefCode  = (int)HttpCode.USER_UNDEF;
                                    result.RefText += string.Format("[{0}] [ResumableUpload] bput Error: JSON Decode Error: text = {1}\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);

                                    return(result);
                                }

                                context = rc.Ctx;
                                if (expiredAt == 0)
                                {
                                    expiredAt = rc.Expired_At;
                                }

                                offset      += chunkSize;
                                leftBytes   -= chunkSize;
                                blockOffset += chunkSize;
                                blockLeft   -= chunkSize;
                                #endregion bput-loop

                                putExtra.ProgressHandler(offset, fileSize);
                            }
                        }

                        #endregion one-block

                        resumeInfo.BlockIndex      = index;
                        resumeInfo.Contexts[index] = context;
                        resumeInfo.ExpiredAt       = expiredAt;
                        if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
                        {
                            ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
                        }
                        ++index;
                    }
                }

                hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
                if (hr.Code != (int)HttpCode.OK)
                {
                    result.Shadow(hr);
                    result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);

                    return(result);
                }

                if (File.Exists(putExtra.ResumeRecordFile))
                {
                    File.Delete(putExtra.ResumeRecordFile);
                }
                result.Shadow(hr);
                result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
                                                DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.StackTrace);
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                Exception e = ex;
                while (e != null)
                {
                    sb.Append(e.Message + " ");
                    e = e.InnerException;
                }
                sb.AppendLine();

                result.RefCode  = (int)HttpCode.USER_UNDEF;
                result.RefText += sb.ToString();
            }
            finally
            {
                if (stream != null)
                {
                    stream.Close();
                    stream.Dispose();
                }
            }

            return(result);
        }
Пример #2
0
        /// <summary>
        /// 分片上传/断点续上传,带有自定义进度处理和上传控制,检查CRC32,可自动重试
        /// </summary>
        /// <param name="stream">待上传文件流</param>
        /// <param name="key">要保存的文件名称</param>
        /// <param name="upToken">上传凭证</param>
        /// <param name="putExtra">可选配置参数</param>
        /// <returns>上传文件后返回结果</returns>
        public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
        {
            HttpResult result = new HttpResult();

            //check put extra
            if (putExtra == null)
            {
                putExtra = new PutExtra();
            }
            if (putExtra.ProgressHandler == null)
            {
                putExtra.ProgressHandler = DefaultUploadProgressHandler;
            }
            if (putExtra.UploadController == null)
            {
                putExtra.UploadController = DefaultUploadController;
            }

            if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64))
            {
                putExtra.BlockUploadThreads = 1;
            }

            using (stream)
            {
                //start to upload
                try
                {
                    long uploadedBytes = 0;
                    long fileSize      = stream.Length;
                    long blockCount    = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE;

                    //check resume record file
                    ResumeInfo resumeInfo = null;
                    if (File.Exists(putExtra.ResumeRecordFile))
                    {
                        resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
                        if (resumeInfo != null && fileSize == resumeInfo.FileSize)
                        {
                            //check whether ctx expired
                            if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
                            {
                                resumeInfo = null;
                            }
                        }
                    }
                    if (resumeInfo == null)
                    {
                        resumeInfo = new ResumeInfo()
                        {
                            FileSize   = fileSize,
                            BlockCount = blockCount,
                            Contexts   = new string[blockCount],
                            ExpiredAt  = 0,
                        };
                    }

                    //calc upload progress
                    for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
                    {
                        string context = resumeInfo.Contexts[blockIndex];
                        if (!string.IsNullOrEmpty(context))
                        {
                            uploadedBytes += BLOCK_SIZE;
                        }
                    }

                    //set upload progress
                    putExtra.ProgressHandler(uploadedBytes, fileSize);

                    //init block upload error
                    //check not finished blocks to upload
                    UploadControllerAction        upCtrl            = putExtra.UploadController();
                    ManualResetEvent              manualResetEvent  = new ManualResetEvent(false);
                    Dictionary <long, byte[]>     blockDataDict     = new Dictionary <long, byte[]>();
                    Dictionary <long, HttpResult> blockMakeResults  = new Dictionary <long, HttpResult>();
                    Dictionary <string, long>     uploadedBytesDict = new Dictionary <string, long>();
                    uploadedBytesDict.Add("UploadProgress", uploadedBytes);
                    byte[] blockBuffer = new byte[BLOCK_SIZE];
                    for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
                    {
                        string context = resumeInfo.Contexts[blockIndex];
                        if (string.IsNullOrEmpty(context))
                        {
                            //check upload controller action before each chunk
                            while (true)
                            {
                                upCtrl = putExtra.UploadController();

                                if (upCtrl == UploadControllerAction.Aborted)
                                {
                                    result.Code     = (int)HttpCode.USER_CANCELED;
                                    result.RefCode  = (int)HttpCode.USER_CANCELED;
                                    result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                                    manualResetEvent.Set();
                                    return(result);
                                }
                                else if (upCtrl == UploadControllerAction.Suspended)
                                {
                                    result.RefCode  = (int)HttpCode.USER_PAUSED;
                                    result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
                                                                    DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                                    manualResetEvent.WaitOne(1000);
                                }
                                else if (upCtrl == UploadControllerAction.Activated)
                                {
                                    break;
                                }
                            }

                            long offset = blockIndex * BLOCK_SIZE;
                            stream.Seek(offset, SeekOrigin.Begin);
                            int    blockLen  = stream.Read(blockBuffer, 0, BLOCK_SIZE);
                            byte[] blockData = new byte[blockLen];
                            Array.Copy(blockBuffer, blockData, blockLen);
                            blockDataDict.Add(blockIndex, blockData);

                            if (blockDataDict.Count == putExtra.BlockUploadThreads)
                            {
                                processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
                                //check mkblk results
                                foreach (int blkIndex in blockMakeResults.Keys)
                                {
                                    HttpResult mkblkRet = blockMakeResults[blkIndex];
                                    if (mkblkRet.Code != 200)
                                    {
                                        result = mkblkRet;
                                        manualResetEvent.Set();
                                        return(result);
                                    }
                                }
                                blockDataDict.Clear();
                                blockMakeResults.Clear();
                                if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
                                {
                                    ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
                                }
                            }
                        }
                    }

                    if (blockDataDict.Count > 0)
                    {
                        processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
                        //check mkblk results
                        foreach (int blkIndex in blockMakeResults.Keys)
                        {
                            HttpResult mkblkRet = blockMakeResults[blkIndex];
                            if (mkblkRet.Code != 200)
                            {
                                result = mkblkRet;
                                manualResetEvent.Set();
                                return(result);
                            }
                        }
                        blockDataDict.Clear();
                        blockMakeResults.Clear();
                        if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
                        {
                            ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
                        }
                    }

                    if (upCtrl == UploadControllerAction.Activated)
                    {
                        HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
                        if (hr.Code != (int)HttpCode.OK)
                        {
                            result.Shadow(hr);
                            result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
                                                            DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
                        }

                        if (File.Exists(putExtra.ResumeRecordFile))
                        {
                            File.Delete(putExtra.ResumeRecordFile);
                        }
                        result.Shadow(hr);
                        result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
                                                        DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
                    }
                    else
                    {
                        result.Code     = (int)HttpCode.USER_CANCELED;
                        result.RefCode  = (int)HttpCode.USER_CANCELED;
                        result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n",
                                                        DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                    }

                    manualResetEvent.Set();
                    return(result);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.StackTrace);
                    StringBuilder sb = new StringBuilder();
                    sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
                    Exception e = ex;
                    while (e != null)
                    {
                        sb.Append(e.Message + " ");
                        e = e.InnerException;
                    }
                    sb.AppendLine();

                    result.RefCode  = (int)HttpCode.USER_UNDEF;
                    result.RefText += sb.ToString();
                }
            }

            return(result);
        }