private void processMakeBlocks(Dictionary <long, byte[]> blockDataDict, string upToken,
PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
Dictionary <string, long> uploadedBytesDict, long fileSize)
{
int taskMax = blockDataDict.Count;
ManualResetEvent[] doneEvents = new ManualResetEvent[taskMax];
int eventIndex = 0;
object progressLock = new object();
foreach (long blockIndex in blockDataDict.Keys)
{
//signal task
ManualResetEvent doneEvent = new ManualResetEvent(false);
doneEvents[eventIndex] = doneEvent;
eventIndex += 1;
//queue task
byte[] blockData = blockDataDict[blockIndex];
ResumeBlocker resumeBlocker = new ResumeBlocker(doneEvent, blockData, blockIndex, upToken, putExtra,
resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize);
ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlock), resumeBlocker);
}
try
{
WaitHandle.WaitAll(doneEvents);
}
catch (Exception ex)
{
Console.WriteLine("wait all exceptions:" + ex.StackTrace);
//pass
}
}
private void ProcessMakeBlocks(
Dictionary <long, byte[]> blockDataDict,
string upToken,
PutExtra putExtra,
ResumeInfo resumeInfo,
Dictionary <long, HttpResult> blockMakeResults,
Dictionary <string, long> uploadedBytesDict,
long fileSize)
{
var taskMax = blockDataDict.Count;
var doneEvents = new ManualResetEvent[taskMax];
var eventIndex = 0;
var progressLock = new object();
var makeBlockTasks = blockDataDict.Keys.Select(
blockIndex =>
{
//signal task
var doneEvent = new ManualResetEvent(false);
doneEvents[eventIndex] = doneEvent;
eventIndex += 1;
//queue task
var blockData = blockDataDict[blockIndex];
var resumeBlocker = new ResumeBlocker(
doneEvent,
blockData,
blockIndex,
upToken,
putExtra,
resumeInfo,
blockMakeResults,
progressLock,
uploadedBytesDict,
fileSize);
return(MakeBlock(resumeBlocker));
}).ToArray();
try
{
Task.WaitAll(makeBlockTasks);
// ReSharper disable once CoVariantArrayConversion
WaitHandle.WaitAll(doneEvents);
}
catch (Exception ex)
{
Console.WriteLine("wait all exceptions:" + ex.StackTrace);
//pass
}
}
/// <summary>
/// 分片上传，支持断点续上传，带有自定义进度处理、高级控制功能
/// </summary>
/// <param name="localFile">本地待上传的文件名</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="putExtra">上传可选配置</param>
/// <returns>上传文件后的返回结果</returns>
public HttpResult UploadFile(string localFile, string key, string token, PutExtra putExtra)
{
try
{
FileStream fs = new FileStream(localFile, FileMode.Open);
return(this.UploadStream(fs, key, token, putExtra));
}
catch (Exception ex)
{
HttpResult ret = HttpResult.InvalidFile;
ret.RefText = ex.Message;
return(ret);
}
}
public ResumeBlocker(ManualResetEvent doneEvent, byte[] blockBuffer, long blockIndex, string uploadToken,
PutExtra putExtra, ResumeInfo resumeInfo, Dictionary <long, HttpResult> blockMakeResults,
object progressLock, Dictionary <string, long> uploadedBytesDict, long fileSize)
{
this.DoneEvent = doneEvent;
this.BlockBuffer = blockBuffer;
this.BlockIndex = blockIndex;
this.UploadToken = uploadToken;
this.PutExtra = putExtra;
this.ResumeInfo = resumeInfo;
this.BlockMakeResults = blockMakeResults;
this.ProgressLock = progressLock;
this.UploadedBytesDict = uploadedBytesDict;
this.FileSize = fileSize;
}
/// <summary>
/// 上传文件数据流，根据文件大小以及设置的阈值(用户初始化UploadManager时可指定该值)自动选择：
/// 若文件大小超过设定阈值，使用ResumableUploader，否则使用FormUploader
/// </summary>
/// <param name="stream">待上传的数据流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string token, PutExtra extra)
{
HttpResult result = new HttpResult();
if (stream.Length > this.config.PutThreshold)
{
ResumableUploader resumeUploader = new ResumableUploader(this.config);
result = resumeUploader.UploadStream(stream, key, token, extra);
}
else
{
FormUploader formUploader = new FormUploader(this.config);
result = formUploader.UploadStream(stream, key, token, extra);
}
return(result);
}
/// <summary>
/// 上传文件，根据文件大小以及设置的阈值(用户初始化UploadManager时可指定该值)自动选择：
/// 若文件大小超过设定阈值，使用ResumableUploader，否则使用FormUploader
/// </summary>
/// <param name="localFile">本地待上传的文件名</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public HttpResult UploadFile(string localFile, string key, string token, PutExtra extra)
{
HttpResult result = new HttpResult();
System.IO.FileInfo fi = new System.IO.FileInfo(localFile);
if (fi.Length > this.config.PutThreshold)
{
ResumableUploader resumeUploader = new ResumableUploader(config);
result = resumeUploader.UploadFile(localFile, key, token, extra);
}
else
{
FormUploader formUploader = new FormUploader(config);
result = formUploader.UploadFile(localFile, key, token, extra);
}
return(result);
}
/// <summary>
/// 根据已上传的所有分片数据创建文件
/// </summary>
/// <param name="fileName">源文件名</param>
/// <param name="size">文件大小</param>
/// <param name="key">要保存的文件名</param>
/// <param name="contexts">所有数据块的Context</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">用户指定的额外参数</param>
/// <returns>此操作执行后的返回结果</returns>
private HttpResult MakeFile(string fileName, long size, string key, string upToken, PutExtra putExtra, string[] contexts)
{
HttpResult result = new HttpResult();
try
{
string fnameStr = "fname";
string mimeTypeStr = "";
string keyStr = "";
string paramStr = "";
//check file name
if (!string.IsNullOrEmpty(fileName))
{
fnameStr = string.Format("/fname/{0}", Base64.UrlSafeBase64Encode(fileName));
}
//check mime type
if (!string.IsNullOrEmpty(putExtra.MimeType))
{
mimeTypeStr = string.Format("/mimeType/{0}", Base64.UrlSafeBase64Encode(putExtra.MimeType));
}
//check key
if (!string.IsNullOrEmpty(key))
{
keyStr = string.Format("/key/{0}", Base64.UrlSafeBase64Encode(key));
}
//check extra params
if (putExtra.Params != null && putExtra.Params.Count > 0)
{
StringBuilder sb = new StringBuilder();
foreach (var kvp in putExtra.Params)
{
string k = kvp.Key;
string v = kvp.Value;
if (k.StartsWith("x:") && !string.IsNullOrEmpty(v))
{
sb.AppendFormat("/{0}/{1}", k, v);
}
}
paramStr = sb.ToString();
}
//get upload host
string ak = UpToken.GetAccessKeyFromUpToken(upToken);
string bucket = UpToken.GetBucketFromUpToken(upToken);
if (ak == null || bucket == null)
{
return(HttpResult.InvalidToken);
}
string uploadHost = this.config.UpHost(ak, bucket);
string url = string.Format("{0}/mkfile/{1}{2}{3}{4}{5}", uploadHost, size, mimeTypeStr, fnameStr, keyStr, paramStr);
string body = string.Join(",", contexts);
string upTokenStr = string.Format("UpToken {0}", upToken);
result = httpManager.PostText(url, body, upTokenStr);
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] mkfile Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException)
{
QiniuException qex = (QiniuException)ex;
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
/// <summary>
/// 上传数据流
/// </summary>
/// <param name="stream">(确定长度的)数据流</param>
/// <param name="key">要保存的key</param>
/// <param name="token">上传凭证</param>
/// <param name="putExtra">上传可选设置</param>
/// <returns>上传数据流后的返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string token, PutExtra putExtra)
{
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (string.IsNullOrEmpty(putExtra.MimeType))
{
putExtra.MimeType = "application/octet-stream";
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
string fname = key;
if (string.IsNullOrEmpty(key))
{
fname = "fname_temp";
}
HttpResult result = new HttpResult();
try
{
string boundary = HttpManager.CreateFormDataBoundary();
StringBuilder bodyBuilder = new StringBuilder();
bodyBuilder.AppendLine("--" + boundary);
if (key != null)
{
//write key when it is not null
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"key\"");
bodyBuilder.AppendLine();
bodyBuilder.AppendLine(key);
bodyBuilder.AppendLine("--" + boundary);
}
//write token
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"token\"");
bodyBuilder.AppendLine();
bodyBuilder.AppendLine(token);
bodyBuilder.AppendLine("--" + boundary);
//write extra params
if (putExtra.Params != null && putExtra.Params.Count > 0)
{
foreach (var p in putExtra.Params)
{
if (p.Key.StartsWith("x:"))
{
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"{0}\"", p.Key);
bodyBuilder.AppendLine();
bodyBuilder.AppendLine();
bodyBuilder.AppendLine(p.Value);
bodyBuilder.AppendLine("--" + boundary);
}
}
}
//prepare data buffer
int bufferSize = 1024 * 1024;
byte[] buffer = new byte[bufferSize];
int bytesRead = 0;
putExtra.ProgressHandler(0, stream.Length);
MemoryStream dataMS = new MemoryStream();
while ((bytesRead = stream.Read(buffer, 0, bufferSize)) != 0)
{
dataMS.Write(buffer, 0, bytesRead);
}
//write crc32
uint crc32 = CRC32.CheckSumBytes(dataMS.ToArray());
//write key when it is not null
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"crc32\"");
bodyBuilder.AppendLine();
bodyBuilder.AppendLine(crc32.ToString());
bodyBuilder.AppendLine("--" + boundary);
//write fname
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"file\"; filename=\"{0}\"", fname);
bodyBuilder.AppendLine();
//write mime type
bodyBuilder.AppendFormat("Content-Type: {0}", putExtra.MimeType);
bodyBuilder.AppendLine();
bodyBuilder.AppendLine();
//write file data
StringBuilder bodyEnd = new StringBuilder();
bodyEnd.AppendLine();
bodyEnd.AppendLine("--" + boundary + "--");
byte[] partData1 = Encoding.UTF8.GetBytes(bodyBuilder.ToString());
byte[] partData2 = dataMS.ToArray();
byte[] partData3 = Encoding.UTF8.GetBytes(bodyEnd.ToString());
MemoryStream ms = new MemoryStream();
ms.Write(partData1, 0, partData1.Length);
ms.Write(partData2, 0, partData2.Length);
ms.Write(partData3, 0, partData3.Length);
//get upload host
string ak = UpToken.GetAccessKeyFromUpToken(token);
string bucket = UpToken.GetBucketFromUpToken(token);
if (ak == null || bucket == null)
{
return(HttpResult.InvalidToken);
}
string uploadHost = this.config.UpHost(ak, bucket);
putExtra.ProgressHandler(stream.Length / 5, stream.Length);
result = httpManager.PostMultipart(uploadHost, ms.ToArray(), boundary, null);
putExtra.ProgressHandler(stream.Length, stream.Length);
if (result.Code == (int)HttpCode.OK)
{
result.RefText += string.Format("[{0}] [FormUpload] Uploaded: #STREAM# ==> \"{1}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), key);
}
else
{
result.RefText += string.Format("[{0}] [FormUpload] Failed: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Code, result.Text);
}
//close memory stream
ms.Close();
dataMS.Close();
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [FormUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException)
{
QiniuException qex = (QiniuException)ex;
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
finally
{
if (stream != null)
{
try
{
stream.Close();
stream.Dispose();
}
catch (Exception) { }
}
}
return(result);
}
/// <summary>
/// 上传数据
/// </summary>
/// <param name="data">待上传的数据</param>
/// <param name="key">要保存的key</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传数据后的返回结果</returns>
public HttpResult UploadData(byte[] data, string key, string token, PutExtra extra)
{
MemoryStream stream = new MemoryStream(data);
return(this.UploadStream(stream, key, token, extra));
}
/// <summary>
/// 上传数据
/// </summary>
/// <param name="data">待上传的数据</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public HttpResult UploadData(byte[] data, string key, string token, PutExtra extra)
{
FormUploader formUploader = new FormUploader(this.config);
return(formUploader.UploadData(data, key, token, extra));
}
/// <summary>
/// 创建块(携带首片数据),同时检查CRC32
/// </summary>
/// <param name="resumeBlockerObj">创建分片上次的块请求</param>
private void MakeBlock(object resumeBlockerObj)
{
ResumeBlocker resumeBlocker = (ResumeBlocker)resumeBlockerObj;
ManualResetEvent doneEvent = resumeBlocker.DoneEvent;
Dictionary <long, HttpResult> blockMakeResults = resumeBlocker.BlockMakeResults;
PutExtra putExtra = resumeBlocker.PutExtra;
long blockIndex = resumeBlocker.BlockIndex;
HttpResult result = new HttpResult();
//check whether to cancel
while (true)
{
UploadControllerAction upCtl = resumeBlocker.PutExtra.UploadController();
if (upCtl == UploadControllerAction.Suspended)
{
doneEvent.WaitOne(1000);
continue;
}
else if (upCtl == UploadControllerAction.Aborted)
{
doneEvent.Set();
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkblk {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), blockIndex);
blockMakeResults.Add(blockIndex, result);
return;
}
else
{
break;
}
}
byte[] blockBuffer = resumeBlocker.BlockBuffer;
int blockSize = blockBuffer.Length;
string upToken = resumeBlocker.UploadToken;
Dictionary <string, long> uploadedBytesDict = resumeBlocker.UploadedBytesDict;
long fileSize = resumeBlocker.FileSize;
object progressLock = resumeBlocker.ProgressLock;
ResumeInfo resumeInfo = resumeBlocker.ResumeInfo;
try
{
//get upload host
string ak = UpToken.GetAccessKeyFromUpToken(upToken);
string bucket = UpToken.GetBucketFromUpToken(upToken);
if (ak == null || bucket == null)
{
result = HttpResult.InvalidToken;
doneEvent.Set();
return;
}
string uploadHost = this.config.UpHost(ak, bucket);
string url = string.Format("{0}/mkblk/{1}", uploadHost, blockSize);
string upTokenStr = string.Format("UpToken {0}", upToken);
using (MemoryStream ms = new MemoryStream(blockBuffer, 0, blockSize))
{
byte[] data = ms.ToArray();
result = httpManager.PostData(url, data, upTokenStr);
if (result.Code == (int)HttpCode.OK)
{
ResumeContext rc = JsonConvert.DeserializeObject <ResumeContext>(result.Text);
if (rc.Crc32 > 0)
{
uint crc_1 = rc.Crc32;
uint crc_2 = CRC32.CheckSumSlice(blockBuffer, 0, blockSize);
if (crc_1 != crc_2)
{
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
result.RefText += string.Format(" CRC32: remote={0}, local={1}\n", crc_1, crc_2);
}
else
{
//write the mkblk context
resumeInfo.Contexts[blockIndex] = rc.Ctx;
resumeInfo.ExpiredAt = rc.ExpiredAt;
lock (progressLock)
{
uploadedBytesDict["UploadProgress"] += blockSize;
}
putExtra.ProgressHandler(uploadedBytesDict["UploadProgress"], fileSize);
}
}
else
{
result.RefText += string.Format("[{0}] JSON Decode Error: text = {1}",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Text);
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
}
}
else
{
result.RefCode = (int)HttpCode.USER_NEED_RETRY;
}
}
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] mkblk Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException)
{
QiniuException qex = (QiniuException)ex;
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
//return the http result
blockMakeResults.Add(blockIndex, result);
doneEvent.Set();
}
/// <summary>
/// 分片上传/断点续上传，带有自定义进度处理和上传控制，检查CRC32，可自动重试
/// </summary>
/// <param name="stream">待上传文件流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">可选配置参数</param>
/// <returns>上传文件后返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
{
HttpResult result = new HttpResult();
//check put extra
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
if (!(putExtra.BlockUploadThreads > 0 && putExtra.BlockUploadThreads <= 64))
{
putExtra.BlockUploadThreads = 1;
}
using (stream)
{
//start to upload
try
{
long uploadedBytes = 0;
long fileSize = stream.Length;
long blockCount = (fileSize + BLOCK_SIZE - 1) / BLOCK_SIZE;
//check resume record file
ResumeInfo resumeInfo = null;
if (File.Exists(putExtra.ResumeRecordFile))
{
resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
if (resumeInfo != null && fileSize == resumeInfo.FileSize)
{
//check whether ctx expired
if (UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
{
resumeInfo = null;
}
}
}
if (resumeInfo == null)
{
resumeInfo = new ResumeInfo()
{
FileSize = fileSize,
BlockCount = blockCount,
Contexts = new string[blockCount],
ExpiredAt = 0,
};
}
//calc upload progress
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (!string.IsNullOrEmpty(context))
{
uploadedBytes += BLOCK_SIZE;
}
}
//set upload progress
putExtra.ProgressHandler(uploadedBytes, fileSize);
//init block upload error
//check not finished blocks to upload
UploadControllerAction upCtrl = putExtra.UploadController();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
Dictionary <long, byte[]> blockDataDict = new Dictionary <long, byte[]>();
Dictionary <long, HttpResult> blockMakeResults = new Dictionary <long, HttpResult>();
Dictionary <string, long> uploadedBytesDict = new Dictionary <string, long>();
uploadedBytesDict.Add("UploadProgress", uploadedBytes);
byte[] blockBuffer = new byte[BLOCK_SIZE];
for (long blockIndex = 0; blockIndex < blockCount; blockIndex++)
{
string context = resumeInfo.Contexts[blockIndex];
if (string.IsNullOrEmpty(context))
{
//check upload controller action before each chunk
while (true)
{
upCtrl = putExtra.UploadController();
if (upCtrl == UploadControllerAction.Aborted)
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.Set();
return(result);
}
else if (upCtrl == UploadControllerAction.Suspended)
{
result.RefCode = (int)HttpCode.USER_PAUSED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
manualResetEvent.WaitOne(1000);
}
else if (upCtrl == UploadControllerAction.Activated)
{
break;
}
}
long offset = blockIndex * BLOCK_SIZE;
stream.Seek(offset, SeekOrigin.Begin);
int blockLen = stream.Read(blockBuffer, 0, BLOCK_SIZE);
byte[] blockData = new byte[blockLen];
Array.Copy(blockBuffer, blockData, blockLen);
blockDataDict.Add(blockIndex, blockData);
if (blockDataDict.Count == putExtra.BlockUploadThreads)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
}
}
if (blockDataDict.Count > 0)
{
processMakeBlocks(blockDataDict, upToken, putExtra, resumeInfo, blockMakeResults, uploadedBytesDict, fileSize);
//check mkblk results
foreach (int blkIndex in blockMakeResults.Keys)
{
HttpResult mkblkRet = blockMakeResults[blkIndex];
if (mkblkRet.Code != 200)
{
result = mkblkRet;
manualResetEvent.Set();
return(result);
}
}
blockDataDict.Clear();
blockMakeResults.Clear();
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
}
if (upCtrl == UploadControllerAction.Activated)
{
HttpResult hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
if (hr.Code != (int)HttpCode.OK)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
}
if (File.Exists(putExtra.ResumeRecordFile))
{
File.Delete(putExtra.ResumeRecordFile);
}
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
}
else
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkfile\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
manualResetEvent.Set();
return(result);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
/// <summary>
/// 上传文件 - 可附加自定义参数
/// </summary>
/// <param name="localFile">待上传的本地文件</param>
/// <param name="key">要保存的目标文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public async Task <HttpResult> UploadFile(string localFile, string key, string token, PutExtra extra)
{
try
{
var fs = new FileStream(localFile, FileMode.Open);
return(await UploadStream(fs, key, token, extra));
}
catch (Exception ex)
{
var ret = HttpResult.InvalidFile;
ret.RefText = ex.Message;
return(ret);
}
}
/// <summary>
/// 根据已上传的所有分片数据创建文件
/// </summary>
/// <param name="fileName">源文件名</param>
/// <param name="size">文件大小</param>
/// <param name="key">要保存的文件名</param>
/// <param name="contexts">所有数据块的Context</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">用户指定的额外参数</param>
/// <returns>此操作执行后的返回结果</returns>
private async Task <HttpResult> MakeFile(string fileName, long size, string key, string upToken, PutExtra putExtra, string[] contexts)
{
var result = new HttpResult();
try
{
var fnameStr = "fname";
var mimeTypeStr = "";
var keyStr = "";
var paramStr = "";
//check file name
if (!string.IsNullOrEmpty(fileName))
{
fnameStr = $"/fname/{Base64.UrlSafeBase64Encode(fileName)}";
}
//check mime type
if (!string.IsNullOrEmpty(putExtra.MimeType))
{
mimeTypeStr = $"/mimeType/{Base64.UrlSafeBase64Encode(putExtra.MimeType)}";
}
//check key
if (!string.IsNullOrEmpty(key))
{
keyStr = $"/key/{Base64.UrlSafeBase64Encode(key)}";
}
//check extra params
if (putExtra.Params != null && putExtra.Params.Count > 0)
{
var sb = new StringBuilder();
foreach (var kvp in putExtra.Params)
{
var k = kvp.Key;
var v = kvp.Value;
if (k.StartsWith("x:") && !string.IsNullOrEmpty(v))
{
sb.Append($"/{k}/{v}");
}
}
paramStr = sb.ToString();
}
//get upload host
var ak = UpToken.GetAccessKeyFromUpToken(upToken);
var bucket = UpToken.GetBucketFromUpToken(upToken);
if (ak == null || bucket == null)
{
return(HttpResult.InvalidToken);
}
var uploadHost = await _config.UpHost(ak, bucket);
var url = $"{uploadHost}/mkfile/{size}{mimeTypeStr}{fnameStr}{keyStr}{paramStr}";
var body = string.Join(",", contexts);
var upTokenStr = $"UpToken {upToken}";
result = await _httpManager.PostTextAsync(url, body, upTokenStr);
}
catch (Exception ex)
{
var sb = new StringBuilder();
sb.Append($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.ffff}] mkfile Error: ");
var e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException qex)
{
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
/// <summary>
/// 上传文件数据流，根据文件大小以及设置的阈值(用户初始化UploadManager时可指定该值)自动选择：
/// 若文件大小超过设定阈值，使用ResumableUploader，否则使用FormUploader
/// </summary>
/// <param name="stream">待上传的数据流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public Task <HttpResult> UploadStream(Stream stream, string key, string token, PutExtra extra)
{
if (stream.Length > _config.PutThreshold)
{
var resumeUploader = new ResumableUploader(_config);
return(resumeUploader.UploadStream(stream, key, token, extra));
}
var formUploader = new FormUploader(_config);
return(formUploader.UploadStream(stream, key, token, extra));
}
/// <summary>
/// 上传文件，根据文件大小以及设置的阈值(用户初始化UploadManager时可指定该值)自动选择：
/// 若文件大小超过设定阈值，使用ResumableUploader，否则使用FormUploader
/// </summary>
/// <param name="localFile">本地待上传的文件名</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public Task <HttpResult> UploadFile(string localFile, string key, string token, PutExtra extra)
{
var fi = new System.IO.FileInfo(localFile);
if (fi.Length > _config.PutThreshold)
{
var resumeUploader = new ResumableUploader(_config);
return(resumeUploader.UploadFile(localFile, key, token, extra));
}
var formUploader = new FormUploader(_config);
return(formUploader.UploadFile(localFile, key, token, extra));
}
/// <summary>
/// 上传数据
/// </summary>
/// <param name="data">待上传的数据</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传文件后的返回结果</returns>
public Task <HttpResult> UploadData(byte[] data, string key, string token, PutExtra extra)
{
var formUploader = new FormUploader(_config);
return(formUploader.UploadData(data, key, token, extra));
}
/// <summary>
/// 上传数据流
/// </summary>
/// <param name="stream">(确定长度的)数据流</param>
/// <param name="key">要保存的key</param>
/// <param name="token">上传凭证</param>
/// <param name="putExtra">上传可选设置</param>
/// <returns>上传数据流后的返回结果</returns>
public async Task <HttpResult> UploadStream(Stream stream, string key, string token, PutExtra putExtra)
{
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (string.IsNullOrEmpty(putExtra.MimeType))
{
putExtra.MimeType = ContentType.APPLICATION_OCTET_STREAM;
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
var fileName = key;
if (string.IsNullOrEmpty(key))
{
fileName = "fname_temp";
}
var result = new HttpResult();
try
{
var boundary = HttpManager.CreateFormDataBoundary();
var content = new MultipartFormDataContent(boundary);
var length = stream.Length;
putExtra.ProgressHandler(0, length);
// Key
if (!string.IsNullOrEmpty(key))
{
content.Add(new StringContent(key), "key");
}
// Token
content.Add(new StringContent(token), "token");
// Other params
if (putExtra.Params != null)
{
foreach (var param in putExtra.Params)
{
content.Add(new StringContent(param.Value), param.Key);
}
}
// Reuse stream
if (!stream.CanSeek)
{
var ms = new MemoryStream((int)stream.Length);
stream.CopyTo(ms);
stream.Dispose();
stream = ms;
}
// CRC32
var crc32 = Crc32.CheckSumStream(stream);
stream.Seek(0, SeekOrigin.Begin);
content.Add(new StringContent(crc32.ToString()), "crc32");
// Primary content
var part = new StreamContent(stream);
part.Headers.ContentType = MediaTypeHeaderValue.Parse(putExtra.MimeType);
content.Add(part, "file", fileName);
// Get upload host
var ak = UpToken.GetAccessKeyFromUpToken(token);
var bucket = UpToken.GetBucketFromUpToken(token);
if (ak == null || bucket == null)
{
return(HttpResult.InvalidToken);
}
var uploadHost = await _config.UpHost(ak, bucket);
// TODO: Real progress
putExtra.ProgressHandler(length / 5, length);
result = await _httpManager.PostAsync(uploadHost, content, boundary);
putExtra.ProgressHandler(length, length);
if (result.Code == (int)HttpCode.OK)
{
result.RefText += $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.ffff}] [FormUpload] Uploaded: #STREAM# ==> \"{key}\"\n";
}
else
{
result.RefText += $"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.ffff}] [FormUpload] Failed: code = {result.Code}, text = {result.Text}\n";
}
}
catch (Exception ex)
{
var sb = new StringBuilder();
sb.Append($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.ffff}] [FormUpload] Error: ");
var e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException qex)
{
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
/// <summary>
/// 上传数据
/// </summary>
/// <param name="data">待上传的数据</param>
/// <param name="key">要保存的key</param>
/// <param name="token">上传凭证</param>
/// <param name="extra">上传可选设置</param>
/// <returns>上传数据后的返回结果</returns>
public Task <HttpResult> UploadData(byte[] data, string key, string token, PutExtra extra)
{
var stream = new MemoryStream(data);
return(UploadStream(stream, key, token, extra));
}
/// <summary>
/// 分片上传/断点续上传，带有自定义进度处理和上传控制，检查CRC32，可自动重试
/// </summary>
/// <param name="stream">待上传文件流</param>
/// <param name="key">要保存的文件名称</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">可选配置参数</param>
/// <returns>上传文件后返回结果</returns>
public HttpResult UploadStream(Stream stream, string key, string upToken, PutExtra putExtra)
{
HttpResult result = new HttpResult();
//check put extra
if (putExtra == null)
{
putExtra = new PutExtra();
}
if (putExtra.ProgressHandler == null)
{
putExtra.ProgressHandler = DefaultUploadProgressHandler;
}
if (putExtra.UploadController == null)
{
putExtra.UploadController = DefaultUploadController;
}
if (putExtra.MaxRetryTimes == 0)
{
putExtra.MaxRetryTimes = DEFAULT_MAX_RETRY_TIMES;
}
//start to upload
try
{
long fileSize = stream.Length;
long chunkSize = CHUNK_SIZE;
long blockSize = BLOCK_SIZE;
byte[] chunkBuffer = new byte[chunkSize];
int blockCount = (int)((fileSize + blockSize - 1) / blockSize);
int index = 0; // zero block
//check resume record file
ResumeInfo resumeInfo = null;
if (File.Exists(putExtra.ResumeRecordFile))
{
bool useLastRecord = false;
resumeInfo = ResumeHelper.Load(putExtra.ResumeRecordFile);
if (resumeInfo != null && fileSize == resumeInfo.FileSize)
{
//check whether ctx expired
if (!UnixTimestamp.IsContextExpired(resumeInfo.ExpiredAt))
{
useLastRecord = true;
}
}
if (useLastRecord)
{
index = resumeInfo.BlockIndex;
}
}
if (resumeInfo == null)
{
resumeInfo = new ResumeInfo()
{
FileSize = fileSize,
BlockIndex = 0,
BlockCount = blockCount,
Contexts = new string[blockCount],
ExpiredAt = 0,
};
}
//read from offset
long offset = index * blockSize;
string context = null;
long expiredAt = 0;
long leftBytes = fileSize - offset;
long blockLeft = 0;
long blockOffset = 0;
HttpResult hr = null;
ResumeContext rc = null;
stream.Seek(offset, SeekOrigin.Begin);
var upts = UploadControllerAction.Activated;
bool bres = true;
var manualResetEvent = new ManualResetEvent(true);
int iTry = 0;
while (leftBytes > 0)
{
// 每上传一个BLOCK之前，都要检查一下UPTS
upts = putExtra.UploadController();
if (upts == UploadControllerAction.Aborted)
{
result.Code = (int)HttpCode.USER_CANCELED;
result.RefCode = (int)HttpCode.USER_CANCELED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
return(result);
}
else if (upts == UploadControllerAction.Suspended)
{
if (bres)
{
bres = false;
manualResetEvent.Reset();
result.RefCode = (int)HttpCode.USER_PAUSED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is paused\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
manualResetEvent.WaitOne(1000);
}
else
{
if (!bres)
{
bres = true;
manualResetEvent.Set();
result.RefCode = (int)HttpCode.USER_RESUMED;
result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is resumed\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
}
#region one-block
#region mkblk
if (leftBytes < BLOCK_SIZE)
{
blockSize = leftBytes;
}
else
{
blockSize = BLOCK_SIZE;
}
if (leftBytes < CHUNK_SIZE)
{
chunkSize = leftBytes;
}
else
{
chunkSize = CHUNK_SIZE;
}
//read data buffer
stream.Read(chunkBuffer, 0, (int)chunkSize);
iTry = 0;
while (++iTry <= putExtra.MaxRetryTimes)
{
result.RefText += string.Format("[{0}] [ResumableUpload] try mkblk#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);
hr = MakeBlock(chunkBuffer, blockSize, chunkSize, upToken);
if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
{
break;
}
}
if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkblk: code = {1}, text = {2}, offset = {3}, blockSize = {4}, chunkSize = {5}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockSize, chunkSize);
return(result);
}
if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
{
result.Shadow(hr);
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += string.Format("[{0}] [ResumableUpload] mkblk Error: JSON Decode Error: text = {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);
return(result);
}
context = rc.Ctx;
offset += chunkSize;
leftBytes -= chunkSize;
#endregion mkblk
putExtra.ProgressHandler(offset, fileSize);
if (leftBytes > 0)
{
blockLeft = blockSize - chunkSize;
blockOffset = chunkSize;
while (blockLeft > 0)
{
#region bput-loop
if (blockLeft < CHUNK_SIZE)
{
chunkSize = blockLeft;
}
else
{
chunkSize = CHUNK_SIZE;
}
stream.Read(chunkBuffer, 0, (int)chunkSize);
iTry = 0;
while (++iTry <= putExtra.MaxRetryTimes)
{
result.RefText += string.Format("[{0}] [ResumableUpload] try bput#{1}\n", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), iTry);
hr = BputChunk(chunkBuffer, blockOffset, chunkSize, context, upToken);
if (hr.Code == (int)HttpCode.OK && hr.RefCode != (int)HttpCode.USER_NEED_RETRY)
{
break;
}
}
if (hr.Code != (int)HttpCode.OK || hr.RefCode == (int)HttpCode.USER_NEED_RETRY)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: bput: code = {1}, text = {2}, offset = {3}, blockOffset = {4}, chunkSize = {5}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text, offset, blockOffset, chunkSize);
return(result);
}
if ((rc = JsonConvert.DeserializeObject <ResumeContext>(hr.Text)) == null)
{
result.Shadow(hr);
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += string.Format("[{0}] [ResumableUpload] bput Error: JSON Decode Error: text = {1}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Text);
return(result);
}
context = rc.Ctx;
if (expiredAt == 0)
{
expiredAt = rc.Expired_At;
}
offset += chunkSize;
leftBytes -= chunkSize;
blockOffset += chunkSize;
blockLeft -= chunkSize;
#endregion bput-loop
putExtra.ProgressHandler(offset, fileSize);
}
}
#endregion one-block
resumeInfo.BlockIndex = index;
resumeInfo.Contexts[index] = context;
resumeInfo.ExpiredAt = expiredAt;
if (!string.IsNullOrEmpty(putExtra.ResumeRecordFile))
{
ResumeHelper.Save(resumeInfo, putExtra.ResumeRecordFile);
}
++index;
}
}
hr = MakeFile(key, fileSize, key, upToken, putExtra, resumeInfo.Contexts);
if (hr.Code != (int)HttpCode.OK)
{
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Error: mkfile: code = {1}, text = {2}\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), hr.Code, hr.Text);
return(result);
}
if (File.Exists(putExtra.ResumeRecordFile))
{
File.Delete(putExtra.ResumeRecordFile);
}
result.Shadow(hr);
result.RefText += string.Format("[{0}] [ResumableUpload] Uploaded: \"{1}\" ==> \"{2}\"\n",
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), putExtra.ResumeRecordFile, key);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] [ResumableUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
finally
{
if (stream != null)
{
stream.Close();
stream.Dispose();
}
}
return(result);
}
/// <summary>
/// 根据已上传的所有分片数据创建文件
/// </summary>
/// <param name="fileName">源文件名</param>
/// <param name="resumeInfo">分片上传记录信息</param>
/// <param name="key">要保存的文件名</param>
/// <param name="upToken">上传凭证</param>
/// <param name="putExtra">用户指定的额外参数</param>
/// <param name="encodedObjectName">Base64编码后的资源名</param>
/// <returns>此操作执行后的返回结果</returns>
private HttpResult completeParts(string fileName, ResumeInfo resumeInfo, string key, string upToken, PutExtra putExtra, string encodedObjectName)
{
HttpResult result = new HttpResult();
try
{
string paramStr = "{}";
if (string.IsNullOrEmpty(fileName))
{
fileName = "fname";
}
if (string.IsNullOrEmpty(putExtra.MimeType))
{
putExtra.MimeType = "";
}
if (string.IsNullOrEmpty(key))
{
key = "";
}
if (putExtra.Params != null)
{
paramStr = JsonConvert.SerializeObject(putExtra.Params);
}
//get upload host
string ak = UpToken.GetAccessKeyFromUpToken(upToken);
string bucket = UpToken.GetBucketFromUpToken(upToken);
if (ak == null || bucket == null)
{
return(HttpResult.InvalidToken);
}
string uploadHost = this.config.UpHost(ak, bucket);
string upTokenStr = string.Format("UpToken {0}", upToken);
Dictionary <string, object> body = new Dictionary <string, object>();
body.Add("fname", fileName);
body.Add("mimeType", putExtra.MimeType);
body.Add("customVars", null);
body.Add("parts", resumeInfo.Etags);
string url = string.Format("{0}/buckets/{1}/objects/{2}/uploads/{3}", uploadHost, bucket, encodedObjectName, resumeInfo.UploadId);
string bodyStr = JsonConvert.SerializeObject(body);
result = httpManager.PostJson(url, bodyStr, upTokenStr);
}
catch (Exception ex)
{
StringBuilder sb = new StringBuilder();
sb.AppendFormat("[{0}] completeParts Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
Exception e = ex;
while (e != null)
{
sb.Append(e.Message + " ");
e = e.InnerException;
}
sb.AppendLine();
if (ex is QiniuException)
{
QiniuException qex = (QiniuException)ex;
result.Code = qex.HttpResult.Code;
result.RefCode = qex.HttpResult.Code;
result.Text = qex.HttpResult.Text;
result.RefText += sb.ToString();
}
else
{
result.RefCode = (int)HttpCode.USER_UNDEF;
result.RefText += sb.ToString();
}
}
return(result);
}
