/// <summary>
/// 得到ChunkSize
/// </summary>
/// <param name="qiNiuConfig">七牛配置</param>
/// <param name="chunkUnit">请求Unit</param>
/// <returns></returns>
internal static ChunkUnit GetChunkUnit(QiNiuStorageConfig qiNiuConfig, ChunkUnit chunkUnit)
{
if (chunkUnit != null)
{
return(chunkUnit);
}
return(qiNiuConfig.ChunkUnit);
}
/// <summary>
/// 得到分片大小
/// </summary>
/// <param name="aliyunConfig">阿里云配置</param>
/// <param name="chunkUnit">分片大小</param>
/// <param name="defaultChunkUnit">默认分片大小</param>
/// <returns></returns>
internal static ChunkUnit GetChunkUnit(
ALiYunStorageConfig aliyunConfig,
ChunkUnit chunkUnit,
Func <ChunkUnit> defaultChunkUnit = null)
{
if (chunkUnit == null && aliyunConfig.ChunkUnit == null)
{
if (defaultChunkUnit == null)
{
throw new BusinessException <string>("未设置分片大小", HttpStatus.Err.Name);
}
return(defaultChunkUnit.Invoke());
}
if (chunkUnit != null)
{
return(chunkUnit);
}
return(aliyunConfig.ChunkUnit);
}
/// <summary>
/// 加载七牛云存储
/// </summary>
/// <param name="services"></param>
/// <param name="configuration"></param>
public static IServiceCollection AddQiNiuStorage(this IServiceCollection services,
IConfiguration configuration)
{
StartUp.Run();
var section = configuration.GetSection(nameof(QiNiuStorageConfig));
if (section == null)
{
throw new BusinessException <string>("七牛云存储配置异常", HttpStatus.Err.Name);
}
QiNiuStorageConfig qiNiuStorageConfig = new QiNiuStorageConfig(section.GetValue <string>("AccessKey"),
section.GetValue <string>("SecretKey"), section.GetValue <ZoneEnum>("DefaultZones"),
section.GetValue <string>("DefaultHost"), section.GetValue <string>("DefaultBucket"))
{
IsUseHttps = section.GetValue <bool>("IsUseHttps"),
UseCdnDomains = section.GetValue <bool>("UseCdnDomains"),
IsAllowOverlap = section.GetValue <bool>("IsAllowOverlap"),
PersistentNotifyUrl = section.GetValue <string>("PersistentNotifyUrl"),
PersistentPipeline = section.GetValue <string>("PersistentPipeline"),
ChunkUnit =
ChunkUnit
.FromValue <ChunkUnit>((section
.GetValue <string>("ChunkUnit").ConvertToInt(ChunkUnit.U2048K.Id)))
};
qiNiuStorageConfig.SetCallBack(section.GetValue <string>("CallbackBodyType").ConvertToInt(CallbackBodyType.Json.Id),
section.GetValue <string>("CallbackHost"), section.GetValue <string>("CallbackUrl"),
section.GetValue <string>("CallbackBody"));
configuration.GetSection(nameof(QiNiuStorageConfig)).Bind(qiNiuStorageConfig);
qiNiuStorageConfig.SetCallBackState(!string.IsNullOrEmpty(qiNiuStorageConfig.CallbackUrl) &&
!string.IsNullOrEmpty(qiNiuStorageConfig.CallbackHost));
return(AddQiNiuStorage(services,
() => qiNiuStorageConfig));
}
public void uploadFile(object obj)
{
FileItem item = obj as FileItem;
if (syncProgressPage.checkCancelSignal() && item.Length > syncSetting.ChunkUploadThreshold)
{
this.doneEvent.Set();
return;
}
string fileFullPath = item.LocalFile;
if (!File.Exists(fileFullPath))
{
Log.Error(string.Format("file not found error, {0}", fileFullPath));
this.doneEvent.Set();
return;
}
//set upload params
int putThreshold = this.syncSetting.ChunkUploadThreshold;
int chunkSize = this.syncSetting.DefaultChunkSize;
bool uploadFromCDN = this.syncSetting.UploadFromCDN;
string myDocPath = System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string recordPath = System.IO.Path.Combine(myDocPath, "qsunsync", "resume");
if (!Directory.Exists(recordPath))
{
Directory.CreateDirectory(recordPath);
}
Mac mac = new Mac(SystemConfig.ACCESS_KEY, SystemConfig.SECRET_KEY);
//current file info
FileInfo fileInfo = new FileInfo(fileFullPath);
long fileLength = fileInfo.Length;
string fileLastModified = fileInfo.LastWriteTimeUtc.ToFileTime().ToString();
//support resume upload
string recorderKey = string.Format("{0}:{1}:{2}:{3}:{4}", this.syncSetting.LocalDirectory,
this.syncSetting.TargetBucket, item.SaveKey, fileFullPath, fileLastModified);
recorderKey = Hashing.CalcMD5X(recorderKey);
this.syncProgressPage.updateUploadLog("准备上传文件 " + fileFullPath);
PutPolicy putPolicy = new PutPolicy();
if (this.syncSetting.OverwriteDuplicate)
{
putPolicy.Scope = this.syncSetting.TargetBucket + ":" + item.SaveKey;
}
else
{
putPolicy.Scope = this.syncSetting.TargetBucket;
}
putPolicy.SetExpires(24 * 30 * 3600);
string uptoken = Auth.CreateUploadToken(mac, putPolicy.ToJsonString());
this.syncProgressPage.updateUploadLog("开始上传文件 " + fileFullPath);
HttpResult result = null;
ChunkUnit cu = (ChunkUnit)(chunkSize / (128 * 1024));
if (item.Length > putThreshold)
{
ResumableUploader ru = new ResumableUploader(uploadFromCDN, cu);
string recordFile = System.IO.Path.Combine(recordPath, Hashing.CalcMD5X(fileFullPath));
UploadProgressHandler upph = new UploadProgressHandler(delegate(long uploaded, long total)
{
this.syncProgressPage.updateSingleFileProgress(taskId, fileFullPath, item.SaveKey, uploaded, fileLength);
});
result = ru.UploadFile(fileFullPath, item.SaveKey, uptoken, recordFile, upph, upController);
}
else
{
FormUploader su = new FormUploader(uploadFromCDN);
result = su.UploadFile(fileFullPath, item.SaveKey, uptoken);
}
if (result.Code == (int)HttpCode.OK)
{
item.Uploaded = true;
this.syncProgressPage.updateUploadLog("上传成功 " + fileFullPath);
this.syncProgressPage.addFileUploadSuccessLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket,
fileFullPath, item.SaveKey));
this.syncProgressPage.updateTotalUploadProgress();
}
else
{
item.Uploaded = false;
this.syncProgressPage.updateUploadLog("上传失败 " + fileFullPath + "，" + result.Text);
this.syncProgressPage.addFileUploadErrorLog(string.Format("{0}\t{1}\t{2}\t{3}", this.syncSetting.TargetBucket,
fileFullPath, item.SaveKey, result.Text));
}
this.doneEvent.Set();
}
/// <summary>
/// 得到分片大小
/// </summary>
/// <returns></returns>
internal static long GetPartSize(
ChunkUnit chunkUnit)
{
return(ChunkUnitList.Where(x => x.Key.Id == chunkUnit.Id).Select(x => x.Value).FirstOrDefault());
}
/// <summary>
/// 计算ChunkSize
/// </summary>
/// <param name="cu"></param>
/// <returns></returns>
public static int GetChunkSize(ChunkUnit cu)
{
var c = (int)cu;
return(c * N);
}
/// <summary>
/// 设置分片上传的“片”大小(单位:字节)，如过不设置则为默认的2MB
/// </summary>
/// <param name="chunkUnit">分片大小</param>
public void SetChunkUnit(ChunkUnit chunkUnit)
{
CHUNK_UNIT = chunkUnit;
}
public void uploadFile(object file)
{
if (syncProgressPage.checkCancelSignal())
{
this.doneEvent.Set();
return;
}
string fileFullPath = file.ToString();
if (!File.Exists(fileFullPath))
{
Log.Error(string.Format("file not found error, {0}", fileFullPath));
this.doneEvent.Set();
return;
}
//check skipped rules
int fileRelativePathIndex = fileFullPath.IndexOf(this.syncSetting.SyncLocalDir);
string fileRelativePath = fileFullPath.Substring(fileRelativePathIndex + this.syncSetting.SyncLocalDir.Length);
if (fileRelativePath.StartsWith("\\"))
{
fileRelativePath = fileRelativePath.Substring(1);
}
string[] skippedPrefixes = this.syncSetting.SkipPrefixes.Split(',');
foreach (string prefix in skippedPrefixes)
{
if (!string.IsNullOrWhiteSpace(prefix))
{
if (fileRelativePath.StartsWith(prefix.Trim()))
{
//skip by prefix
this.syncProgressPage.addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.SyncTargetBucket,
fileFullPath));
this.syncProgressPage.updateUploadLog("按照前缀规则跳过文件不同步 " + fileFullPath);
this.syncProgressPage.updateTotalUploadProgress();
this.doneEvent.Set();
return;
}
}
}
string[] skippedSuffixes = this.syncSetting.SkipSuffixes.Split(',');
foreach (string suffix in skippedSuffixes)
{
if (!string.IsNullOrWhiteSpace(suffix))
{
if (fileRelativePath.EndsWith(suffix.Trim()))
{
//skip by suffix
this.syncProgressPage.addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.SyncTargetBucket,
fileFullPath));
this.syncProgressPage.updateUploadLog("按照后缀规则跳过文件不同步 " + fileFullPath);
this.syncProgressPage.updateTotalUploadProgress();
this.doneEvent.Set();
return;
}
}
}
//generate the file key
string fileKey = "";
if (this.syncSetting.IgnoreDir)
{
//check ignore dir
fileKey = System.IO.Path.GetFileName(fileFullPath);
}
else
{
string newFileFullPath = fileFullPath.Replace('\\', '/');
string newLocalSyncDir = this.syncSetting.SyncLocalDir.Replace('\\', '/');
int fileKeyIndex = newFileFullPath.IndexOf(newLocalSyncDir);
fileKey = newFileFullPath.Substring(fileKeyIndex + newLocalSyncDir.Length);
if (fileKey.StartsWith("/"))
{
fileKey = fileKey.Substring(1);
}
}
//add prefix
fileKey = this.syncSetting.SyncPrefix + fileKey;
//set upload params
string myDocPath = System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string recordPath = System.IO.Path.Combine(myDocPath, "qsunsync", "resume");
if (!Directory.Exists(recordPath))
{
Directory.CreateDirectory(recordPath);
}
bool overwriteUpload = false;
Mac mac = new Mac(SystemConfig.ACCESS_KEY, SystemConfig.SECRET_KEY);
//current file info
System.IO.FileInfo fileInfo = new System.IO.FileInfo(fileFullPath);
long fileLength = fileInfo.Length;
string fileLastModified = fileInfo.LastWriteTimeUtc.ToFileTime().ToString();
//support resume upload
string recorderKey = string.Format("{0}:{1}:{2}:{3}:{4}", this.syncSetting.SyncLocalDir,
this.syncSetting.SyncTargetBucket, fileKey, fileFullPath, fileLastModified);
recorderKey = Tools.md5Hash(recorderKey);
if (syncSetting.CheckRemoteDuplicate)
{
//check remotely
BucketManager bucketManager = new BucketManager(mac, new Config());
StatResult statResult = bucketManager.Stat(this.syncSetting.SyncTargetBucket, fileKey);
if (statResult.Result != null && !string.IsNullOrEmpty(statResult.Result.Hash))
{
//file exists in bucket
string localHash = "";
//cached file info
try
{
CachedHash cachedHash = CachedHash.GetCachedHashByLocalPath(fileFullPath, localHashDB);
string cachedEtag = cachedHash.Etag;
string cachedLmd = cachedHash.LastModified;
if (!string.IsNullOrEmpty(cachedEtag) && !string.IsNullOrEmpty(cachedLmd))
{
if (cachedLmd.Equals(fileLastModified))
{
//file not modified
localHash = cachedEtag;
}
else
{
//file modified, calc the hash and update db
string newEtag = Qiniu.Util.ETag.CalcHash(fileFullPath);
localHash = newEtag;
try
{
CachedHash.UpdateCachedHash(fileFullPath, newEtag, fileLastModified, localHashDB);
}
catch (Exception ex)
{
Log.Error(string.Format("update local hash failed {0}", ex.Message));
}
}
}
else
{
//no record, calc hash and insert into db
string newEtag = Qiniu.Util.ETag.CalcHash(fileFullPath);
localHash = newEtag;
try
{
CachedHash.InsertCachedHash(fileFullPath, newEtag, fileLastModified, localHashDB);
}
catch (Exception ex)
{
Log.Error(string.Format("insert local hash failed {0}", ex.Message));
}
}
}
catch (Exception ex)
{
Log.Error(string.Format("get hash from local db failed {0}", ex.Message));
localHash = Qiniu.Util.ETag.CalcHash(fileFullPath);
}
if (localHash.Equals(statResult.Result.Hash))
{
//same file, no need to upload
this.syncProgressPage.addFileExistsLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
this.syncProgressPage.updateUploadLog("空间已存在，跳过文件 " + fileFullPath);
this.syncProgressPage.updateTotalUploadProgress();
//compatible, insert or update sync log for file
try
{
SyncLog.InsertOrUpdateSyncLog(fileKey, fileFullPath, fileLastModified, this.syncLogDB);
}
catch (Exception ex)
{
Log.Error(string.Format("insert ot update sync log error {0}", ex.Message));
}
this.doneEvent.Set();
return;
}
else
{
if (this.syncSetting.OverwriteFile)
{
overwriteUpload = true;
this.syncProgressPage.updateUploadLog("空间已存在，将覆盖 " + fileFullPath);
}
else
{
this.syncProgressPage.updateUploadLog("空间已存在，不覆盖 " + fileFullPath);
this.syncProgressPage.addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
this.syncProgressPage.updateTotalUploadProgress();
this.doneEvent.Set();
return;
}
}
}
}
else
{
//check locally
try
{
SyncLog syncLog = SyncLog.GetSyncLogByKey(fileKey, this.syncLogDB);
if (!string.IsNullOrEmpty(syncLog.Key))
{
//has sync log and check whether it changes
if (syncLog.LocalPath.Equals(fileFullPath) && syncLog.LastModified.Equals(fileLastModified))
{
this.syncProgressPage.addFileExistsLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
this.syncProgressPage.updateUploadLog("本地检查已同步，跳过" + fileFullPath);
this.syncProgressPage.updateTotalUploadProgress();
this.doneEvent.Set();
return;
}
}
}
catch (Exception ex)
{
Log.Error(string.Format("get sync log failed {0}", ex.Message));
this.doneEvent.Set();
return;
}
if (this.syncSetting.OverwriteFile)
{
overwriteUpload = true;
}
}
//if file not exists or need to overwrite
this.syncProgressPage.updateUploadLog("准备上传文件 " + fileFullPath);
bool uploadByCdn = true;
switch (this.syncSetting.UploadEntryDomain)
{
case 1:
uploadByCdn = false; break;
default:
uploadByCdn = true; break;
}
ChunkUnit chunkSize = ChunkUnit.U4096K;
switch (this.syncSetting.DefaultChunkSize)
{
case 0:
chunkSize = ChunkUnit.U128K; break;
case 1:
chunkSize = ChunkUnit.U256K; break;
case 2:
chunkSize = ChunkUnit.U512K; break;
case 3:
chunkSize = ChunkUnit.U1024K; break;
case 4:
chunkSize = ChunkUnit.U2048K; break;
case 5:
chunkSize = ChunkUnit.U4096K; break;
default:
chunkSize = ChunkUnit.U4096K; break;
}
string resumeFile = System.IO.Path.Combine(recordPath, recorderKey);
Config config = new Config();
config.ChunkSize = chunkSize;
config.UseCdnDomains = uploadByCdn;
config.PutThreshold = this.syncSetting.ChunkUploadThreshold;
UploadManager uploadManager = new UploadManager(config);
PutExtra putExtra = new PutExtra();
putExtra.ResumeRecordFile = resumeFile;
UploadProgressHandler progressHandler = new UploadProgressHandler(delegate(long uploadBytes, long totalBytes)
{
Console.WriteLine("progress: " + uploadBytes + ", " + totalBytes);
double percent = uploadBytes * 1.0 / totalBytes;
this.syncProgressPage.updateSingleFileProgress(taskId, fileFullPath, fileKey, fileLength, percent);
});
putExtra.ProgressHandler = progressHandler;
PutPolicy putPolicy = new PutPolicy();
if (overwriteUpload)
{
putPolicy.Scope = this.syncSetting.SyncTargetBucket + ":" + fileKey;
}
else
{
putPolicy.Scope = this.syncSetting.SyncTargetBucket;
}
putPolicy.SetExpires(24 * 30 * 3600);
//set file type
putPolicy.FileType = this.syncSetting.FileType;
Auth auth = new Auth(mac);
string uptoken = auth.CreateUploadToken(putPolicy.ToJsonString());
this.syncProgressPage.updateUploadLog("开始上传文件 " + fileFullPath);
HttpResult uploadResult = uploadManager.UploadFile(fileFullPath, fileKey, uptoken, putExtra);
if (uploadResult.Code != 200)
{
this.syncProgressPage.updateUploadLog("上传失败 " + fileFullPath + "，" + uploadResult.Text);
this.syncProgressPage.addFileUploadErrorLog(string.Format("{0}\t{1}\t{2}\t{3}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey, uploadResult.Text + "" + uploadResult.RefText));
//file exists error
if (uploadResult.Code == 614)
{
this.syncProgressPage.updateUploadLog("空间已存在，未覆盖 " + fileFullPath);
this.syncProgressPage.addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
}
}
else
{
//insert or update sync log for file
try
{
SyncLog.InsertOrUpdateSyncLog(fileKey, fileFullPath, fileLastModified, this.syncLogDB);
}
catch (Exception ex)
{
Log.Error(string.Format("insert ot update sync log error {0}", ex.Message));
}
//write new file hash to local db
if (!overwriteUpload)
{
PutRet putRet = JsonConvert.DeserializeObject <PutRet>(uploadResult.Text);
string fileHash = putRet.Hash;
if (this.localHashDB != null)
{
try
{
CachedHash.InsertOrUpdateCachedHash(fileFullPath, fileHash, fileLastModified, this.localHashDB);
}
catch (Exception ex)
{
Log.Error(string.Format("insert or update cached hash error {0}", ex.Message));
}
}
Log.Debug(string.Format("insert or update qiniu hash to local: '{0}' => '{1}'", fileFullPath, fileHash));
}
//update
if (overwriteUpload)
{
this.syncProgressPage.addFileOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
}
this.syncProgressPage.updateUploadLog("上传成功 " + fileFullPath);
this.syncProgressPage.addFileUploadSuccessLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.SyncTargetBucket,
fileFullPath, fileKey));
this.syncProgressPage.updateTotalUploadProgress();
}
this.doneEvent.Set();
return;
}
