private void uploadFiles(List <FileItem> filesToUpload)
{
this.uploadedBytes.Clear();
ManualResetEvent[] doneEvents = null;
int taskMax = filesToUpload.Count;
doneEvents = new ManualResetEvent[taskMax];
this.uploadInfos = new UploadInfo[taskMax];
for (int taskId = 0; taskId < taskMax; taskId++)
{
this.uploadInfos[taskId] = new UploadInfo();
doneEvents[taskId] = new ManualResetEvent(false);
FileUploader uploader = new FileUploader(this.syncSetting, doneEvents[taskId], this, taskId, upctl);
ThreadPool.QueueUserWorkItem(new WaitCallback(uploader.uploadFile), filesToUpload[taskId]);
}
try
{
WaitHandle.WaitAll(doneEvents);
CachedHash.BatchInsertOrUpdate(filesToUpload, localHashDB);
SyncLog.BatchInsertOrUpdate(filesToUpload, syncLogDB);
}
catch (Exception ex)
{
Log.Error("wait for job to complete error, " + ex.Message);
}
}
private void createOptionalDB()
{
//check jobs db
if (!File.Exists(this.jobsDbPath))
{
try
{
SyncRecord.CreateSyncRecordDB(this.jobsDbPath);
}
catch (Exception ex)
{
Log.Error(string.Format("create sync record db for job {0} failed due to {1}", this.jobId, ex.Message));
}
}
else
{
DateTime syncDateTime = DateTime.Now;
try
{
SyncRecord.InsertRecord(this.jobId, syncDateTime, this.syncSetting, this.jobsDbPath);
}
catch (Exception ex)
{
Log.Error(string.Format("record sync job failed for job {0} due to {1}", this.jobId, ex.Message));
}
}
if (!File.Exists(this.localHashDBPath))
{
try
{
CachedHash.CreateCachedHashDB(this.localHashDBPath);
}
catch (Exception ex)
{
Log.Error("create cached hash db failed, " + ex.Message);
}
}
if (!File.Exists(this.syncLogDBPath))
{
try
{
SyncLog.CreateSyncLogDB(this.syncLogDBPath);
}
catch (Exception ex)
{
Log.Error(string.Format("create sync log db for job {0} failed due to {1}", this.jobId, ex.Message));
}
}
}
public Hasher()
{
this.contextResolutionCache = new CachedHash<Tuple<IIncludePathContext, string, ModPart>, BuildObject>(
delegate(Tuple<IIncludePathContext, string, ModPart> key)
{
return key.Item1.search(key.Item2, key.Item3);
});
this.parsedIncludesCache = new CachedHash<Tuple<IIncludePathContext, BuildObject>, List<BeatIncludes.LabeledInclude>>(
delegate(Tuple<IIncludePathContext, BuildObject> key)
{
return BeatIncludes.parseLabeledIncludes(key.Item1, key.Item2);
});
}
private void createDirCache(string localSyncDir)
{
DirectoryInfo di = new DirectoryInfo(localSyncDir);
FileInfo[] fileInfos = di.GetFiles("*.*", SearchOption.AllDirectories);
int originalCount = fileInfos.Length;
List <string> files = new List <string>();
string[] keys = new string[originalCount];
bool[] skip = new bool[originalCount];
overwriteDict = new Dictionary <string, bool>();
for (int i = 0; i < originalCount; ++i)
{
files.Add(fileInfos[i].FullName);
if (syncSetting.UseShortFilename)
{
keys[i] = syncSetting.SyncPrefix + fileInfos[i].Name;
}
else
{
keys[i] = syncSetting.SyncPrefix + fileInfos[i].FullName;
}
skip[i] = false;
overwriteDict.Add(fileInfos[i].FullName, false);
}
// 按照前缀/后缀跳过
string[] skippedPrefixes = this.syncSetting.SkipPrefixes.Split(',');
string[] skippedSuffixes = this.syncSetting.SkipSuffixes.Split(',');
for (int i = 0; i < originalCount; ++i)
{
string shortFileName = fileInfos[i].Name;
if (skip[i])
{
continue;
}
if (shouldSkipByPrefix(shortFileName, skippedPrefixes))
{
skip[i] = true;
addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.TargetBucket, files[i]));
updateUploadLog("按照前缀规则跳过文件 " + files[i]);
}
if (skip[i])
{
continue;
}
if (shouldSkipBySuffix(shortFileName, skippedSuffixes))
{
skip[i] = true;
addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.TargetBucket, files[i]));
updateUploadLog("按照后缀规则跳过文件 " + files[i]);
}
}
// 检查本地增量文件
if (syncSetting.CheckNewFiles)
{
List <string> cachedKeys = CachedHash.GetAllKeys(localHashDB);
Dictionary <string, string> itemDict = CachedHash.GetAllItems(localHashDB);
for (int i = 0; i < originalCount; ++i)
{
if (skip[i])
{
continue;
}
string f = files[i];
if (itemDict.ContainsKey(f))
{
string oldHash = itemDict[f];
string newHash = Qiniu.Util.QETag.hash(f);
if (string.Equals(oldHash, newHash))
{
addFileExistsLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
updateUploadLog("本地检查已同步，跳过文件 " + files[i]);
skip[i] = true;
}
}
}
}
// 检查云端同名文件
Qiniu.Util.Mac mac = new Qiniu.Util.Mac(SystemConfig.ACCESS_KEY, SystemConfig.SECRET_KEY);
string[] remoteHash = BucketFileHash.BatchStat(mac, syncSetting.TargetBucket, keys);
// 跳过
if (!syncSetting.OverwriteDuplicate)
{
for (int i = 0; i < originalCount; ++i)
{
if (skip[i])
{
continue;
}
if (!string.IsNullOrEmpty(remoteHash[i]))
{
string localHash = Qiniu.Util.QETag.hash(files[i]);
if (string.Equals(localHash, remoteHash[i]))
{
addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
updateUploadLog("空间已存在相同文件，跳过文件 " + files[i]);
skip[i] = true;
}
}
}
}
else // 覆盖
{
for (int i = 0; i < originalCount; ++i)
{
if (skip[i])
{
continue;
}
if (!string.IsNullOrEmpty(remoteHash[i]))
{
string localHash = Qiniu.Util.QETag.hash(files[i]);
if (string.Equals(localHash, remoteHash[i]))
{
addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
updateUploadLog("空间已存在相同文件，跳过文件 " + files[i]);
overwriteDict[files[i]] = true;
}
}
}
}
using (StreamWriter sw = new StreamWriter(cacheFilePathDone))
{
for (int i = 0; i < files.Count; ++i)
{
if (!skip[i])
{
sw.WriteLine(files[i]);
}
}
}
}
/// <summary>
/// 1.根据设定参数进行必要的配置
/// 2.检查过滤之后，生成待上传的文件列表并展示
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void ButtonCheckFilesToUpload_Click(object sender, RoutedEventArgs e)
{
#region CHECK_SYNC_SETTINS
// 检查并设置AK&SK
if (string.IsNullOrEmpty(this.account.AccessKey) || string.IsNullOrEmpty(this.account.SecretKey))
{
this.SettingsErrorTextBlock.Text = "请设置AK&SK";
return;
}
SystemConfig.ACCESS_KEY = this.account.AccessKey;
SystemConfig.SECRET_KEY = this.account.SecretKey;
// 检查本地同步目录与远程同步空间
string syncDirectory = this.SyncLocalFolderTextBox.Text.Trim();
if (string.IsNullOrEmpty(syncDirectory) || !Directory.Exists(syncDirectory))
{
this.SettingsErrorTextBlock.Text = "请设置本地待同步的目录";
return;
}
if (this.SyncTargetBucketsComboBox.SelectedIndex < 0)
{
this.SettingsErrorTextBlock.Text = "请设置目标空间";
return;
}
string targetBucket = this.SyncTargetBucketsComboBox.SelectedItem.ToString();
if (this.syncSetting == null)
{
this.syncSetting = new SyncSetting();
}
this.syncSetting.LocalDirectory = syncDirectory;
this.syncSetting.TargetBucket = targetBucket;
this.syncSetting.SyncPrefix = this.PrefixTextBox.Text.Trim();
this.syncSetting.SkipPrefixes = this.SkipPrefixesTextBox.Text.Trim();
this.syncSetting.SkipSuffixes = this.SkipSuffixesTextBox.Text.Trim();
this.syncSetting.CheckNewFiles = this.CheckNewFilesCheckBox.IsChecked.Value;
this.syncSetting.UseShortFilename = this.CheckBoxUseShortFilename.IsChecked.Value;
this.syncSetting.OverwriteDuplicate = this.RadioButtonOverwriteDuplicate.IsChecked.Value;
this.syncSetting.SyncThreadCount = (int)this.ThreadCountSlider.Value;
this.syncSetting.ChunkUploadThreshold = (int)this.ChunkUploadThresholdSlider.Value * 1024 * 1024;
this.syncSetting.DefaultChunkSize = this.defaultChunkSize;
this.syncSetting.UploadFromCDN = this.RadioButtonFromCDN.IsChecked.Value;
#endregion CHECK_SYNC_SETTINS
#region SIMULATION
StatResult statResult = this.bucketManager.stat(this.syncSetting.TargetBucket, "NONE_EXIST_KEY");
if (statResult.ResponseInfo.isNetworkBroken())
{
this.SettingsErrorTextBlock.Text = "网络故障";
return;
}
if (statResult.ResponseInfo.StatusCode == 401)
{
//ak & sk not right
this.SettingsErrorTextBlock.Text = "AK 或 SK 不正确";
return;
}
else if (statResult.ResponseInfo.StatusCode == 631)
{
//bucket not exist
this.SettingsErrorTextBlock.Text = "指定空间不存在";
return;
}
else if (statResult.ResponseInfo.StatusCode == 612 ||
statResult.ResponseInfo.StatusCode == 200)
{
//file exists or not
//ignore
}
else if (statResult.ResponseInfo.StatusCode == 400)
{
if (string.IsNullOrEmpty(statResult.ResponseInfo.Error))
{
this.SettingsErrorTextBlock.Text = "未知错误(状态代码400)";
}
else
{
if (statResult.ResponseInfo.Error.Equals("incorrect zone"))
{
this.SettingsErrorTextBlock.Text = "上传入口机房设置错误";
}
else
{
this.SettingsErrorTextBlock.Text = statResult.ResponseInfo.Error;
}
}
return;
}
else
{
this.SettingsErrorTextBlock.Text = "未知错误，请联系七牛";
Log.Error(string.Format("get buckets unknown error, {0}:{1}:{2}:{3}", statResult.ResponseInfo.StatusCode,
statResult.ResponseInfo.Error, statResult.ResponseInfo.ReqId, statResult.Response));
return;
}
#endregion SIMULATION
int numFiles = 0; // 总文件数
int numUpload = 0; // 待上传文件数
uploadItems.Clear();
this.FilesToUploadDataGrid.DataContext = null;
List <string> localFiles = new List <string>(); // 本地待上传的文件
List <string> saveKeys = new List <string>(); // 保存到空间文件名
List <string> fileEtags = new List <string>(); // 待上传文件的ETAG
List <long> lastModified = new List <long>(); // 文件最后修改时间
List <bool> fileSkip = new List <bool>(); // 是否跳过该文件(不上传)
long T0 = (TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1, 0, 0, 0, 0))).Ticks;
#region TRAVERSE_LOCAL_DIRECTORY
DirectoryInfo di = new DirectoryInfo(syncDirectory);
FileInfo[] ffi = di.GetFiles("*.*", SearchOption.AllDirectories);
numFiles = ffi.Length;
string savePrefix = this.PrefixTextBox.Text.Trim();
if (this.syncSetting.UseShortFilename)
{
foreach (var fi in ffi)
{
localFiles.Add(fi.FullName);
saveKeys.Add(savePrefix + fi.Name);
fileEtags.Add("_");
lastModified.Add((fi.LastWriteTime.Ticks - T0) / 10000);
fileSkip.Add(false);
}
}
else
{
foreach (var fi in ffi)
{
localFiles.Add(fi.FullName);
saveKeys.Add(savePrefix + fi.FullName);
fileEtags.Add("_");
lastModified.Add((fi.LastWriteTime.Ticks - T0) / 10000);
fileSkip.Add(false);
}
}
#endregion TRAVERSE_LOCAL_DIRECTORY
#region CHECK_PREFIX_SUFFX
string skipPrefixes = this.SkipPrefixesTextBox.Text.Trim();
string skipSuffixes = this.SkipSuffixesTextBox.Text.Trim();
for (int i = 0; i < numFiles; ++i)
{
string saveKey = saveKeys[i];
string[] ssPrfx = skipPrefixes.Split(',');
foreach (string prefix in ssPrfx)
{
if (!string.IsNullOrWhiteSpace(prefix))
{
if (saveKey.StartsWith(prefix.Trim()))
{
fileSkip[i] = true;
break;
}
}
}
string[] ssSufx = skipSuffixes.Split(',');
foreach (string suffix in ssSufx)
{
if (!string.IsNullOrWhiteSpace(suffix))
{
if (saveKey.EndsWith(suffix.Trim()))
{
fileSkip[i] = true;
break;
}
}
}
}
#endregion CHECK_PREFIX_SUFFX
string hashDBFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "qsunsync", "local_hash.db");
if (!File.Exists(hashDBFile))
{
CachedHash.CreateCachedHashDB(hashDBFile);
}
SQLiteConnection localHashDBConn = new SQLiteConnection(string.Format("data source = {0}", hashDBFile));
localHashDBConn.Open();
Dictionary <string, HashDBItem> localHashDict = CachedHash.GetAllItems(localHashDBConn);
localHashDBConn.Close();
#region CHECK_LOCAL_DUPLICATE
if (this.syncSetting.CheckNewFiles)
{
for (int i = 0; i < numFiles; ++i)
{
if (fileSkip[i])
{
continue;
}
string fileName = localFiles[i];
if (localHashDict.ContainsKey(fileName))
{
string oldEtag = localHashDict[fileName].FileHash;
string newEtag = QETag.hash(fileName);
if (string.Equals(oldEtag, newEtag))
{
fileSkip[i] = true;
}
else
{
fileEtags[i] = newEtag;
}
}
else
{
fileEtags[i] = QETag.hash(fileName);
}
}
}
else
{
for (int i = 0; i < numFiles; ++i)
{
if (fileSkip[i])
{
continue;
}
fileEtags[i] = QETag.hash(localFiles[i]);
}
}
#endregion CHECK_LOCAL_DUPLICATE
#region CHECK_REMOTE_DUPLICATES
// 如果选择了“强制覆盖”，那么无需进行远程检查
if (!this.syncSetting.OverwriteDuplicate)
{
List <string> remoteHash = new List <string>();
List <long> remoteUpdate = new List <long>();
try
{
Mac mac = new Mac(this.account.AccessKey, this.account.SecretKey);
BucketFileHash.BatchStat(mac, targetBucket, saveKeys, fileSkip, ref remoteHash, ref remoteUpdate);
for (int i = 0, k = 0; i < numFiles; ++i)
{
if (fileSkip[i])
{
continue;
}
if (string.Equals(fileEtags[i], remoteHash[k]))
{
// 云端已存在相同文件，跳过
fileSkip[i] = true;
}
++k;
}
}
catch (Exception ex)
{
this.SettingsErrorTextBlock.Text = ex.Message;
Log.Error(ex.Message);
}
}
#endregion CHECK_REMOTE_DUPLICATES
#region SHOW_UPLOAD_DETAILS
this.SyncSettingTabControl.SelectedItem = this.TabItemFilesToUploadDetail;
numUpload = numFiles;
foreach (var b in fileSkip)
{
if (b)
{
--numUpload;
}
}
if (numUpload < 1)
{
TextBlockFilesToUploadSummery.Text = "没有待上传的文件";
return;
}
double N = 0;
for (int i = 0; i < numFiles; ++i)
{
if (fileSkip[i])
{
continue;
}
string fsize = "0";
long n = ffi[i].Length;
double K = 1.0 * n / 1024.0;
double M = 0.0;
if (K > 1024.0)
{
M = K / 1024.0;
fsize = string.Format("{0:0.00}MB", M);
}
else if (K > 1.0)
{
fsize = string.Format("{0:0.00}KB", K);
}
else
{
fsize = string.Format("{0}B", n);
}
N += n;
uploadItems.Add(new UploadItem()
{
LocalFile = localFiles[i],
SaveKey = saveKeys[i],
FileSize = fsize,
FileHash = fileEtags[i],
LastUpdate = lastModified[i].ToString()
});
}
string vol = "";
double mega = 1024.0 * 1024;
double kilo = 1024.0;
if (N > mega)
{
vol = string.Format("{0:0.00}MB", N / mega);
}
else if (N > kilo)
{
vol = string.Format("{0:0.00}KB", N / kilo);
}
else
{
vol = string.Format("{0}B", N);
}
TextBlockFilesToUploadSummery.Text = string.Format("待上传的文件总数:{0}, 总大小:{1}", numUpload, vol);
Dispatcher.Invoke(new Action(delegate
{
ObservableCollection <UploadItem> dataSource = new ObservableCollection <UploadItem>();
foreach (var d in uploadItems)
{
dataSource.Add(d);
}
this.FilesToUploadDataGrid.DataContext = dataSource;
}));
ButtonStartSync.IsEnabled = true;
#endregion SHOW_UPLOAD_DETAILS
}
/// <summary>
/// 更新 2016-09-22 16:40 fengyh
/// </summary>
private void processUpload()
{
try
{
while (currentIndex < uploadItems.Count)
{
if (this.cancelSignal)
{
return;
}
int itemsLeft = uploadItems.Count - currentIndex;
if (itemsLeft < this.syncSetting.SyncThreadCount)
{
this.uploadFiles(itemsLeft);
currentIndex += itemsLeft;
}
else
{
this.uploadFiles(this.syncSetting.SyncThreadCount);
currentIndex += this.syncSetting.SyncThreadCount;
}
}
#region UPDATE_HASH_DB
if (dbItems.Count > 0)
{
SQLiteConnection sqlConn = null;
try
{
if (!File.Exists(this.localHashDBPath))
{
CachedHash.CreateCachedHashDB(this.localHashDBPath);
}
var qsb = new SQLiteConnectionStringBuilder {
DataSource = this.localHashDBPath
};
sqlConn = new SQLiteConnection(qsb.ToString());
sqlConn.Open();
CachedHash.BatchInsertOrUpdate(dbItems, sqlConn);
}
catch (Exception ex)
{
Log.Error(ex.Message);
}
finally
{
if (sqlConn != null)
{
sqlConn.Close();
}
}
}
#endregion UPDATE_HASH_DB
}
catch (Exception ex)
{
Log.Fatal(ex.Message);
}
}
public FetchModuleCache(IIncludePathContext context)
{
this.beatIncludes = new BeatIncludes(context);
this.cache = new CachedHash<string, BuildObject>(this.fetchModule);
}