private void uploadFiles(List <FileItem> filesToUpload)
        {
            this.uploadedBytes.Clear();
            ManualResetEvent[] doneEvents = null;
            int taskMax = filesToUpload.Count;

            doneEvents       = new ManualResetEvent[taskMax];
            this.uploadInfos = new UploadInfo[taskMax];
            for (int taskId = 0; taskId < taskMax; taskId++)
            {
                this.uploadInfos[taskId] = new UploadInfo();
                doneEvents[taskId]       = new ManualResetEvent(false);
                FileUploader uploader = new FileUploader(this.syncSetting, doneEvents[taskId], this, taskId, upctl);
                ThreadPool.QueueUserWorkItem(new WaitCallback(uploader.uploadFile), filesToUpload[taskId]);
            }

            try
            {
                WaitHandle.WaitAll(doneEvents);
                CachedHash.BatchInsertOrUpdate(filesToUpload, localHashDB);
                SyncLog.BatchInsertOrUpdate(filesToUpload, syncLogDB);
            }
            catch (Exception ex)
            {
                Log.Error("wait for job to complete error, " + ex.Message);
            }
        }
Example #2
0
        private void createOptionalDB()
        {
            //check jobs db
            if (!File.Exists(this.jobsDbPath))
            {
                try
                {
                    SyncRecord.CreateSyncRecordDB(this.jobsDbPath);
                }
                catch (Exception ex)
                {
                    Log.Error(string.Format("create sync record db for job {0} failed due to {1}", this.jobId, ex.Message));
                }
            }
            else
            {
                DateTime syncDateTime = DateTime.Now;
                try
                {
                    SyncRecord.InsertRecord(this.jobId, syncDateTime, this.syncSetting, this.jobsDbPath);
                }
                catch (Exception ex)
                {
                    Log.Error(string.Format("record sync job failed for job {0} due to {1}", this.jobId, ex.Message));
                }
            }

            if (!File.Exists(this.localHashDBPath))
            {
                try
                {
                    CachedHash.CreateCachedHashDB(this.localHashDBPath);
                }
                catch (Exception ex)
                {
                    Log.Error("create cached hash db failed, " + ex.Message);
                }
            }

            if (!File.Exists(this.syncLogDBPath))
            {
                try
                {
                    SyncLog.CreateSyncLogDB(this.syncLogDBPath);
                }
                catch (Exception ex)
                {
                    Log.Error(string.Format("create sync log db for job {0} failed due to {1}", this.jobId, ex.Message));
                }
            }
        }
Example #3
0
        public Hasher()
        {
            this.contextResolutionCache = new CachedHash<Tuple<IIncludePathContext, string, ModPart>, BuildObject>(
                delegate(Tuple<IIncludePathContext, string, ModPart> key)
                {
                    return key.Item1.search(key.Item2, key.Item3);
                });

            this.parsedIncludesCache = new CachedHash<Tuple<IIncludePathContext, BuildObject>, List<BeatIncludes.LabeledInclude>>(
                delegate(Tuple<IIncludePathContext, BuildObject> key)
                {
                    return BeatIncludes.parseLabeledIncludes(key.Item1, key.Item2);
                });
        }
Example #4
0
        private void createDirCache(string localSyncDir)
        {
            DirectoryInfo di = new DirectoryInfo(localSyncDir);

            FileInfo[] fileInfos = di.GetFiles("*.*", SearchOption.AllDirectories);

            int originalCount = fileInfos.Length;

            List <string> files = new List <string>();

            string[] keys = new string[originalCount];
            bool[]   skip = new bool[originalCount];
            overwriteDict = new Dictionary <string, bool>();
            for (int i = 0; i < originalCount; ++i)
            {
                files.Add(fileInfos[i].FullName);

                if (syncSetting.UseShortFilename)
                {
                    keys[i] = syncSetting.SyncPrefix + fileInfos[i].Name;
                }
                else
                {
                    keys[i] = syncSetting.SyncPrefix + fileInfos[i].FullName;
                }

                skip[i] = false;
                overwriteDict.Add(fileInfos[i].FullName, false);
            }

            // 按照前缀/后缀跳过
            string[] skippedPrefixes = this.syncSetting.SkipPrefixes.Split(',');
            string[] skippedSuffixes = this.syncSetting.SkipSuffixes.Split(',');
            for (int i = 0; i < originalCount; ++i)
            {
                string shortFileName = fileInfos[i].Name;

                if (skip[i])
                {
                    continue;
                }
                if (shouldSkipByPrefix(shortFileName, skippedPrefixes))
                {
                    skip[i] = true;
                    addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.TargetBucket, files[i]));
                    updateUploadLog("按照前缀规则跳过文件 " + files[i]);
                }

                if (skip[i])
                {
                    continue;
                }
                if (shouldSkipBySuffix(shortFileName, skippedSuffixes))
                {
                    skip[i] = true;
                    addFileSkippedLog(string.Format("{0}\t{1}", this.syncSetting.TargetBucket, files[i]));
                    updateUploadLog("按照后缀规则跳过文件 " + files[i]);
                }
            }

            // 检查本地增量文件
            if (syncSetting.CheckNewFiles)
            {
                List <string> cachedKeys = CachedHash.GetAllKeys(localHashDB);

                Dictionary <string, string> itemDict = CachedHash.GetAllItems(localHashDB);

                for (int i = 0; i < originalCount; ++i)
                {
                    if (skip[i])
                    {
                        continue;
                    }

                    string f = files[i];
                    if (itemDict.ContainsKey(f))
                    {
                        string oldHash = itemDict[f];
                        string newHash = Qiniu.Util.QETag.hash(f);
                        if (string.Equals(oldHash, newHash))
                        {
                            addFileExistsLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
                            updateUploadLog("本地检查已同步,跳过文件 " + files[i]);
                            skip[i] = true;
                        }
                    }
                }
            }

            // 检查云端同名文件
            Qiniu.Util.Mac mac        = new Qiniu.Util.Mac(SystemConfig.ACCESS_KEY, SystemConfig.SECRET_KEY);
            string[]       remoteHash = BucketFileHash.BatchStat(mac, syncSetting.TargetBucket, keys);

            // 跳过
            if (!syncSetting.OverwriteDuplicate)
            {
                for (int i = 0; i < originalCount; ++i)
                {
                    if (skip[i])
                    {
                        continue;
                    }

                    if (!string.IsNullOrEmpty(remoteHash[i]))
                    {
                        string localHash = Qiniu.Util.QETag.hash(files[i]);
                        if (string.Equals(localHash, remoteHash[i]))
                        {
                            addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
                            updateUploadLog("空间已存在相同文件,跳过文件 " + files[i]);
                            skip[i] = true;
                        }
                    }
                }
            }
            else // 覆盖
            {
                for (int i = 0; i < originalCount; ++i)
                {
                    if (skip[i])
                    {
                        continue;
                    }

                    if (!string.IsNullOrEmpty(remoteHash[i]))
                    {
                        string localHash = Qiniu.Util.QETag.hash(files[i]);
                        if (string.Equals(localHash, remoteHash[i]))
                        {
                            addFileNotOverwriteLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, files[i], keys[i]));
                            updateUploadLog("空间已存在相同文件,跳过文件 " + files[i]);
                            overwriteDict[files[i]] = true;
                        }
                    }
                }
            }

            using (StreamWriter sw = new StreamWriter(cacheFilePathDone))
            {
                for (int i = 0; i < files.Count; ++i)
                {
                    if (!skip[i])
                    {
                        sw.WriteLine(files[i]);
                    }
                }
            }
        }
Example #5
0
        /// <summary>
        /// 1.根据设定参数进行必要的配置
        /// 2.检查过滤之后,生成待上传的文件列表并展示
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void ButtonCheckFilesToUpload_Click(object sender, RoutedEventArgs e)
        {
            #region CHECK_SYNC_SETTINS

            // 检查并设置AK&SK
            if (string.IsNullOrEmpty(this.account.AccessKey) || string.IsNullOrEmpty(this.account.SecretKey))
            {
                this.SettingsErrorTextBlock.Text = "请设置AK&SK";
                return;
            }

            SystemConfig.ACCESS_KEY = this.account.AccessKey;
            SystemConfig.SECRET_KEY = this.account.SecretKey;

            // 检查本地同步目录与远程同步空间
            string syncDirectory = this.SyncLocalFolderTextBox.Text.Trim();
            if (string.IsNullOrEmpty(syncDirectory) || !Directory.Exists(syncDirectory))
            {
                this.SettingsErrorTextBlock.Text = "请设置本地待同步的目录";
                return;
            }
            if (this.SyncTargetBucketsComboBox.SelectedIndex < 0)
            {
                this.SettingsErrorTextBlock.Text = "请设置目标空间";
                return;
            }

            string targetBucket = this.SyncTargetBucketsComboBox.SelectedItem.ToString();

            if (this.syncSetting == null)
            {
                this.syncSetting = new SyncSetting();
            }

            this.syncSetting.LocalDirectory       = syncDirectory;
            this.syncSetting.TargetBucket         = targetBucket;
            this.syncSetting.SyncPrefix           = this.PrefixTextBox.Text.Trim();
            this.syncSetting.SkipPrefixes         = this.SkipPrefixesTextBox.Text.Trim();
            this.syncSetting.SkipSuffixes         = this.SkipSuffixesTextBox.Text.Trim();
            this.syncSetting.CheckNewFiles        = this.CheckNewFilesCheckBox.IsChecked.Value;
            this.syncSetting.UseShortFilename     = this.CheckBoxUseShortFilename.IsChecked.Value;
            this.syncSetting.OverwriteDuplicate   = this.RadioButtonOverwriteDuplicate.IsChecked.Value;
            this.syncSetting.SyncThreadCount      = (int)this.ThreadCountSlider.Value;
            this.syncSetting.ChunkUploadThreshold = (int)this.ChunkUploadThresholdSlider.Value * 1024 * 1024;
            this.syncSetting.DefaultChunkSize     = this.defaultChunkSize;
            this.syncSetting.UploadFromCDN        = this.RadioButtonFromCDN.IsChecked.Value;

            #endregion CHECK_SYNC_SETTINS

            #region SIMULATION

            StatResult statResult = this.bucketManager.stat(this.syncSetting.TargetBucket, "NONE_EXIST_KEY");

            if (statResult.ResponseInfo.isNetworkBroken())
            {
                this.SettingsErrorTextBlock.Text = "网络故障";
                return;
            }

            if (statResult.ResponseInfo.StatusCode == 401)
            {
                //ak & sk not right
                this.SettingsErrorTextBlock.Text = "AK 或 SK 不正确";
                return;
            }
            else if (statResult.ResponseInfo.StatusCode == 631)
            {
                //bucket not exist
                this.SettingsErrorTextBlock.Text = "指定空间不存在";
                return;
            }
            else if (statResult.ResponseInfo.StatusCode == 612 ||
                     statResult.ResponseInfo.StatusCode == 200)
            {
                //file exists or not
                //ignore
            }
            else if (statResult.ResponseInfo.StatusCode == 400)
            {
                if (string.IsNullOrEmpty(statResult.ResponseInfo.Error))
                {
                    this.SettingsErrorTextBlock.Text = "未知错误(状态代码400)";
                }
                else
                {
                    if (statResult.ResponseInfo.Error.Equals("incorrect zone"))
                    {
                        this.SettingsErrorTextBlock.Text = "上传入口机房设置错误";
                    }
                    else
                    {
                        this.SettingsErrorTextBlock.Text = statResult.ResponseInfo.Error;
                    }
                }
                return;
            }
            else
            {
                this.SettingsErrorTextBlock.Text = "未知错误,请联系七牛";
                Log.Error(string.Format("get buckets unknown error, {0}:{1}:{2}:{3}", statResult.ResponseInfo.StatusCode,
                                        statResult.ResponseInfo.Error, statResult.ResponseInfo.ReqId, statResult.Response));
                return;
            }

            #endregion SIMULATION

            int numFiles  = 0; // 总文件数
            int numUpload = 0; // 待上传文件数
            uploadItems.Clear();
            this.FilesToUploadDataGrid.DataContext = null;

            List <string> localFiles   = new List <string>(); // 本地待上传的文件
            List <string> saveKeys     = new List <string>(); // 保存到空间文件名
            List <string> fileEtags    = new List <string>(); // 待上传文件的ETAG
            List <long>   lastModified = new List <long>();   // 文件最后修改时间
            List <bool>   fileSkip     = new List <bool>();   // 是否跳过该文件(不上传)

            long T0 = (TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1, 0, 0, 0, 0))).Ticks;

            #region TRAVERSE_LOCAL_DIRECTORY

            DirectoryInfo di  = new DirectoryInfo(syncDirectory);
            FileInfo[]    ffi = di.GetFiles("*.*", SearchOption.AllDirectories);
            numFiles = ffi.Length;

            string savePrefix = this.PrefixTextBox.Text.Trim();

            if (this.syncSetting.UseShortFilename)
            {
                foreach (var fi in ffi)
                {
                    localFiles.Add(fi.FullName);
                    saveKeys.Add(savePrefix + fi.Name);
                    fileEtags.Add("_");
                    lastModified.Add((fi.LastWriteTime.Ticks - T0) / 10000);
                    fileSkip.Add(false);
                }
            }
            else
            {
                foreach (var fi in ffi)
                {
                    localFiles.Add(fi.FullName);
                    saveKeys.Add(savePrefix + fi.FullName);
                    fileEtags.Add("_");
                    lastModified.Add((fi.LastWriteTime.Ticks - T0) / 10000);
                    fileSkip.Add(false);
                }
            }

            #endregion TRAVERSE_LOCAL_DIRECTORY

            #region CHECK_PREFIX_SUFFX

            string skipPrefixes = this.SkipPrefixesTextBox.Text.Trim();
            string skipSuffixes = this.SkipSuffixesTextBox.Text.Trim();

            for (int i = 0; i < numFiles; ++i)
            {
                string saveKey = saveKeys[i];

                string[] ssPrfx = skipPrefixes.Split(',');
                foreach (string prefix in ssPrfx)
                {
                    if (!string.IsNullOrWhiteSpace(prefix))
                    {
                        if (saveKey.StartsWith(prefix.Trim()))
                        {
                            fileSkip[i] = true;
                            break;
                        }
                    }
                }

                string[] ssSufx = skipSuffixes.Split(',');
                foreach (string suffix in ssSufx)
                {
                    if (!string.IsNullOrWhiteSpace(suffix))
                    {
                        if (saveKey.EndsWith(suffix.Trim()))
                        {
                            fileSkip[i] = true;
                            break;
                        }
                    }
                }
            }
            #endregion CHECK_PREFIX_SUFFX

            string hashDBFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "qsunsync", "local_hash.db");
            if (!File.Exists(hashDBFile))
            {
                CachedHash.CreateCachedHashDB(hashDBFile);
            }
            SQLiteConnection localHashDBConn = new SQLiteConnection(string.Format("data source = {0}", hashDBFile));
            localHashDBConn.Open();
            Dictionary <string, HashDBItem> localHashDict = CachedHash.GetAllItems(localHashDBConn);
            localHashDBConn.Close();

            #region CHECK_LOCAL_DUPLICATE

            if (this.syncSetting.CheckNewFiles)
            {
                for (int i = 0; i < numFiles; ++i)
                {
                    if (fileSkip[i])
                    {
                        continue;
                    }

                    string fileName = localFiles[i];

                    if (localHashDict.ContainsKey(fileName))
                    {
                        string oldEtag = localHashDict[fileName].FileHash;
                        string newEtag = QETag.hash(fileName);

                        if (string.Equals(oldEtag, newEtag))
                        {
                            fileSkip[i] = true;
                        }
                        else
                        {
                            fileEtags[i] = newEtag;
                        }
                    }
                    else
                    {
                        fileEtags[i] = QETag.hash(fileName);
                    }
                }
            }
            else
            {
                for (int i = 0; i < numFiles; ++i)
                {
                    if (fileSkip[i])
                    {
                        continue;
                    }
                    fileEtags[i] = QETag.hash(localFiles[i]);
                }
            }

            #endregion CHECK_LOCAL_DUPLICATE

            #region CHECK_REMOTE_DUPLICATES

            // 如果选择了“强制覆盖”,那么无需进行远程检查
            if (!this.syncSetting.OverwriteDuplicate)
            {
                List <string> remoteHash   = new List <string>();
                List <long>   remoteUpdate = new List <long>();

                try
                {
                    Mac mac = new Mac(this.account.AccessKey, this.account.SecretKey);
                    BucketFileHash.BatchStat(mac, targetBucket, saveKeys, fileSkip, ref remoteHash, ref remoteUpdate);

                    for (int i = 0, k = 0; i < numFiles; ++i)
                    {
                        if (fileSkip[i])
                        {
                            continue;
                        }

                        if (string.Equals(fileEtags[i], remoteHash[k]))
                        {
                            // 云端已存在相同文件,跳过
                            fileSkip[i] = true;
                        }

                        ++k;
                    }
                }
                catch (Exception ex)
                {
                    this.SettingsErrorTextBlock.Text = ex.Message;
                    Log.Error(ex.Message);
                }
            }

            #endregion CHECK_REMOTE_DUPLICATES

            #region SHOW_UPLOAD_DETAILS

            this.SyncSettingTabControl.SelectedItem = this.TabItemFilesToUploadDetail;

            numUpload = numFiles;

            foreach (var b in fileSkip)
            {
                if (b)
                {
                    --numUpload;
                }
            }

            if (numUpload < 1)
            {
                TextBlockFilesToUploadSummery.Text = "没有待上传的文件";
                return;
            }


            double N = 0;

            for (int i = 0; i < numFiles; ++i)
            {
                if (fileSkip[i])
                {
                    continue;
                }

                string fsize = "0";

                long   n = ffi[i].Length;
                double K = 1.0 * n / 1024.0;
                double M = 0.0;
                if (K > 1024.0)
                {
                    M     = K / 1024.0;
                    fsize = string.Format("{0:0.00}MB", M);
                }
                else if (K > 1.0)
                {
                    fsize = string.Format("{0:0.00}KB", K);
                }
                else
                {
                    fsize = string.Format("{0}B", n);
                }

                N += n;

                uploadItems.Add(new UploadItem()
                {
                    LocalFile  = localFiles[i],
                    SaveKey    = saveKeys[i],
                    FileSize   = fsize,
                    FileHash   = fileEtags[i],
                    LastUpdate = lastModified[i].ToString()
                });
            }

            string vol  = "";
            double mega = 1024.0 * 1024;
            double kilo = 1024.0;
            if (N > mega)
            {
                vol = string.Format("{0:0.00}MB", N / mega);
            }
            else if (N > kilo)
            {
                vol = string.Format("{0:0.00}KB", N / kilo);
            }
            else
            {
                vol = string.Format("{0}B", N);
            }

            TextBlockFilesToUploadSummery.Text = string.Format("待上传的文件总数:{0}, 总大小:{1}", numUpload, vol);

            Dispatcher.Invoke(new Action(delegate
            {
                ObservableCollection <UploadItem> dataSource = new ObservableCollection <UploadItem>();
                foreach (var d in uploadItems)
                {
                    dataSource.Add(d);
                }
                this.FilesToUploadDataGrid.DataContext = dataSource;
            }));

            ButtonStartSync.IsEnabled = true;

            #endregion SHOW_UPLOAD_DETAILS
        }
Example #6
0
        /// <summary>
        /// 更新 2016-09-22 16:40 fengyh
        /// </summary>
        private void processUpload()
        {
            try
            {
                while (currentIndex < uploadItems.Count)
                {
                    if (this.cancelSignal)
                    {
                        return;
                    }

                    int itemsLeft = uploadItems.Count - currentIndex;

                    if (itemsLeft < this.syncSetting.SyncThreadCount)
                    {
                        this.uploadFiles(itemsLeft);
                        currentIndex += itemsLeft;
                    }
                    else
                    {
                        this.uploadFiles(this.syncSetting.SyncThreadCount);
                        currentIndex += this.syncSetting.SyncThreadCount;
                    }
                }


                #region UPDATE_HASH_DB

                if (dbItems.Count > 0)
                {
                    SQLiteConnection sqlConn = null;
                    try
                    {
                        if (!File.Exists(this.localHashDBPath))
                        {
                            CachedHash.CreateCachedHashDB(this.localHashDBPath);
                        }

                        var qsb = new SQLiteConnectionStringBuilder {
                            DataSource = this.localHashDBPath
                        };
                        sqlConn = new SQLiteConnection(qsb.ToString());
                        sqlConn.Open();
                        CachedHash.BatchInsertOrUpdate(dbItems, sqlConn);
                    }
                    catch (Exception ex)
                    {
                        Log.Error(ex.Message);
                    }
                    finally
                    {
                        if (sqlConn != null)
                        {
                            sqlConn.Close();
                        }
                    }
                }

                #endregion UPDATE_HASH_DB
            }
            catch (Exception ex)
            {
                Log.Fatal(ex.Message);
            }
        }
Example #7
0
 public FetchModuleCache(IIncludePathContext context)
 {
     this.beatIncludes = new BeatIncludes(context);
     this.cache = new CachedHash<string, BuildObject>(this.fetchModule);
 }