private void processUpload(string cacheFilePath) { try { string filePath = null; //count using (StreamReader sr = new StreamReader(this.cacheFilePathDone, Encoding.UTF8)) { while ((filePath = sr.ReadLine()) != null) { this.totalCount += 1; } } //upload using (StreamReader sr = new StreamReader(this.cacheFilePathDone, Encoding.UTF8)) { while ((filePath = sr.ReadLine()) != null) { if (this.cancelSignal) { return; } if (this.batchOpFiles.Count < this.syncSetting.SyncThreadCount) { FileInfo fi = new FileInfo(filePath); string fullName = fi.FullName.Replace('\\', '/'); string relativeName = fullName.Substring(syncSetting.LocalDirectory.Length + 1); string fn = syncSetting.SyncPrefix; switch (syncSetting.FilenameKind) { case 0: fn += fullName; break; case 1: fn += relativeName; break; case 2: default: fn += fi.Name; break; } FileItem item = new FileItem() { LocalFile = filePath, SaveKey = fn, FileHash = Qiniu.Util.QETag.hash(filePath), LastUpdate = fi.LastWriteTime.Ticks.ToString() }; this.batchOpFiles.Add(item); } else { this.uploadFiles(this.batchOpFiles); this.batchOpFiles.Clear(); FileInfo fi = new FileInfo(filePath); string fullName = fi.FullName.Replace('\\', '/'); string relativeName = fullName.Substring(syncSetting.LocalDirectory.Length + 1); string fn = syncSetting.SyncPrefix; switch (syncSetting.FilenameKind) { case 0: fn += fi.Name; break; case 1: fn += relativeName; break; case 2: default: fn += fullName; break; } FileItem item = new FileItem() { LocalFile = filePath, SaveKey = fn, FileHash = Qiniu.Util.QETag.hash(filePath), LastUpdate = fi.LastWriteTime.Ticks.ToString() }; this.batchOpFiles.Add(item); } } } } catch (Exception ex) { Log.Fatal(string.Format("open cache file {0} failed due to {1}", this.cacheFilePathDone, ex.Message)); } }
private void processUpload(string cacheFilePath) { try { string filePath = null; //count using (StreamReader sr = new StreamReader(this.cacheFilePathDone, Encoding.UTF8)) { while ((filePath = sr.ReadLine()) != null) { this.totalCount += 1; } } //upload using (StreamReader sr = new StreamReader(this.cacheFilePathDone, Encoding.UTF8)) { while ((filePath = sr.ReadLine()) != null) { if (this.cancelSignal) { return; } if (this.batchOpFiles.Count < this.syncSetting.SyncThreadCount) { FileInfo fi = new FileInfo(filePath); string fn = syncSetting.SyncPrefix; if(syncSetting.UseShortFilename) { fn += fi.Name; } else { fn += filePath; } FileItem item = new FileItem() { LocalFile = filePath, SaveKey = fn, FileHash = Qiniu.Util.QETag.hash(filePath), LastUpdate = fi.LastWriteTime.Ticks.ToString() }; this.batchOpFiles.Add(item); } else { this.uploadFiles(this.batchOpFiles); this.batchOpFiles.Clear(); FileInfo fi = new FileInfo(filePath); string fn = syncSetting.SyncPrefix; if (syncSetting.UseShortFilename) { fn += fi.Name; } else { fn += filePath; } FileItem item = new FileItem() { LocalFile = filePath, SaveKey = fn, FileHash = Qiniu.Util.QETag.hash(filePath), LastUpdate = fi.LastWriteTime.Ticks.ToString() }; this.batchOpFiles.Add(item); } } } } catch (Exception ex) { Log.Fatal(string.Format("open cache file {0} failed due to {1}", this.cacheFilePathDone, ex.Message)); } }
public void uploadFile(object obj) { FileItem item = obj as FileItem; if (syncProgressPage.checkCancelSignal()) { this.doneEvent.Set(); return; } string fileFullPath = item.LocalFile; if (!File.Exists(fileFullPath)) { Log.Error(string.Format("file not found error, {0}", fileFullPath)); this.doneEvent.Set(); return; } //set upload params int putThreshold = this.syncSetting.ChunkUploadThreshold; int chunkSize = this.syncSetting.DefaultChunkSize; bool uploadFromCDN = this.syncSetting.UploadFromCDN; string myDocPath = System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); string recordPath = System.IO.Path.Combine(myDocPath, "qsunsync", "resume"); if (!Directory.Exists(recordPath)) { Directory.CreateDirectory(recordPath); } Mac mac = new Mac(SystemConfig.ACCESS_KEY, SystemConfig.SECRET_KEY); //current file info FileInfo fileInfo = new FileInfo(fileFullPath); long fileLength = fileInfo.Length; string fileLastModified = fileInfo.LastWriteTimeUtc.ToFileTime().ToString(); //support resume upload string recorderKey = string.Format("{0}:{1}:{2}:{3}:{4}", this.syncSetting.LocalDirectory, this.syncSetting.TargetBucket, item.SaveKey, fileFullPath, fileLastModified); recorderKey = Hashing.CalcMD5X(recorderKey); this.syncProgressPage.updateUploadLog("准备上传文件 " + fileFullPath); PutPolicy putPolicy = new PutPolicy(); if (this.syncSetting.OverwriteDuplicate) { putPolicy.Scope = this.syncSetting.TargetBucket + ":" + item.SaveKey; } else { putPolicy.Scope = this.syncSetting.TargetBucket; } putPolicy.SetExpires(24 * 30 * 3600); string uptoken = Auth.CreateUploadToken(mac, putPolicy.ToJsonString()); this.syncProgressPage.updateUploadLog("开始上传文件 " + fileFullPath); HttpResult result = null; ChunkUnit cu = (ChunkUnit)(chunkSize / (128 * 1024)); if (item.Length > putThreshold) { ResumableUploader ru = new ResumableUploader(uploadFromCDN, cu); string recordFile = System.IO.Path.Combine(recordPath, Hashing.CalcMD5X(fileFullPath)); UploadProgressHandler upph = new UploadProgressHandler(delegate(long uploaded, long total) { this.syncProgressPage.updateSingleFileProgress(taskId, fileFullPath, item.SaveKey, uploaded, fileLength); }); result = ru.UploadFile(fileFullPath, item.SaveKey, uptoken, recordFile, upph, upController); } else { FormUploader su = new FormUploader(uploadFromCDN); result = su.UploadFile(fileFullPath, item.SaveKey, uptoken); } if (result.Code == (int)HttpCode.OK) { this.syncProgressPage.updateUploadLog("上传成功 " + fileFullPath); this.syncProgressPage.addFileUploadSuccessLog(string.Format("{0}\t{1}\t{2}", this.syncSetting.TargetBucket, fileFullPath, item.SaveKey)); this.syncProgressPage.updateTotalUploadProgress(); } else { this.syncProgressPage.updateUploadLog("上传失败 " + fileFullPath + "," + result.Text); this.syncProgressPage.addFileUploadErrorLog(string.Format("{0}\t{1}\t{2}\t{3}", this.syncSetting.TargetBucket, fileFullPath, item.SaveKey, result.Text)); } this.doneEvent.Set(); }