private async Task SyncCoreAsync(SyncFileInfo syncFileInfo, string currentFile, SyncRunResult result) { if (!(Activator.CreateInstance(_copyProgress) is ISyncProgress bar)) { throw new ApplicationException($"Invalid progress bar implementation: {_copyProgress}"); } var fileCopyUtil = new FileCopyUtil(syncFileInfo.SourceFile, syncFileInfo.TargetFile, syncFileInfo.Overwrite); var stopwatch = Stopwatch.StartNew(); await fileCopyUtil.CopyAsync( async() => { await bar.InitAsync($"{currentFile} ({ByteSize.FromBytes(fileCopyUtil.TotalBytes)})"); }, async e => { if (e != null) { _logger.Error($"Copying {currentFile} failed.", e); } else { await bar.CompleteAsync( "Finished", stopwatch.Elapsed, (double)fileCopyUtil.TotalBytes / stopwatch.ElapsedMilliseconds); result.TargetAffectedFileCount++; } }); }
private void Sync(string localDirectory, string ftpPath, Configuration config) { var localFiles = Directory.GetFiles(localDirectory); var ftpDirectory = GetDirectoryContent(ftpPath, false); var unprocessedFiles = new HashSet <FtpFileInfo>(ftpDirectory.Files); var localFileInfos = new HashSet <SyncFileInfo>(); _trackedFiles.TrackDirectory(localDirectory, ftpDirectory.FullPath); // check all files in current directory foreach (var localFilePath in localFiles) { var filename = Path.GetFileName(localFilePath); var fullLocalPath = localDirectory.CombinePath(filename); var syncFileInfo = new SyncFileInfo { LocalPath = fullLocalPath, LocalDetail = GetFileFullInfo(localFilePath), UpdateFtpDetail = true }; bool performUpload = false; var ftpInfo = ftpDirectory.Files.FirstOrDefault(i => i.FileName == ftpDirectory.FullPath.CombineFtp(filename)); try { localFileInfos.Add(syncFileInfo); var oldSyncInfo = _trackedFiles.Files.ByLocalPath(fullLocalPath); //.FirstOrDefault(i => i.LocalPath == fullLocalPath); if (ftpInfo != null) { unprocessedFiles.Remove(ftpInfo); } if (oldSyncInfo == null || string.IsNullOrEmpty(oldSyncInfo.FtpDetail)) { // file is untracked => new on client if (ftpInfo == null) { // file is not on server -> simple upload performUpload = true; } else { // file is already on server but not localy treated, we don't know, if the server and client file are the same. // Idea: we could download the file and compare, then we'd know if there is conflict or not, // but if the files differ, we would still have to decide wheather overwrite or skip. This is easier. if (_conflictResolver.OverwriteUntrackedFile(filename)) { Log.Warning("File {0} is already on server, overwritten.".Expand(filename)); performUpload = true; } else { Log.Warning("File {0} is already on server, skipped.".Expand(filename)); syncFileInfo.UpdateFtpDetail = false; } } } else { // tracked file if (ftpInfo == null) { // not on server -> new -> upload performUpload = true; } else { // we have current version from server if (oldSyncInfo.FtpDetail == ftpInfo.FtpDetail) { // file has changed locally or we upload all files -> upload if ((oldSyncInfo.LocalDetail != GetFileFullInfo(localFilePath)) || !config.UploadChangesOnly) { performUpload = true; } else { Log.Verbose("File not changed ({0})".Expand(filename)); } } else { // server has different version than we have -> server change -> conflict if (_conflictResolver.OverwriteServerModified(filename)) { Log.Warning("File {0} modified on server, overwritten.".Expand(filename)); performUpload = true; } else { Log.Warning("File {0} modified on server, skipped.".Expand(filename)); syncFileInfo.UpdateFtpDetail = false; } } } } if (performUpload) { Try(() => UploadFile(localFilePath, ftpDirectory.FullPath.CombineFtp(filename)), "Uploading file {0}... ".Expand(filename), len => "{0} bytes uploaded".Expand(len)); //Log.Info("File {0} uploaded ({1} bytes)".Expand(filename, len)); } } catch (Exception e) { Handle(e); syncFileInfo.UpdateFtpDetail = false; //if (!conflictedFiles.Contains(ftpInfo)) conflictedFiles.Add(ftpInfo); } } // delete files foreach (var fileInfo in unprocessedFiles) { var isTracked = _trackedFiles.Files.ByFtpDetail(fileInfo.FtpDetail) != null; // TODO: untracked files might be also downloaded and keep tracked! if (_conflictResolver.DeleteFile(fileInfo.FileName, isTracked)) { Try(() => DeleteFile(fileInfo.FileName), "Deleting file {0}...".Expand(fileInfo.FileName)); } } // recursively process directories on client var localDirs = Directory.GetDirectories(localDirectory); var ftpDirsToDelete = ftpDirectory.Directories.ToList(); // check directories foreach (var localDir in localDirs) { try { var dirname = new DirectoryInfo(localDir).Name; var ftpSubdirFullPath = ftpDirectory.FullPath.CombineFtp(dirname); var subdirInfo = ftpDirectory.Directories.FirstOrDefault(d => d.FullPath == ftpSubdirFullPath); bool dirExists = true; if (subdirInfo == null) { // directory is not on server => create Try(() => MakeDirectory( path: ftpSubdirFullPath), initialMessage: "Creating directory {0}...".Expand(dirname), onError: () => { dirExists = false; }); } else { ftpDirsToDelete.Remove(subdirInfo); } if (dirExists) { Sync(localDirectory.CombinePath(dirname), ftpSubdirFullPath, config); } else { Log.Error("Directory {0} was not synchronized.".Expand(dirname)); } } catch (Exception ex) { Handle(ex); } } // delete ftp directories which have been deleted on client (optional) { ftpDirsToDelete.ForEach( ftpDirToDelete => { bool isTracked = _trackedFiles.IsDirectoryTracked(ftpDirToDelete.FullPath); if (_conflictResolver.DeleteDirectory(ftpDirToDelete.FullPath, isTracked)) { Try( () => DeleteDirectory(GetDirectoryContent(ftpDirToDelete.FullPath)), "Deleting directory {0}...".Expand(ftpDirToDelete.FullPath) ); } }); } // update ftpinfo on local settings to remember the current state for each file (not directories) var ftpInfos = GetDirectoryContent(ftpDirectory.FullPath, false); // update local files state - we already have everything except ftpinfo in syncInfos foreach (var localInfo in localFileInfos) { var filename = Path.GetFileName(localInfo.LocalPath); var fullFtpPath = ftpDirectory.FullPath.CombineFtp(filename); if (localInfo.UpdateFtpDetail) { // update ftpinfo so that we keep current state for next update localInfo.FtpDetail = ftpInfos.Files.Where(f => f.FileName == fullFtpPath).Select(f => f.FtpDetail).FirstOrDefault(); } else { // for conflicted files use old ftpinfo, so that it's used next time again var fileInfo = _trackedFiles.Files.ByLocalPath(localInfo.LocalPath); if (fileInfo != null) { localInfo.FtpDetail = fileInfo.FtpDetail; } else { Log.Warning("Could not update information for file {0}".Expand(localInfo.LocalPath)); } } _trackedFiles.TrackFile(localInfo); } }
public void TrackFile(SyncFileInfo syncFileInfo) { _newfiles.Add(syncFileInfo); }
// File Copy (Creates list of local file names, paths, lm dates, and created dates) public async Task CopyAll(DirectoryInfo source) { var clientFiles = new List <SyncFileInfo>(); foreach (FileInfo fi in source.GetFiles()) { var clientFile = new SyncFileInfo { created = fi.CreationTime, lastModified = fi.LastWriteTime, filePath = fi.FullName, fileName = fi.Name, }; clientFiles.Add(clientFile); /*if (File.Exists(Path.Combine(target.FullName, fi.Name))) * { * string tFileName = Path.Combine(target.FullName, fi.Name); * FileInfo f2 = new FileInfo(tFileName); * DateTime lm = f2.LastWriteTime; * Console.WriteLine(@"File {0}\{1} already exists. Last modified {3}.", target.FullName, fi.Name, tFileName, lm); * * try * { * if (lastModified > lm) * { * Console.WriteLine(@"Source file {0}\{1} last modified {2} is newer than the target file {3}\{4} last modified {5}. Updating...", fi.DirectoryName, fi.Name, lastModified.ToString(), target.FullName, fi.Name, lm.ToString()); * fi.CopyTo(Path.Combine(target.FullName, fi.Name), true); * } * else * { * Console.WriteLine(@"Destination File {0}\{1} Skipped", target.FullName, fi.Name); * } * } * catch (Exception ex) * { * Console.WriteLine(ex.Message); * }*/ } // JSON payload and HTTP routing var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", userToken); var payload = JsonConvert.SerializeObject(clientFiles); var content = new StringContent(payload, Encoding.UTF8, "application/json"); var result = await httpClient.PostAsync("HTTP://*****:*****@"Copying {0}\{1}", target.FullName, fi.Name); * fi.CopyTo(Path.Combine(target.FullName, fi.Name), true); * } * * } * * foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) * { * DirectoryInfo nextTargetSubDir = target.CreateSubdirectory(diSourceSubDir.Name); * CopyAll(diSourceSubDir, nextTargetSubDir); * }*/ }
public void Add(SyncFileInfo info) { _byLocalPath.Add(info.LocalPath, info); _byFtpDetail.Add(info.FtpDetail, info); }
private async Task IngestCoreAsync(SyncConfig syncConfig, string src, string target, SyncIngestResult result) { _cancellationToken.ThrowIfCancellationRequested(); Directory.CreateDirectory(target); foreach (var dir in Directory.EnumerateDirectories(src, "*", SearchOption.TopDirectoryOnly)) { _cancellationToken.ThrowIfCancellationRequested(); result.SourceDirCount++; await IngestCoreAsync(syncConfig, dir, Path.Combine(target, new DirectoryInfo(dir).Name), result); } foreach (var file in Directory.EnumerateFiles(src, "*", SearchOption.TopDirectoryOnly)) { _cancellationToken.ThrowIfCancellationRequested(); result.SourceFileCount++; var fileName = Path.GetFileName(file); if (_options.ExcludePatterns.Value != null && _options.ExcludePatterns.Value.Any()) { var matched = false; foreach (var excludePattern in _options.ExcludePatterns.Value) { if (excludePattern.IsMatch(fileName)) { matched = true; break; } } if (matched) { continue; } } if (_options.IncludePatterns.Value != null && _options.IncludePatterns.Value.Any()) { var matched = false; foreach (var includePattern in _options.IncludePatterns.Value) { if (includePattern.IsMatch(fileName)) { matched = true; break; } } if (!matched) { continue; } } var fileInfo = new FileInfo(file); if (fileInfo.LastWriteTime < _options.NewerModifyDateTime) { continue; } var targetFilePath = Path.Combine(target, fileName); var syncInfo = new SyncFileInfo(syncConfig, file, targetFilePath); if (!File.Exists(targetFilePath)) { _syncQueue.Enqueue(syncInfo); } else { if (_options.Force) { syncInfo.Overwrite = true; _syncQueue.Enqueue(syncInfo); } else if (_options.Strict) { var exactlySame = await FileCompareUtil.ExactlySameAsync(file, targetFilePath, _cancellationToken); if (!exactlySame) { syncInfo.Overwrite = true; _syncQueue.Enqueue(syncInfo); } } } } }