// the assumption is that CSV file has the following structure: // <firstFileType>,<firstBasePath>,<firstFileFullPath>,<firstFileId>, // <secondFileType>,<secondBasePath>,<secondFullPath>,<secondFileId> public static void InitFileMappingFromCsv(SyncExecution syncExec, string folderMappingKey) { string sourceBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item1; string destBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item2; if (syncExec.SourceFiles.Count(f => f.Value.basePath == sourceBasePath) + syncExec.DestFiles.Count(f => f.Value.basePath == destBasePath) == 0) { throw new Exception(folderMappingKey + "Could not perform the mapping: source and destination files have not been loaded yet."); } string fileMappingFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"]; string FileMappingCsvFullPath = fileMappingFolder + @"\" + folderMappingKey + ".csv"; if (File.Exists(FileMappingCsvFullPath)) { var linesRead = 0; Console.WriteLine("Fetching data from file " + FileMappingCsvFullPath + "..."); var fileMapping = syncExec.FileMappingFromCsv; // Read data from CSV file try { var totalLines = 0; using (var reader = File.OpenText(FileMappingCsvFullPath)) { while (reader.ReadLine() != null) { totalLines++; } } using (CsvFileReader reader = new CsvFileReader(FileMappingCsvFullPath)) { CsvRow row = new CsvRow(); int mappingsAdded = 0; while (reader.ReadRow(row)) { if (row.Count < 2) { break; } var firstFileType = (FileType)Enum.Parse(typeof(FileType), row[0]); var firstBasePath = row[1]; var firstFileFullPath = row[2]; var firstFileId = row[3]; var firstFileExtended = syncExec.GetFileByIdOrPath(firstFileType, firstFileId, firstFileFullPath); FileExtended secondFileExtended; var secondFileType = row[4]; var secondBasePath = row[5].Trim(); var secondFullPath = row[6].Trim(); var secondFileId = row[7].Trim(); if (String.IsNullOrEmpty(secondFileType) || String.IsNullOrEmpty(secondBasePath) || String.IsNullOrEmpty(secondFileId) ) { continue; } else { var secondFileType2 = (FileType)Enum.Parse(typeof(FileType), secondFileType); secondFileExtended = syncExec.GetFileByIdOrPath(secondFileType2, secondFileId, secondFullPath); } if (firstFileExtended != null) { if (fileMapping.ContainsKey(firstFileExtended)) { continue; } fileMapping.Add(firstFileExtended, secondFileExtended); mappingsAdded++; } else { if (secondFileExtended != null && !fileMapping.ContainsKey(secondFileExtended)) { fileMapping.Add(secondFileExtended, null); mappingsAdded++; } } if (firstFileExtended != null || secondFileExtended != null) { // the following method allows to determine file renaming, deletion, moving syncExec.AppendActionListWithDeleteRenameMove(firstFileExtended, secondFileExtended, row); } // if this mapping needs to be persisted: else if (syncExec.SyncConfig.AreBasePathsIncluded(firstBasePath, secondBasePath)) { syncExec.CsvMappingToPersist.Add(row); } linesRead++; //int destFilesUnmappedCount = //syncExec.FilesMissingInMapping.Count(s => s.fileType == FileType.Destination); var completionPercentage = (int)Math.Round(100 * (double)mappingsAdded / (syncExec.DestFiles.Count)); //Console.Write("\r\tlines read: " + linesRead + "; mapping completion: " // + completionPercentage + "%"); Init.DisplayCompletionInfo("line read", linesRead, totalLines); } Console.WriteLine("\rCompleted. File mapping lines read from csv: " + linesRead); } } catch (Exception ex) { linesRead = 0; fileMapping.Clear(); Console.WriteLine("could not read the csv file: " + ex.Message); Console.WriteLine("clearing file mapping, proceeding with populating from paths..."); } } else { Console.WriteLine("CSV file was not found, proceeding..."); } }
public void PerformActions() { Console.WriteLine(); var actionsToPerform = _actionList.FindAll(x => x.ActionType != ActionType.None); if (actionsToPerform.Count == 0) { Console.WriteLine("No changes have been detected - no actions needed"); return; } var syncWatch = new Stopwatch(); syncWatch.Start(); Console.WriteLine("Starting synchronization...."); foreach (var action in actionsToPerform) { var filesDict = WorkingWithFiles.GetSourceAndDestFile(action.File1, action.File2); FileExtended sourceFile = filesDict[FileType.Source]; FileExtended destFile = filesDict[FileType.Destination]; try { switch (action.ActionType) { case ActionType.Create: var creatingFileOp = action.ActionDirection == Direction.SourceToDestination ? sourceFile.fullPath : destFile.fullPath; DisplaySyncProcessStats("Copying file " + creatingFileOp); ActionCreate(sourceFile, destFile, action.ActionDirection); _filesCreated++; //Init.DisplayCompletionInfo("actions performed", // _filesCreated + _filesDeleted + _filesMoved + _filesRenamed + _filesRenamedMoved + _filesUpdated, // ActionsList.Count); break; case ActionType.Update: var updatingFile = action.ActionDirection == Direction.SourceToDestination ? destFile.fullPath : sourceFile.fullPath; DisplaySyncProcessStats("Updating file " + updatingFile); ActionUpdate(sourceFile, destFile, action.ActionDirection); _filesUpdated++; break; case ActionType.RenameMove: var movingFile = action.ActionDirection == Direction.SourceToDestination ? destFile.fullPath : sourceFile.fullPath; DisplaySyncProcessStats("Renaming and moving file " + movingFile); ActionRenameMove(sourceFile, destFile, action.ActionDirection); _filesRenamedMoved++; break; case ActionType.Rename: var renamingFile = action.ActionDirection == Direction.SourceToDestination ? destFile.fullPath : sourceFile.fullPath; DisplaySyncProcessStats("Renaming file " + renamingFile); ActionRenameMove(sourceFile, destFile, action.ActionDirection); _filesRenamed++; break; case ActionType.Move: var movingFile2 = action.ActionDirection == Direction.SourceToDestination ? destFile.fullPath : sourceFile.fullPath; DisplaySyncProcessStats("Renaming and moving file " + movingFile2); ActionRenameMove(sourceFile, destFile, action.ActionDirection); _filesMoved++; break; //case ActionType.Delete: // var archivingFile = action.ActionDirection == Direction.SourceToDestination // ? destFile.fullPath // : sourceFile.fullPath; // DisplaySyncProcessStats("Archiving file " + archivingFile); // ActionDelete(sourceFile, destFile, action.ActionDirection); // _filesDeleted++; // break; } action.SyncSuccess = true; } catch (Exception ex) { action.SyncSuccess = false; action.ExceptionMessage = ex.Message; _failedActions.Add(action); } } syncWatch.Stop(); Console.WriteLine("\n\nSynchronization complete! Elapsed time: " + Init.FormatTime(syncWatch.ElapsedMilliseconds)); }
public static void RemoveDuplicates(SyncExecution syncExec) { var w = new Stopwatch(); w.Start(); Console.WriteLine(); Console.WriteLine("Checking for duplicate files..."); var sourceFilesList = syncExec.SourceFiles.Values.ToList(); var destFilesList = syncExec.DestFiles.Values.ToList(); ExcludeSomeFolders(sourceFilesList, syncExec.SyncConfig); ExcludeSomeFolders(destFilesList, syncExec.SyncConfig); sourceFilesList.Sort(); destFilesList.Sort(); var sourceFilesToProcess = new List <FileExtended>(sourceFilesList); var destFilesToProcess = new List <FileExtended>(destFilesList); var duplSourceFiles = new List <FileExtended>() { Capacity = sourceFilesToProcess.Count }; var duplDestFiles = new List <FileExtended>() { Capacity = destFilesToProcess.Count }; CollectDuplicateFiles(sourceFilesToProcess, duplSourceFiles); CollectDuplicateFiles(destFilesToProcess, duplDestFiles); if (duplSourceFiles.Count == 0 && duplDestFiles.Count == 0) { Console.WriteLine("No duplicates have been found"); } else { var duplSourceValues = duplSourceFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); var duplDestValues = duplDestFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); int duplSourceCount = duplSourceFiles.Count - duplSourceValues.Count; int duplDestCount = duplDestFiles.Count - duplDestValues.Count; Console.WriteLine("Duplicates found: \n" + "\tsource files " + duplSourceCount + "\n" + "\tdestination files " + duplDestCount); Console.WriteLine("performing cleanup of source files..."); CleanupDuplicateFiles(duplSourceFiles, FileType.Source, syncExec); Console.WriteLine("\nperforming cleanup of destination files..."); CleanupDuplicateFiles(duplDestFiles, FileType.Destination, syncExec); Console.WriteLine("\nCleanup complete"); } //WriteToLog(duplSourceFiles, duplDestFiles, logFile); w.Stop(); Console.WriteLine("elapsed time: " + Init.FormatTime(w.ElapsedMilliseconds)); }