private static void CleanupDuplicateFiles(List <FileExtended> duplFiles, FileType fileType, SyncExecution syncExec) { var currentStep = 0; string syncLog = syncExec.SyncConfig.SyncLog; string archiveFolder = syncExec.SyncConfig.Parameters["ArchiveFolder"]; var filesList = fileType == FileType.Source ? syncExec.SourceFiles : syncExec.DestFiles; var duplValues = duplFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); foreach (var v in duplValues) { var tempFiles = duplFiles.FindAll(x => x.FileNameAndSize == v); while (tempFiles.Count > 1) { int lastInd = tempFiles.Count - 1; var lastFile = tempFiles[lastInd]; tempFiles.Remove(lastFile); duplFiles.Remove(lastFile); filesList.Remove(lastFile.fileID); WorkingWithFiles.ArchiveFile(lastFile, syncLog, archiveFolder, "duplicate"); currentStep++; Init.DisplayCompletionInfo("files processed", currentStep, duplFiles.Count - duplValues.Count + currentStep); } } Console.Write("\rDone. "); }
public static void MapFiles(SyncExecution syncExec) { var watchFileMapping = new Stopwatch(); var watchFileMappingFromCsv = new Stopwatch(); Console.WriteLine("\nPreparing file mapping..."); watchFileMapping.Start(); //watchFileMappingFromCsv.Start(); Console.WriteLine("populating filemapping from csv:"); foreach (var folderPair in syncExec.SyncConfig.FolderMappings) { CSVHelper.InitFileMappingFromCsv(syncExec, folderPair.Key); } //watchFileMappingFromCsv.Stop(); //Console.WriteLine("Done. Elapsed time: " // + FormatTime(watchFileMappingFromCsv.ElapsedMilliseconds)); // append existing file mapping if app_config has been modified later than csv mapping file // or if csv file does not exist if (syncExec.FilesMissingInMapping.Any()) { AddMissingFileMappingFromPaths(syncExec); } watchFileMapping.Stop(); Console.WriteLine("File mapping complete!"); Console.WriteLine($"elapsed time: {FormatTime(watchFileMapping.ElapsedMilliseconds)}"); }
public static void InitializeFiles(SyncExecution syncExec) { Console.WriteLine("\n"); Console.WriteLine("Fetching source and destination files..."); var folderMappings = syncExec.SyncConfig.FolderMappings; var watchInitFiles = new Stopwatch(); var watchSourceFiles = new Stopwatch(); var watchDestFiles = new Stopwatch(); var sourceFilesTemp = new List <string>(); var destFilesTemp = new List <string>(); watchInitFiles.Start(); foreach (var pair in folderMappings) { WorkingWithFiles.GetFiles(pair.Value.Item1, sourceFilesTemp); WorkingWithFiles.GetFiles(pair.Value.Item2, destFilesTemp); } _totalSourceFilesCount = sourceFilesTemp.Count; _totalDestFilesCount = destFilesTemp.Count; Console.WriteLine("Source files: " + _totalSourceFilesCount); Console.WriteLine("Destination Files: " + _totalDestFilesCount); Console.WriteLine("Populating source files list..."); foreach (var pair in folderMappings) { PopulateSourceFiles(syncExec, pair.Value.Item1, sourceFilesTemp); } Console.WriteLine("\rDone. "); Console.WriteLine("Populating destination files list..."); foreach (var pair in folderMappings) { PopulateDestFiles(syncExec, pair.Value.Item2, destFilesTemp); } Console.WriteLine("\rDone. "); Console.WriteLine($"Elapsed time: {FormatTime(watchInitFiles.ElapsedMilliseconds)}"); }
// the assumption is that CSV file has the following structure: // <firstFileType>,<firstBasePath>,<firstFileFullPath>,<firstFileId>, // <secondFileType>,<secondBasePath>,<secondFullPath>,<secondFileId> public static void InitFileMappingFromCsv(SyncExecution syncExec, string folderMappingKey) { string sourceBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item1; string destBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item2; if (syncExec.SourceFiles.Count(f => f.Value.basePath == sourceBasePath) + syncExec.DestFiles.Count(f => f.Value.basePath == destBasePath) == 0) { throw new Exception(folderMappingKey + "Could not perform the mapping: source and destination files have not been loaded yet."); } string fileMappingFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"]; string FileMappingCsvFullPath = fileMappingFolder + @"\" + folderMappingKey + ".csv"; if (File.Exists(FileMappingCsvFullPath)) { var linesRead = 0; Console.WriteLine("Fetching data from file " + FileMappingCsvFullPath + "..."); var fileMapping = syncExec.FileMappingFromCsv; // Read data from CSV file try { var totalLines = 0; using (var reader = File.OpenText(FileMappingCsvFullPath)) { while (reader.ReadLine() != null) { totalLines++; } } using (CsvFileReader reader = new CsvFileReader(FileMappingCsvFullPath)) { CsvRow row = new CsvRow(); int mappingsAdded = 0; while (reader.ReadRow(row)) { if (row.Count < 2) { break; } var firstFileType = (FileType)Enum.Parse(typeof(FileType), row[0]); var firstBasePath = row[1]; var firstFileFullPath = row[2]; var firstFileId = row[3]; var firstFileExtended = syncExec.GetFileByIdOrPath(firstFileType, firstFileId, firstFileFullPath); FileExtended secondFileExtended; var secondFileType = row[4]; var secondBasePath = row[5].Trim(); var secondFullPath = row[6].Trim(); var secondFileId = row[7].Trim(); if (String.IsNullOrEmpty(secondFileType) || String.IsNullOrEmpty(secondBasePath) || String.IsNullOrEmpty(secondFileId) ) { continue; } else { var secondFileType2 = (FileType)Enum.Parse(typeof(FileType), secondFileType); secondFileExtended = syncExec.GetFileByIdOrPath(secondFileType2, secondFileId, secondFullPath); } if (firstFileExtended != null) { if (fileMapping.ContainsKey(firstFileExtended)) { continue; } fileMapping.Add(firstFileExtended, secondFileExtended); mappingsAdded++; } else { if (secondFileExtended != null && !fileMapping.ContainsKey(secondFileExtended)) { fileMapping.Add(secondFileExtended, null); mappingsAdded++; } } if (firstFileExtended != null || secondFileExtended != null) { // the following method allows to determine file renaming, deletion, moving syncExec.AppendActionListWithDeleteRenameMove(firstFileExtended, secondFileExtended, row); } // if this mapping needs to be persisted: else if (syncExec.SyncConfig.AreBasePathsIncluded(firstBasePath, secondBasePath)) { syncExec.CsvMappingToPersist.Add(row); } linesRead++; //int destFilesUnmappedCount = //syncExec.FilesMissingInMapping.Count(s => s.fileType == FileType.Destination); var completionPercentage = (int)Math.Round(100 * (double)mappingsAdded / (syncExec.DestFiles.Count)); //Console.Write("\r\tlines read: " + linesRead + "; mapping completion: " // + completionPercentage + "%"); Init.DisplayCompletionInfo("line read", linesRead, totalLines); } Console.WriteLine("\rCompleted. File mapping lines read from csv: " + linesRead); } } catch (Exception ex) { linesRead = 0; fileMapping.Clear(); Console.WriteLine("could not read the csv file: " + ex.Message); Console.WriteLine("clearing file mapping, proceeding with populating from paths..."); } } else { Console.WriteLine("CSV file was not found, proceeding..."); } }
public static void SaveFileMappingToCsv(SyncExecution syncExec) { Console.WriteLine(); Console.WriteLine("Saving file mapping to CSV:"); // Write data to CSV file var fileMapping = syncExec.FileMapping; var csvMappingToPersist = syncExec.CsvMappingToPersist; string fileMappingCsvFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"]; if (!Directory.Exists(fileMappingCsvFolder)) { Directory.CreateDirectory(fileMappingCsvFolder); } foreach (var folderPair in syncExec.SyncConfig.FolderMappings) { string fileMappingCsvLocation = fileMappingCsvFolder + @"\" + folderPair.Key + ".csv"; string sourceFolder = folderPair.Value.Item1; string destFolder = folderPair.Value.Item2; List <CsvRow> csvMappingToPersistPartial = syncExec.GetFileMappingPersistentByFoldMapKey(folderPair.Key); using (var writer = new CsvFileWriter(fileMappingCsvLocation)) { foreach (var csvRow in csvMappingToPersistPartial) { writer.WriteRow(csvRow); } foreach (var filePair in fileMapping) { var file1 = filePair.Key; var file2 = filePair.Value; if (file1.basePath != sourceFolder && file1.basePath != destFolder) { continue; } if (file1 == null || file2 == null) { continue; } var file2FileType = file2.fileType.ToString(); var file2BasePath = file2.basePath; var file2FileID = file2.fileID; var file2FullPath = file2.fullPath; var row = new CsvRow { file1.fileType.ToString(), file1.basePath, file1.fullPath, file1.fileID, file2FileType, file2BasePath, file2FullPath, file2FileID }; writer.WriteRow(row); } } } Console.WriteLine("\tsaving to CSV completed"); }
public static void RemoveDuplicates(SyncExecution syncExec) { var w = new Stopwatch(); w.Start(); Console.WriteLine(); Console.WriteLine("Checking for duplicate files..."); var sourceFilesList = syncExec.SourceFiles.Values.ToList(); var destFilesList = syncExec.DestFiles.Values.ToList(); ExcludeSomeFolders(sourceFilesList, syncExec.SyncConfig); ExcludeSomeFolders(destFilesList, syncExec.SyncConfig); sourceFilesList.Sort(); destFilesList.Sort(); var sourceFilesToProcess = new List <FileExtended>(sourceFilesList); var destFilesToProcess = new List <FileExtended>(destFilesList); var duplSourceFiles = new List <FileExtended>() { Capacity = sourceFilesToProcess.Count }; var duplDestFiles = new List <FileExtended>() { Capacity = destFilesToProcess.Count }; CollectDuplicateFiles(sourceFilesToProcess, duplSourceFiles); CollectDuplicateFiles(destFilesToProcess, duplDestFiles); if (duplSourceFiles.Count == 0 && duplDestFiles.Count == 0) { Console.WriteLine("No duplicates have been found"); } else { var duplSourceValues = duplSourceFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); var duplDestValues = duplDestFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); int duplSourceCount = duplSourceFiles.Count - duplSourceValues.Count; int duplDestCount = duplDestFiles.Count - duplDestValues.Count; Console.WriteLine("Duplicates found: \n" + "\tsource files " + duplSourceCount + "\n" + "\tdestination files " + duplDestCount); Console.WriteLine("performing cleanup of source files..."); CleanupDuplicateFiles(duplSourceFiles, FileType.Source, syncExec); Console.WriteLine("\nperforming cleanup of destination files..."); CleanupDuplicateFiles(duplDestFiles, FileType.Destination, syncExec); Console.WriteLine("\nCleanup complete"); } //WriteToLog(duplSourceFiles, duplDestFiles, logFile); w.Stop(); Console.WriteLine("elapsed time: " + Init.FormatTime(w.ElapsedMilliseconds)); }
private static void PopulateSourceFiles(SyncExecution syncExec, string sourceFolder, List <string> filesList) { syncExec.SourceFiles = GetFilesDictionary(syncExec.SyncConfig, sourceFolder, filesList, FileType.Source); }
private static void PopulateDestFiles(SyncExecution syncExec, string destinationFolder, List <string> filesList) { syncExec.DestFiles = GetFilesDictionary(syncExec.SyncConfig, destinationFolder, filesList, FileType.Destination); }
private static void AddMissingFileMappingFromPaths(SyncExecution syncExec) { //if (syncExec.SourceFiles.Count > 0) //{ // launch timer var watchAddMissingFilesToMapping = new Stopwatch(); watchAddMissingFilesToMapping.Start(); //var relativePathComparer = new RelativePathComparer(); var sourceFilesMissingInMapping = syncExec.FilesMissingInMapping. Where(x => x.fileType == FileType.Source).ToList(); var destFilesMissingInMapping = syncExec.FilesMissingInMapping. Where(x => x.fileType == FileType.Destination).ToList(); Console.WriteLine("Sorting files missing in mapping..."); sourceFilesMissingInMapping.Sort(); destFilesMissingInMapping.Sort(); Console.WriteLine("sorting complete"); int expectedFileMappingEntriesCount = Math.Max(sourceFilesMissingInMapping.Count, destFilesMissingInMapping.Count); var fileMappingFromPaths = syncExec.FileMappingFromPaths; var sourceFilesWithoutCounterpart = new List <FileExtended> { Capacity = expectedFileMappingEntriesCount }; var sourceFilesToProcess = new List <FileExtended>(sourceFilesMissingInMapping); var destFilesToProcess = new List <FileExtended>(destFilesMissingInMapping); int filesToProcessTotal = sourceFilesToProcess.Count + destFilesToProcess.Count; Console.WriteLine(); Console.WriteLine("Populating missing FileMapping from paths:"); int filesProcessed = 0; while (sourceFilesToProcess.Count > 0) { filesProcessed++; int lastInd = sourceFilesToProcess.Count - 1; FileExtended sourceFile = sourceFilesToProcess[lastInd]; FileExtended destMatch = null; for (int i = destFilesToProcess.Count - 1; i >= 0; i--) { FileExtended destFile = destFilesToProcess[i]; if (destFile.RelativePath == sourceFile.RelativePath) { destMatch = destFile; break; } } if (destMatch != null) { destFilesToProcess.Remove(destMatch); fileMappingFromPaths.Add(sourceFile, destMatch); filesProcessed++; } else { for (int i = destFilesToProcess.Count - 1; i >= 0; i--) { FileExtended destFile = destFilesToProcess[i]; if (destFile.FileNameAndSize == sourceFile.FileNameAndSize) { destMatch = destFile; break; } } if (destMatch != null) { destFilesToProcess.Remove(destMatch); fileMappingFromPaths.Add(sourceFile, destMatch); filesProcessed++; } else { sourceFilesWithoutCounterpart.Add(sourceFile); } } sourceFilesToProcess.Remove(sourceFile); DisplayCompletionInfo("files processed", filesProcessed, filesToProcessTotal); } foreach (var sourceFile in sourceFilesWithoutCounterpart) { filesProcessed++; fileMappingFromPaths.Add(sourceFile, null); DisplayCompletionInfo("files processed", filesProcessed, filesToProcessTotal); } while (destFilesToProcess.Count > 0) { int destFileInd = destFilesToProcess.Count - 1; var destFile = destFilesToProcess[destFileInd]; filesProcessed++; fileMappingFromPaths.Add(destFile, null); destFilesToProcess.Remove(destFile); DisplayCompletionInfo("files processed", filesProcessed, filesToProcessTotal); } Console.Write("\rfinished populating missing FileMapping from paths. Added " + fileMappingFromPaths.Count + " entries."); Console.WriteLine("\nelapsed time: " + FormatTime(watchAddMissingFilesToMapping.ElapsedMilliseconds)); //} //else //{ // throw new Exception("Source files not loaded yet"); //} }