private static void CleanupDuplicateFiles(List <FileExtended> duplFiles, FileType fileType, SyncExecution syncExec) { var currentStep = 0; string syncLog = syncExec.SyncConfig.SyncLog; string archiveFolder = syncExec.SyncConfig.Parameters["ArchiveFolder"]; var filesList = fileType == FileType.Source ? syncExec.SourceFiles : syncExec.DestFiles; var duplValues = duplFiles .Select(x => x.FileNameAndSize) .Distinct() .ToList(); foreach (var v in duplValues) { var tempFiles = duplFiles.FindAll(x => x.FileNameAndSize == v); while (tempFiles.Count > 1) { int lastInd = tempFiles.Count - 1; var lastFile = tempFiles[lastInd]; tempFiles.Remove(lastFile); duplFiles.Remove(lastFile); filesList.Remove(lastFile.fileID); WorkingWithFiles.ArchiveFile(lastFile, syncLog, archiveFolder, "duplicate"); currentStep++; Init.DisplayCompletionInfo("files processed", currentStep, duplFiles.Count - duplValues.Count + currentStep); } } Console.Write("\rDone. "); }
// the assumption is that CSV file has the following structure: // <firstFileType>,<firstBasePath>,<firstFileFullPath>,<firstFileId>, // <secondFileType>,<secondBasePath>,<secondFullPath>,<secondFileId> public static void InitFileMappingFromCsv(SyncExecution syncExec, string folderMappingKey) { string sourceBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item1; string destBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item2; if (syncExec.SourceFiles.Count(f => f.Value.basePath == sourceBasePath) + syncExec.DestFiles.Count(f => f.Value.basePath == destBasePath) == 0) { throw new Exception(folderMappingKey + "Could not perform the mapping: source and destination files have not been loaded yet."); } string fileMappingFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"]; string FileMappingCsvFullPath = fileMappingFolder + @"\" + folderMappingKey + ".csv"; if (File.Exists(FileMappingCsvFullPath)) { var linesRead = 0; Console.WriteLine("Fetching data from file " + FileMappingCsvFullPath + "..."); var fileMapping = syncExec.FileMappingFromCsv; // Read data from CSV file try { var totalLines = 0; using (var reader = File.OpenText(FileMappingCsvFullPath)) { while (reader.ReadLine() != null) { totalLines++; } } using (CsvFileReader reader = new CsvFileReader(FileMappingCsvFullPath)) { CsvRow row = new CsvRow(); int mappingsAdded = 0; while (reader.ReadRow(row)) { if (row.Count < 2) { break; } var firstFileType = (FileType)Enum.Parse(typeof(FileType), row[0]); var firstBasePath = row[1]; var firstFileFullPath = row[2]; var firstFileId = row[3]; var firstFileExtended = syncExec.GetFileByIdOrPath(firstFileType, firstFileId, firstFileFullPath); FileExtended secondFileExtended; var secondFileType = row[4]; var secondBasePath = row[5].Trim(); var secondFullPath = row[6].Trim(); var secondFileId = row[7].Trim(); if (String.IsNullOrEmpty(secondFileType) || String.IsNullOrEmpty(secondBasePath) || String.IsNullOrEmpty(secondFileId) ) { continue; } else { var secondFileType2 = (FileType)Enum.Parse(typeof(FileType), secondFileType); secondFileExtended = syncExec.GetFileByIdOrPath(secondFileType2, secondFileId, secondFullPath); } if (firstFileExtended != null) { if (fileMapping.ContainsKey(firstFileExtended)) { continue; } fileMapping.Add(firstFileExtended, secondFileExtended); mappingsAdded++; } else { if (secondFileExtended != null && !fileMapping.ContainsKey(secondFileExtended)) { fileMapping.Add(secondFileExtended, null); mappingsAdded++; } } if (firstFileExtended != null || secondFileExtended != null) { // the following method allows to determine file renaming, deletion, moving syncExec.AppendActionListWithDeleteRenameMove(firstFileExtended, secondFileExtended, row); } // if this mapping needs to be persisted: else if (syncExec.SyncConfig.AreBasePathsIncluded(firstBasePath, secondBasePath)) { syncExec.CsvMappingToPersist.Add(row); } linesRead++; //int destFilesUnmappedCount = //syncExec.FilesMissingInMapping.Count(s => s.fileType == FileType.Destination); var completionPercentage = (int)Math.Round(100 * (double)mappingsAdded / (syncExec.DestFiles.Count)); //Console.Write("\r\tlines read: " + linesRead + "; mapping completion: " // + completionPercentage + "%"); Init.DisplayCompletionInfo("line read", linesRead, totalLines); } Console.WriteLine("\rCompleted. File mapping lines read from csv: " + linesRead); } } catch (Exception ex) { linesRead = 0; fileMapping.Clear(); Console.WriteLine("could not read the csv file: " + ex.Message); Console.WriteLine("clearing file mapping, proceeding with populating from paths..."); } } else { Console.WriteLine("CSV file was not found, proceeding..."); } }