private static void CleanupDuplicateFiles(List <FileExtended> duplFiles, FileType fileType,
SyncExecution syncExec)
{
var currentStep = 0;
string syncLog = syncExec.SyncConfig.SyncLog;
string archiveFolder = syncExec.SyncConfig.Parameters["ArchiveFolder"];
var filesList = fileType == FileType.Source ? syncExec.SourceFiles : syncExec.DestFiles;
var duplValues = duplFiles
.Select(x => x.FileNameAndSize)
.Distinct()
.ToList();
foreach (var v in duplValues)
{
var tempFiles = duplFiles.FindAll(x => x.FileNameAndSize == v);
while (tempFiles.Count > 1)
{
int lastInd = tempFiles.Count - 1;
var lastFile = tempFiles[lastInd];
tempFiles.Remove(lastFile);
duplFiles.Remove(lastFile);
filesList.Remove(lastFile.fileID);
WorkingWithFiles.ArchiveFile(lastFile, syncLog, archiveFolder, "duplicate");
currentStep++;
Init.DisplayCompletionInfo("files processed",
currentStep, duplFiles.Count - duplValues.Count + currentStep);
}
}
Console.Write("\rDone. ");
}
public static void MapFiles(SyncExecution syncExec)
{
var watchFileMapping = new Stopwatch();
var watchFileMappingFromCsv = new Stopwatch();
Console.WriteLine("\nPreparing file mapping...");
watchFileMapping.Start();
//watchFileMappingFromCsv.Start();
Console.WriteLine("populating filemapping from csv:");
foreach (var folderPair in syncExec.SyncConfig.FolderMappings)
{
CSVHelper.InitFileMappingFromCsv(syncExec, folderPair.Key);
}
//watchFileMappingFromCsv.Stop();
//Console.WriteLine("Done. Elapsed time: "
// + FormatTime(watchFileMappingFromCsv.ElapsedMilliseconds));
// append existing file mapping if app_config has been modified later than csv mapping file
// or if csv file does not exist
if (syncExec.FilesMissingInMapping.Any())
{
AddMissingFileMappingFromPaths(syncExec);
}
watchFileMapping.Stop();
Console.WriteLine("File mapping complete!");
Console.WriteLine($"elapsed time: {FormatTime(watchFileMapping.ElapsedMilliseconds)}");
}
public static void InitializeFiles(SyncExecution syncExec)
{
Console.WriteLine("\n");
Console.WriteLine("Fetching source and destination files...");
var folderMappings = syncExec.SyncConfig.FolderMappings;
var watchInitFiles = new Stopwatch();
var watchSourceFiles = new Stopwatch();
var watchDestFiles = new Stopwatch();
var sourceFilesTemp = new List <string>();
var destFilesTemp = new List <string>();
watchInitFiles.Start();
foreach (var pair in folderMappings)
{
WorkingWithFiles.GetFiles(pair.Value.Item1, sourceFilesTemp);
WorkingWithFiles.GetFiles(pair.Value.Item2, destFilesTemp);
}
_totalSourceFilesCount = sourceFilesTemp.Count;
_totalDestFilesCount = destFilesTemp.Count;
Console.WriteLine("Source files: " + _totalSourceFilesCount);
Console.WriteLine("Destination Files: " + _totalDestFilesCount);
Console.WriteLine("Populating source files list...");
foreach (var pair in folderMappings)
{
PopulateSourceFiles(syncExec, pair.Value.Item1, sourceFilesTemp);
}
Console.WriteLine("\rDone. ");
Console.WriteLine("Populating destination files list...");
foreach (var pair in folderMappings)
{
PopulateDestFiles(syncExec, pair.Value.Item2, destFilesTemp);
}
Console.WriteLine("\rDone. ");
Console.WriteLine($"Elapsed time: {FormatTime(watchInitFiles.ElapsedMilliseconds)}");
}
// the assumption is that CSV file has the following structure:
// <firstFileType>,<firstBasePath>,<firstFileFullPath>,<firstFileId>,
// <secondFileType>,<secondBasePath>,<secondFullPath>,<secondFileId>
public static void InitFileMappingFromCsv(SyncExecution syncExec, string folderMappingKey)
{
string sourceBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item1;
string destBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item2;
if (syncExec.SourceFiles.Count(f => f.Value.basePath == sourceBasePath)
+ syncExec.DestFiles.Count(f => f.Value.basePath == destBasePath)
== 0)
{
throw new Exception(folderMappingKey
+ "Could not perform the mapping: source and destination files have not been loaded yet.");
}
string fileMappingFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"];
string FileMappingCsvFullPath = fileMappingFolder + @"\" + folderMappingKey + ".csv";
if (File.Exists(FileMappingCsvFullPath))
{
var linesRead = 0;
Console.WriteLine("Fetching data from file " + FileMappingCsvFullPath + "...");
var fileMapping = syncExec.FileMappingFromCsv;
// Read data from CSV file
try
{
var totalLines = 0;
using (var reader = File.OpenText(FileMappingCsvFullPath))
{
while (reader.ReadLine() != null)
{
totalLines++;
}
}
using (CsvFileReader reader = new CsvFileReader(FileMappingCsvFullPath))
{
CsvRow row = new CsvRow();
int mappingsAdded = 0;
while (reader.ReadRow(row))
{
if (row.Count < 2)
{
break;
}
var firstFileType = (FileType)Enum.Parse(typeof(FileType), row[0]);
var firstBasePath = row[1];
var firstFileFullPath = row[2];
var firstFileId = row[3];
var firstFileExtended = syncExec.GetFileByIdOrPath(firstFileType, firstFileId, firstFileFullPath);
FileExtended secondFileExtended;
var secondFileType = row[4];
var secondBasePath = row[5].Trim();
var secondFullPath = row[6].Trim();
var secondFileId = row[7].Trim();
if (String.IsNullOrEmpty(secondFileType)
||
String.IsNullOrEmpty(secondBasePath)
||
String.IsNullOrEmpty(secondFileId)
)
{
continue;
}
else
{
var secondFileType2 = (FileType)Enum.Parse(typeof(FileType), secondFileType);
secondFileExtended = syncExec.GetFileByIdOrPath(secondFileType2, secondFileId, secondFullPath);
}
if (firstFileExtended != null)
{
if (fileMapping.ContainsKey(firstFileExtended))
{
continue;
}
fileMapping.Add(firstFileExtended, secondFileExtended);
mappingsAdded++;
}
else
{
if (secondFileExtended != null && !fileMapping.ContainsKey(secondFileExtended))
{
fileMapping.Add(secondFileExtended, null);
mappingsAdded++;
}
}
if (firstFileExtended != null
||
secondFileExtended != null)
{
// the following method allows to determine file renaming, deletion, moving
syncExec.AppendActionListWithDeleteRenameMove(firstFileExtended, secondFileExtended,
row);
}
// if this mapping needs to be persisted:
else if (syncExec.SyncConfig.AreBasePathsIncluded(firstBasePath, secondBasePath))
{
syncExec.CsvMappingToPersist.Add(row);
}
linesRead++;
//int destFilesUnmappedCount =
//syncExec.FilesMissingInMapping.Count(s => s.fileType == FileType.Destination);
var completionPercentage = (int)Math.Round(100 * (double)mappingsAdded
/ (syncExec.DestFiles.Count));
//Console.Write("\r\tlines read: " + linesRead + "; mapping completion: "
// + completionPercentage + "%");
Init.DisplayCompletionInfo("line read", linesRead, totalLines);
}
Console.WriteLine("\rCompleted. File mapping lines read from csv: " + linesRead);
}
}
catch (Exception ex)
{
linesRead = 0;
fileMapping.Clear();
Console.WriteLine("could not read the csv file: " + ex.Message);
Console.WriteLine("clearing file mapping, proceeding with populating from paths...");
}
}
else
{
Console.WriteLine("CSV file was not found, proceeding...");
}
}
public static void SaveFileMappingToCsv(SyncExecution syncExec)
{
Console.WriteLine();
Console.WriteLine("Saving file mapping to CSV:");
// Write data to CSV file
var fileMapping = syncExec.FileMapping;
var csvMappingToPersist = syncExec.CsvMappingToPersist;
string fileMappingCsvFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"];
if (!Directory.Exists(fileMappingCsvFolder))
{
Directory.CreateDirectory(fileMappingCsvFolder);
}
foreach (var folderPair in syncExec.SyncConfig.FolderMappings)
{
string fileMappingCsvLocation = fileMappingCsvFolder + @"\"
+ folderPair.Key + ".csv";
string sourceFolder = folderPair.Value.Item1;
string destFolder = folderPair.Value.Item2;
List <CsvRow> csvMappingToPersistPartial = syncExec.GetFileMappingPersistentByFoldMapKey(folderPair.Key);
using (var writer = new CsvFileWriter(fileMappingCsvLocation))
{
foreach (var csvRow in csvMappingToPersistPartial)
{
writer.WriteRow(csvRow);
}
foreach (var filePair in fileMapping)
{
var file1 = filePair.Key;
var file2 = filePair.Value;
if (file1.basePath != sourceFolder && file1.basePath != destFolder)
{
continue;
}
if (file1 == null || file2 == null)
{
continue;
}
var file2FileType = file2.fileType.ToString();
var file2BasePath = file2.basePath;
var file2FileID = file2.fileID;
var file2FullPath = file2.fullPath;
var row = new CsvRow
{
file1.fileType.ToString(),
file1.basePath,
file1.fullPath,
file1.fileID,
file2FileType,
file2BasePath,
file2FullPath,
file2FileID
};
writer.WriteRow(row);
}
}
}
Console.WriteLine("\tsaving to CSV completed");
}
public static void RemoveDuplicates(SyncExecution syncExec)
{
var w = new Stopwatch();
w.Start();
Console.WriteLine();
Console.WriteLine("Checking for duplicate files...");
var sourceFilesList = syncExec.SourceFiles.Values.ToList();
var destFilesList = syncExec.DestFiles.Values.ToList();
ExcludeSomeFolders(sourceFilesList, syncExec.SyncConfig);
ExcludeSomeFolders(destFilesList, syncExec.SyncConfig);
sourceFilesList.Sort();
destFilesList.Sort();
var sourceFilesToProcess = new List <FileExtended>(sourceFilesList);
var destFilesToProcess = new List <FileExtended>(destFilesList);
var duplSourceFiles = new List <FileExtended>()
{
Capacity = sourceFilesToProcess.Count
};
var duplDestFiles = new List <FileExtended>()
{
Capacity = destFilesToProcess.Count
};
CollectDuplicateFiles(sourceFilesToProcess, duplSourceFiles);
CollectDuplicateFiles(destFilesToProcess, duplDestFiles);
if (duplSourceFiles.Count == 0 && duplDestFiles.Count == 0)
{
Console.WriteLine("No duplicates have been found");
}
else
{
var duplSourceValues = duplSourceFiles
.Select(x => x.FileNameAndSize)
.Distinct()
.ToList();
var duplDestValues = duplDestFiles
.Select(x => x.FileNameAndSize)
.Distinct()
.ToList();
int duplSourceCount = duplSourceFiles.Count - duplSourceValues.Count;
int duplDestCount = duplDestFiles.Count - duplDestValues.Count;
Console.WriteLine("Duplicates found: \n" +
"\tsource files " + duplSourceCount + "\n" +
"\tdestination files " + duplDestCount);
Console.WriteLine("performing cleanup of source files...");
CleanupDuplicateFiles(duplSourceFiles, FileType.Source, syncExec);
Console.WriteLine("\nperforming cleanup of destination files...");
CleanupDuplicateFiles(duplDestFiles, FileType.Destination, syncExec);
Console.WriteLine("\nCleanup complete");
}
//WriteToLog(duplSourceFiles, duplDestFiles, logFile);
w.Stop();
Console.WriteLine("elapsed time: " + Init.FormatTime(w.ElapsedMilliseconds));
}
private static void PopulateSourceFiles(SyncExecution syncExec, string sourceFolder, List <string> filesList)
{
syncExec.SourceFiles = GetFilesDictionary(syncExec.SyncConfig, sourceFolder, filesList, FileType.Source);
}
private static void PopulateDestFiles(SyncExecution syncExec, string destinationFolder, List <string> filesList)
{
syncExec.DestFiles = GetFilesDictionary(syncExec.SyncConfig, destinationFolder, filesList, FileType.Destination);
}
private static void AddMissingFileMappingFromPaths(SyncExecution syncExec)
{
//if (syncExec.SourceFiles.Count > 0)
//{
// launch timer
var watchAddMissingFilesToMapping = new Stopwatch();
watchAddMissingFilesToMapping.Start();
//var relativePathComparer = new RelativePathComparer();
var sourceFilesMissingInMapping = syncExec.FilesMissingInMapping.
Where(x => x.fileType == FileType.Source).ToList();
var destFilesMissingInMapping = syncExec.FilesMissingInMapping.
Where(x => x.fileType == FileType.Destination).ToList();
Console.WriteLine("Sorting files missing in mapping...");
sourceFilesMissingInMapping.Sort();
destFilesMissingInMapping.Sort();
Console.WriteLine("sorting complete");
int expectedFileMappingEntriesCount = Math.Max(sourceFilesMissingInMapping.Count,
destFilesMissingInMapping.Count);
var fileMappingFromPaths = syncExec.FileMappingFromPaths;
var sourceFilesWithoutCounterpart = new List <FileExtended>
{
Capacity = expectedFileMappingEntriesCount
};
var sourceFilesToProcess = new List <FileExtended>(sourceFilesMissingInMapping);
var destFilesToProcess = new List <FileExtended>(destFilesMissingInMapping);
int filesToProcessTotal = sourceFilesToProcess.Count + destFilesToProcess.Count;
Console.WriteLine();
Console.WriteLine("Populating missing FileMapping from paths:");
int filesProcessed = 0;
while (sourceFilesToProcess.Count > 0)
{
filesProcessed++;
int lastInd = sourceFilesToProcess.Count - 1;
FileExtended sourceFile = sourceFilesToProcess[lastInd];
FileExtended destMatch = null;
for (int i = destFilesToProcess.Count - 1; i >= 0; i--)
{
FileExtended destFile = destFilesToProcess[i];
if (destFile.RelativePath == sourceFile.RelativePath)
{
destMatch = destFile;
break;
}
}
if (destMatch != null)
{
destFilesToProcess.Remove(destMatch);
fileMappingFromPaths.Add(sourceFile, destMatch);
filesProcessed++;
}
else
{
for (int i = destFilesToProcess.Count - 1; i >= 0; i--)
{
FileExtended destFile = destFilesToProcess[i];
if (destFile.FileNameAndSize == sourceFile.FileNameAndSize)
{
destMatch = destFile;
break;
}
}
if (destMatch != null)
{
destFilesToProcess.Remove(destMatch);
fileMappingFromPaths.Add(sourceFile, destMatch);
filesProcessed++;
}
else
{
sourceFilesWithoutCounterpart.Add(sourceFile);
}
}
sourceFilesToProcess.Remove(sourceFile);
DisplayCompletionInfo("files processed", filesProcessed,
filesToProcessTotal);
}
foreach (var sourceFile in sourceFilesWithoutCounterpart)
{
filesProcessed++;
fileMappingFromPaths.Add(sourceFile, null);
DisplayCompletionInfo("files processed", filesProcessed,
filesToProcessTotal);
}
while (destFilesToProcess.Count > 0)
{
int destFileInd = destFilesToProcess.Count - 1;
var destFile = destFilesToProcess[destFileInd];
filesProcessed++;
fileMappingFromPaths.Add(destFile, null);
destFilesToProcess.Remove(destFile);
DisplayCompletionInfo("files processed", filesProcessed,
filesToProcessTotal);
}
Console.Write("\rfinished populating missing FileMapping from paths. Added " + fileMappingFromPaths.Count + " entries.");
Console.WriteLine("\nelapsed time: " + FormatTime(watchAddMissingFilesToMapping.ElapsedMilliseconds));
//}
//else
//{
// throw new Exception("Source files not loaded yet");
//}
}
