// the assumption is that CSV file has the following structure:
// <firstFileType>,<firstBasePath>,<firstFileFullPath>,<firstFileId>,
// <secondFileType>,<secondBasePath>,<secondFullPath>,<secondFileId>
public static void InitFileMappingFromCsv(SyncExecution syncExec, string folderMappingKey)
{
string sourceBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item1;
string destBasePath = syncExec.SyncConfig.FolderMappings[folderMappingKey].Item2;
if (syncExec.SourceFiles.Count(f => f.Value.basePath == sourceBasePath)
+ syncExec.DestFiles.Count(f => f.Value.basePath == destBasePath)
== 0)
{
throw new Exception(folderMappingKey
+ "Could not perform the mapping: source and destination files have not been loaded yet.");
}
string fileMappingFolder = syncExec.SyncConfig.Parameters["FileMappingFolder"];
string FileMappingCsvFullPath = fileMappingFolder + @"\" + folderMappingKey + ".csv";
if (File.Exists(FileMappingCsvFullPath))
{
var linesRead = 0;
Console.WriteLine("Fetching data from file " + FileMappingCsvFullPath + "...");
var fileMapping = syncExec.FileMappingFromCsv;
// Read data from CSV file
try
{
var totalLines = 0;
using (var reader = File.OpenText(FileMappingCsvFullPath))
{
while (reader.ReadLine() != null)
{
totalLines++;
}
}
using (CsvFileReader reader = new CsvFileReader(FileMappingCsvFullPath))
{
CsvRow row = new CsvRow();
int mappingsAdded = 0;
while (reader.ReadRow(row))
{
if (row.Count < 2)
{
break;
}
var firstFileType = (FileType)Enum.Parse(typeof(FileType), row[0]);
var firstBasePath = row[1];
var firstFileFullPath = row[2];
var firstFileId = row[3];
var firstFileExtended = syncExec.GetFileByIdOrPath(firstFileType, firstFileId, firstFileFullPath);
FileExtended secondFileExtended;
var secondFileType = row[4];
var secondBasePath = row[5].Trim();
var secondFullPath = row[6].Trim();
var secondFileId = row[7].Trim();
if (String.IsNullOrEmpty(secondFileType)
||
String.IsNullOrEmpty(secondBasePath)
||
String.IsNullOrEmpty(secondFileId)
)
{
continue;
}
else
{
var secondFileType2 = (FileType)Enum.Parse(typeof(FileType), secondFileType);
secondFileExtended = syncExec.GetFileByIdOrPath(secondFileType2, secondFileId, secondFullPath);
}
if (firstFileExtended != null)
{
if (fileMapping.ContainsKey(firstFileExtended))
{
continue;
}
fileMapping.Add(firstFileExtended, secondFileExtended);
mappingsAdded++;
}
else
{
if (secondFileExtended != null && !fileMapping.ContainsKey(secondFileExtended))
{
fileMapping.Add(secondFileExtended, null);
mappingsAdded++;
}
}
if (firstFileExtended != null
||
secondFileExtended != null)
{
// the following method allows to determine file renaming, deletion, moving
syncExec.AppendActionListWithDeleteRenameMove(firstFileExtended, secondFileExtended,
row);
}
// if this mapping needs to be persisted:
else if (syncExec.SyncConfig.AreBasePathsIncluded(firstBasePath, secondBasePath))
{
syncExec.CsvMappingToPersist.Add(row);
}
linesRead++;
//int destFilesUnmappedCount =
//syncExec.FilesMissingInMapping.Count(s => s.fileType == FileType.Destination);
var completionPercentage = (int)Math.Round(100 * (double)mappingsAdded
/ (syncExec.DestFiles.Count));
//Console.Write("\r\tlines read: " + linesRead + "; mapping completion: "
// + completionPercentage + "%");
Init.DisplayCompletionInfo("line read", linesRead, totalLines);
}
Console.WriteLine("\rCompleted. File mapping lines read from csv: " + linesRead);
}
}
catch (Exception ex)
{
linesRead = 0;
fileMapping.Clear();
Console.WriteLine("could not read the csv file: " + ex.Message);
Console.WriteLine("clearing file mapping, proceeding with populating from paths...");
}
}
else
{
Console.WriteLine("CSV file was not found, proceeding...");
}
}
public void PerformActions()
{
Console.WriteLine();
var actionsToPerform = _actionList.FindAll(x => x.ActionType != ActionType.None);
if (actionsToPerform.Count == 0)
{
Console.WriteLine("No changes have been detected - no actions needed");
return;
}
var syncWatch = new Stopwatch();
syncWatch.Start();
Console.WriteLine("Starting synchronization....");
foreach (var action in actionsToPerform)
{
var filesDict = WorkingWithFiles.GetSourceAndDestFile(action.File1, action.File2);
FileExtended sourceFile = filesDict[FileType.Source];
FileExtended destFile = filesDict[FileType.Destination];
try
{
switch (action.ActionType)
{
case ActionType.Create:
var creatingFileOp = action.ActionDirection == Direction.SourceToDestination
? sourceFile.fullPath
: destFile.fullPath;
DisplaySyncProcessStats("Copying file " + creatingFileOp);
ActionCreate(sourceFile, destFile, action.ActionDirection);
_filesCreated++;
//Init.DisplayCompletionInfo("actions performed",
// _filesCreated + _filesDeleted + _filesMoved + _filesRenamed + _filesRenamedMoved + _filesUpdated,
// ActionsList.Count);
break;
case ActionType.Update:
var updatingFile = action.ActionDirection == Direction.SourceToDestination
? destFile.fullPath
: sourceFile.fullPath;
DisplaySyncProcessStats("Updating file " + updatingFile);
ActionUpdate(sourceFile, destFile, action.ActionDirection);
_filesUpdated++;
break;
case ActionType.RenameMove:
var movingFile = action.ActionDirection == Direction.SourceToDestination
? destFile.fullPath
: sourceFile.fullPath;
DisplaySyncProcessStats("Renaming and moving file " + movingFile);
ActionRenameMove(sourceFile, destFile, action.ActionDirection);
_filesRenamedMoved++;
break;
case ActionType.Rename:
var renamingFile = action.ActionDirection == Direction.SourceToDestination
? destFile.fullPath
: sourceFile.fullPath;
DisplaySyncProcessStats("Renaming file " + renamingFile);
ActionRenameMove(sourceFile, destFile, action.ActionDirection);
_filesRenamed++;
break;
case ActionType.Move:
var movingFile2 = action.ActionDirection == Direction.SourceToDestination
? destFile.fullPath
: sourceFile.fullPath;
DisplaySyncProcessStats("Renaming and moving file " + movingFile2);
ActionRenameMove(sourceFile, destFile, action.ActionDirection);
_filesMoved++;
break;
//case ActionType.Delete:
// var archivingFile = action.ActionDirection == Direction.SourceToDestination
// ? destFile.fullPath
// : sourceFile.fullPath;
// DisplaySyncProcessStats("Archiving file " + archivingFile);
// ActionDelete(sourceFile, destFile, action.ActionDirection);
// _filesDeleted++;
// break;
}
action.SyncSuccess = true;
}
catch (Exception ex)
{
action.SyncSuccess = false;
action.ExceptionMessage = ex.Message;
_failedActions.Add(action);
}
}
syncWatch.Stop();
Console.WriteLine("\n\nSynchronization complete! Elapsed time: "
+ Init.FormatTime(syncWatch.ElapsedMilliseconds));
}
public static void RemoveDuplicates(SyncExecution syncExec)
{
var w = new Stopwatch();
w.Start();
Console.WriteLine();
Console.WriteLine("Checking for duplicate files...");
var sourceFilesList = syncExec.SourceFiles.Values.ToList();
var destFilesList = syncExec.DestFiles.Values.ToList();
ExcludeSomeFolders(sourceFilesList, syncExec.SyncConfig);
ExcludeSomeFolders(destFilesList, syncExec.SyncConfig);
sourceFilesList.Sort();
destFilesList.Sort();
var sourceFilesToProcess = new List <FileExtended>(sourceFilesList);
var destFilesToProcess = new List <FileExtended>(destFilesList);
var duplSourceFiles = new List <FileExtended>()
{
Capacity = sourceFilesToProcess.Count
};
var duplDestFiles = new List <FileExtended>()
{
Capacity = destFilesToProcess.Count
};
CollectDuplicateFiles(sourceFilesToProcess, duplSourceFiles);
CollectDuplicateFiles(destFilesToProcess, duplDestFiles);
if (duplSourceFiles.Count == 0 && duplDestFiles.Count == 0)
{
Console.WriteLine("No duplicates have been found");
}
else
{
var duplSourceValues = duplSourceFiles
.Select(x => x.FileNameAndSize)
.Distinct()
.ToList();
var duplDestValues = duplDestFiles
.Select(x => x.FileNameAndSize)
.Distinct()
.ToList();
int duplSourceCount = duplSourceFiles.Count - duplSourceValues.Count;
int duplDestCount = duplDestFiles.Count - duplDestValues.Count;
Console.WriteLine("Duplicates found: \n" +
"\tsource files " + duplSourceCount + "\n" +
"\tdestination files " + duplDestCount);
Console.WriteLine("performing cleanup of source files...");
CleanupDuplicateFiles(duplSourceFiles, FileType.Source, syncExec);
Console.WriteLine("\nperforming cleanup of destination files...");
CleanupDuplicateFiles(duplDestFiles, FileType.Destination, syncExec);
Console.WriteLine("\nCleanup complete");
}
//WriteToLog(duplSourceFiles, duplDestFiles, logFile);
w.Stop();
Console.WriteLine("elapsed time: " + Init.FormatTime(w.ElapsedMilliseconds));
}
