private async Task SyncCoreAsync(SyncFileInfo syncFileInfo, string currentFile, SyncRunResult result)
{
if (!(Activator.CreateInstance(_copyProgress) is ISyncProgress bar))
{
throw new ApplicationException($"Invalid progress bar implementation: {_copyProgress}");
}
var fileCopyUtil =
new FileCopyUtil(syncFileInfo.SourceFile, syncFileInfo.TargetFile, syncFileInfo.Overwrite);
var stopwatch = Stopwatch.StartNew();
await fileCopyUtil.CopyAsync(
async() => { await bar.InitAsync($"{currentFile} ({ByteSize.FromBytes(fileCopyUtil.TotalBytes)})"); },
async e =>
{
if (e != null)
{
_logger.Error($"Copying {currentFile} failed.", e);
}
else
{
await bar.CompleteAsync(
"Finished",
stopwatch.Elapsed,
(double)fileCopyUtil.TotalBytes / stopwatch.ElapsedMilliseconds);
result.TargetAffectedFileCount++;
}
});
}
private void Sync(string localDirectory, string ftpPath, Configuration config)
{
var localFiles = Directory.GetFiles(localDirectory);
var ftpDirectory = GetDirectoryContent(ftpPath, false);
var unprocessedFiles = new HashSet <FtpFileInfo>(ftpDirectory.Files);
var localFileInfos = new HashSet <SyncFileInfo>();
_trackedFiles.TrackDirectory(localDirectory, ftpDirectory.FullPath);
// check all files in current directory
foreach (var localFilePath in localFiles)
{
var filename = Path.GetFileName(localFilePath);
var fullLocalPath = localDirectory.CombinePath(filename);
var syncFileInfo = new SyncFileInfo
{
LocalPath = fullLocalPath,
LocalDetail = GetFileFullInfo(localFilePath),
UpdateFtpDetail = true
};
bool performUpload = false;
var ftpInfo = ftpDirectory.Files.FirstOrDefault(i => i.FileName == ftpDirectory.FullPath.CombineFtp(filename));
try
{
localFileInfos.Add(syncFileInfo);
var oldSyncInfo = _trackedFiles.Files.ByLocalPath(fullLocalPath); //.FirstOrDefault(i => i.LocalPath == fullLocalPath);
if (ftpInfo != null)
{
unprocessedFiles.Remove(ftpInfo);
}
if (oldSyncInfo == null || string.IsNullOrEmpty(oldSyncInfo.FtpDetail))
{
// file is untracked => new on client
if (ftpInfo == null)
{
// file is not on server -> simple upload
performUpload = true;
}
else
{
// file is already on server but not localy treated, we don't know, if the server and client file are the same.
// Idea: we could download the file and compare, then we'd know if there is conflict or not,
// but if the files differ, we would still have to decide wheather overwrite or skip. This is easier.
if (_conflictResolver.OverwriteUntrackedFile(filename))
{
Log.Warning("File {0} is already on server, overwritten.".Expand(filename));
performUpload = true;
}
else
{
Log.Warning("File {0} is already on server, skipped.".Expand(filename));
syncFileInfo.UpdateFtpDetail = false;
}
}
}
else
{
// tracked file
if (ftpInfo == null)
{
// not on server -> new -> upload
performUpload = true;
}
else
{
// we have current version from server
if (oldSyncInfo.FtpDetail == ftpInfo.FtpDetail)
{
// file has changed locally or we upload all files -> upload
if ((oldSyncInfo.LocalDetail != GetFileFullInfo(localFilePath)) || !config.UploadChangesOnly)
{
performUpload = true;
}
else
{
Log.Verbose("File not changed ({0})".Expand(filename));
}
}
else
{
// server has different version than we have -> server change -> conflict
if (_conflictResolver.OverwriteServerModified(filename))
{
Log.Warning("File {0} modified on server, overwritten.".Expand(filename));
performUpload = true;
}
else
{
Log.Warning("File {0} modified on server, skipped.".Expand(filename));
syncFileInfo.UpdateFtpDetail = false;
}
}
}
}
if (performUpload)
{
Try(() =>
UploadFile(localFilePath, ftpDirectory.FullPath.CombineFtp(filename)),
"Uploading file {0}... ".Expand(filename),
len => "{0} bytes uploaded".Expand(len));
//Log.Info("File {0} uploaded ({1} bytes)".Expand(filename, len));
}
}
catch (Exception e)
{
Handle(e);
syncFileInfo.UpdateFtpDetail = false;
//if (!conflictedFiles.Contains(ftpInfo)) conflictedFiles.Add(ftpInfo);
}
}
// delete files
foreach (var fileInfo in unprocessedFiles)
{
var isTracked = _trackedFiles.Files.ByFtpDetail(fileInfo.FtpDetail) != null;
// TODO: untracked files might be also downloaded and keep tracked!
if (_conflictResolver.DeleteFile(fileInfo.FileName, isTracked))
{
Try(() => DeleteFile(fileInfo.FileName), "Deleting file {0}...".Expand(fileInfo.FileName));
}
}
// recursively process directories on client
var localDirs = Directory.GetDirectories(localDirectory);
var ftpDirsToDelete = ftpDirectory.Directories.ToList();
// check directories
foreach (var localDir in localDirs)
{
try
{
var dirname = new DirectoryInfo(localDir).Name;
var ftpSubdirFullPath = ftpDirectory.FullPath.CombineFtp(dirname);
var subdirInfo = ftpDirectory.Directories.FirstOrDefault(d => d.FullPath == ftpSubdirFullPath);
bool dirExists = true;
if (subdirInfo == null)
{
// directory is not on server => create
Try(() => MakeDirectory(
path: ftpSubdirFullPath),
initialMessage: "Creating directory {0}...".Expand(dirname),
onError: () =>
{
dirExists = false;
});
}
else
{
ftpDirsToDelete.Remove(subdirInfo);
}
if (dirExists)
{
Sync(localDirectory.CombinePath(dirname), ftpSubdirFullPath, config);
}
else
{
Log.Error("Directory {0} was not synchronized.".Expand(dirname));
}
}
catch (Exception ex)
{
Handle(ex);
}
}
// delete ftp directories which have been deleted on client (optional)
{
ftpDirsToDelete.ForEach(
ftpDirToDelete =>
{
bool isTracked = _trackedFiles.IsDirectoryTracked(ftpDirToDelete.FullPath);
if (_conflictResolver.DeleteDirectory(ftpDirToDelete.FullPath, isTracked))
{
Try(
() => DeleteDirectory(GetDirectoryContent(ftpDirToDelete.FullPath)),
"Deleting directory {0}...".Expand(ftpDirToDelete.FullPath)
);
}
});
}
// update ftpinfo on local settings to remember the current state for each file (not directories)
var ftpInfos = GetDirectoryContent(ftpDirectory.FullPath, false);
// update local files state - we already have everything except ftpinfo in syncInfos
foreach (var localInfo in localFileInfos)
{
var filename = Path.GetFileName(localInfo.LocalPath);
var fullFtpPath = ftpDirectory.FullPath.CombineFtp(filename);
if (localInfo.UpdateFtpDetail)
{
// update ftpinfo so that we keep current state for next update
localInfo.FtpDetail = ftpInfos.Files.Where(f => f.FileName == fullFtpPath).Select(f => f.FtpDetail).FirstOrDefault();
}
else
{
// for conflicted files use old ftpinfo, so that it's used next time again
var fileInfo = _trackedFiles.Files.ByLocalPath(localInfo.LocalPath);
if (fileInfo != null)
{
localInfo.FtpDetail = fileInfo.FtpDetail;
}
else
{
Log.Warning("Could not update information for file {0}".Expand(localInfo.LocalPath));
}
}
_trackedFiles.TrackFile(localInfo);
}
}
public void TrackFile(SyncFileInfo syncFileInfo)
{
_newfiles.Add(syncFileInfo);
}
// File Copy (Creates list of local file names, paths, lm dates, and created dates)
public async Task CopyAll(DirectoryInfo source)
{
var clientFiles = new List <SyncFileInfo>();
foreach (FileInfo fi in source.GetFiles())
{
var clientFile = new SyncFileInfo
{
created = fi.CreationTime,
lastModified = fi.LastWriteTime,
filePath = fi.FullName,
fileName = fi.Name,
};
clientFiles.Add(clientFile);
/*if (File.Exists(Path.Combine(target.FullName, fi.Name)))
* {
* string tFileName = Path.Combine(target.FullName, fi.Name);
* FileInfo f2 = new FileInfo(tFileName);
* DateTime lm = f2.LastWriteTime;
* Console.WriteLine(@"File {0}\{1} already exists. Last modified {3}.", target.FullName, fi.Name, tFileName, lm);
*
* try
* {
* if (lastModified > lm)
* {
* Console.WriteLine(@"Source file {0}\{1} last modified {2} is newer than the target file {3}\{4} last modified {5}. Updating...", fi.DirectoryName, fi.Name, lastModified.ToString(), target.FullName, fi.Name, lm.ToString());
* fi.CopyTo(Path.Combine(target.FullName, fi.Name), true);
* }
* else
* {
* Console.WriteLine(@"Destination File {0}\{1} Skipped", target.FullName, fi.Name);
* }
* }
* catch (Exception ex)
* {
* Console.WriteLine(ex.Message);
* }*/
}
// JSON payload and HTTP routing
var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", userToken);
var payload = JsonConvert.SerializeObject(clientFiles);
var content = new StringContent(payload, Encoding.UTF8, "application/json");
var result = await httpClient.PostAsync("HTTP://*****:*****@"Copying {0}\{1}", target.FullName, fi.Name);
* fi.CopyTo(Path.Combine(target.FullName, fi.Name), true);
* }
*
* }
*
* foreach (DirectoryInfo diSourceSubDir in source.GetDirectories())
* {
* DirectoryInfo nextTargetSubDir = target.CreateSubdirectory(diSourceSubDir.Name);
* CopyAll(diSourceSubDir, nextTargetSubDir);
* }*/
}
public void Add(SyncFileInfo info)
{
_byLocalPath.Add(info.LocalPath, info);
_byFtpDetail.Add(info.FtpDetail, info);
}
private async Task IngestCoreAsync(SyncConfig syncConfig, string src, string target, SyncIngestResult result)
{
_cancellationToken.ThrowIfCancellationRequested();
Directory.CreateDirectory(target);
foreach (var dir in Directory.EnumerateDirectories(src, "*", SearchOption.TopDirectoryOnly))
{
_cancellationToken.ThrowIfCancellationRequested();
result.SourceDirCount++;
await IngestCoreAsync(syncConfig, dir, Path.Combine(target, new DirectoryInfo(dir).Name), result);
}
foreach (var file in Directory.EnumerateFiles(src, "*", SearchOption.TopDirectoryOnly))
{
_cancellationToken.ThrowIfCancellationRequested();
result.SourceFileCount++;
var fileName = Path.GetFileName(file);
if (_options.ExcludePatterns.Value != null && _options.ExcludePatterns.Value.Any())
{
var matched = false;
foreach (var excludePattern in _options.ExcludePatterns.Value)
{
if (excludePattern.IsMatch(fileName))
{
matched = true;
break;
}
}
if (matched)
{
continue;
}
}
if (_options.IncludePatterns.Value != null && _options.IncludePatterns.Value.Any())
{
var matched = false;
foreach (var includePattern in _options.IncludePatterns.Value)
{
if (includePattern.IsMatch(fileName))
{
matched = true;
break;
}
}
if (!matched)
{
continue;
}
}
var fileInfo = new FileInfo(file);
if (fileInfo.LastWriteTime < _options.NewerModifyDateTime)
{
continue;
}
var targetFilePath = Path.Combine(target, fileName);
var syncInfo = new SyncFileInfo(syncConfig, file, targetFilePath);
if (!File.Exists(targetFilePath))
{
_syncQueue.Enqueue(syncInfo);
}
else
{
if (_options.Force)
{
syncInfo.Overwrite = true;
_syncQueue.Enqueue(syncInfo);
}
else if (_options.Strict)
{
var exactlySame =
await FileCompareUtil.ExactlySameAsync(file, targetFilePath, _cancellationToken);
if (!exactlySame)
{
syncInfo.Overwrite = true;
_syncQueue.Enqueue(syncInfo);
}
}
}
}
}
