private void AddDirectoryContents(string path)
{
try
{
var scanDirectory = new ScanDirectory(_sender._logger, _sender._excludeList,
false, _cancellationTokenSource.Token);
foreach (var srcEntry in scanDirectory.ScanPath(_sender._srcPath, path))
{
_sender.AddChange(FsSenderChange.CreateChange(srcEntry.Path));
}
}
catch (OperationCanceledException)
{
}
}
private void OnWatcherRenamed(object source, RenamedEventArgs e)
{
var path = GetPath(e.FullPath);
var oldPath = GetPath(e.OldFullPath);
// ignore event for srcPath (don't know why it occurs rarely)
if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(oldPath))
{
return;
}
if (_gitIsBusy && oldPath == GitIndexLockFilename)
{
SetGitIsBusy(false);
}
// is new file excluded?
if (_excludeList.IsMatch(path))
{
// old file is not excluded -> delete it
if (!_excludeList.IsMatch(oldPath))
{
AddChange(FsSenderChange.CreateRemove(oldPath));
}
// both files are excluded -> do nothing
}
else // new file is not excluded
{
// old file is excluded -> send change with withSubdirectories
if (_excludeList.IsMatch(oldPath))
{
AddChange(FsSenderChange.CreateChange(path), withSubdirectories: true);
}
else
{
// both files are not excluded -> send rename
AddChange(FsSenderChange.CreateRename(path, oldPath));
}
}
}
private void OnWatcherChanged(object source, FileSystemEventArgs e)
{
var path = GetPath(e.FullPath);
// ignore event for srcPath (don't know why it occurs rarely)
if (string.IsNullOrEmpty(path))
{
return;
}
if (!_gitIsBusy && e.ChangeType == WatcherChangeTypes.Created && path == GitIndexLockFilename)
{
SetGitIsBusy(true);
}
if (_excludeList.IsMatch(path))
{
return;
}
AddChange(FsSenderChange.CreateChange(path), withSubdirectories: e.ChangeType == WatcherChangeTypes.Created);
}
private void OnWatcherDeleted(object source, FileSystemEventArgs e)
{
var path = GetPath(e.FullPath);
// ignore event for srcPath (don't know why it occurs rarely)
if (string.IsNullOrEmpty(path))
{
return;
}
if (_gitIsBusy && path == GitIndexLockFilename)
{
SetGitIsBusy(false);
}
if (_excludeList.IsMatch(path))
{
return;
}
AddChange(FsSenderChange.CreateRemove(path));
}
private void AddChange(FsSenderChange fsSenderChange, bool notifyHasWork = true, bool withSubdirectories = false)
{
// ignore empty path
if (string.IsNullOrEmpty(fsSenderChange.Path))
{
return;
}
lock (_changes)
{
if (_changes.TryGetValue(fsSenderChange.Path, out var oldFsChange))
{
oldFsChange.Expired = true;
}
if (fsSenderChange.ChangeType == FsChangeType.Rename &&
_changes.TryGetValue(fsSenderChange.OldPath, out var oldPathFsChange))
{
// rename -> delete old, create new
oldPathFsChange.Expired = true;
_changes[fsSenderChange.OldPath] = FsSenderChange.CreateRemove(fsSenderChange.OldPath);
fsSenderChange = FsSenderChange.CreateChange(fsSenderChange.Path);
}
_changes[fsSenderChange.Path] = fsSenderChange;
}
if (withSubdirectories)
{
_pathScanner.Add(fsSenderChange.Path);
}
if (notifyHasWork)
{
UpdateHasWork();
}
}
private bool SendChanges()
{
// fetch changes
lock (_changes)
{
if (_changes.Count == 0)
{
return(true);
}
// filter not ready changes
_applyRequest.SetChanges(_changes.Values.Where(x => x.IsReady));
}
// nothing to send -> has not ready changes
if (!_applyRequest.HasChanges)
{
// waiting for change is getting ready or we get new ready changes
Thread.Sleep(FsSenderChange.WaitForReadyTimeoutMs);
return(true);
}
if (!_isSending)
{
_logger.Log("Sending");
_isSending = true;
}
var sw = SlimStopwatch.StartNew();
var response = _agentStarter.SendCommand <ApplyResponse>(_applyRequest);
var responseResult = response.Result.ToDictionary(x => x.Key, y => y);
bool hasErrors = false;
// process sent changes
lock (_changes)
{
foreach (var fsChange in _applyRequest.SentChanges)
{
if (!fsChange.Expired)
{
_changes.Remove(fsChange.Path);
if (responseResult.TryGetValue(fsChange.Path, out var fsChangeResult))
{
if (fsChangeResult.ResultCode != FsChangeResultCode.Ok)
{
var withSubdirectories = false;
// ignore sender errors: just resend
if (fsChangeResult.ResultCode != FsChangeResultCode.SenderError)
{
// if rename failed -> send change with withSubdirectories
if (fsChange.ChangeType == FsChangeType.Rename)
{
withSubdirectories = true;
}
else
{
hasErrors = true;
_logger.Log(
$"Change apply error {fsChange.ChangeType} {fsChange.Path}: {fsChangeResult.ErrorMessage ?? "-"}", LogLevel.Error);
}
}
AddChange(FsSenderChange.CreateChange(fsChange.Path), false, withSubdirectories);
}
}
}
else
{
// remove expired
if (_changes.TryGetValue(fsChange.Path, out var oldFsChange) && oldFsChange.Expired)
{
_changes.Remove(fsChange.Path);
}
}
}
}
_sentReporter.Report(_applyRequest.SentChanges, _applyRequest.SentChangesSize, sw.Elapsed);
_applyRequest.ClearChanges();
UpdateHasWork();
return(!hasErrors);
}
private void Scan()
{
var sw = SlimStopwatch.StartNew();
List <FsEntry> srcList = null;
Dictionary <string, FsEntry> destList;
/*
* Start agent before scan source
*
* Old timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ................................. Finish
*
* New timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ....................... Finish
*
* A failed start could cause unnecessary scanning source in old timeline.
* No need to scan source before start in most cases because it is about as fast as the scan destination.
*/
using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token))
{
var cancellationToken = tokenSource.Token;
cancellationToken.ThrowIfCancellationRequested();
_agentStarter.Start();
// scan source
var task = Task.Run(() =>
{
try
{
var swScanSource = SlimStopwatch.StartNew();
var scanDirectory =
new ScanDirectory(_logger, _excludeList, cancellationToken: cancellationToken);
srcList = scanDirectory.ScanPath(_srcPath).ToList();
cancellationToken.ThrowIfCancellationRequested();
_logger.Log($"Scanned source {srcList.Count} items in {swScanSource.ElapsedMilliseconds} ms");
}
catch (OperationCanceledException)
{
srcList = null;
}
}, cancellationToken);
try
{
var swScanDestination = SlimStopwatch.StartNew();
// scan destination
var response = _agentStarter.SendCommand <ScanResponse>(new ScanRequest(_logger));
destList = response.FileList.ToDictionary(x => x.Path, y => y);
_logger.Log(
$"Scanned destination {destList.Count} items in {swScanDestination.ElapsedMilliseconds} ms");
task.Wait(cancellationToken);
}
catch (Exception)
{
tokenSource.Cancel();
throw;
}
}
// During scan, changes could come from file system events or from PathScanner, we should not overwrite them.
var itemsCount = 0;
long changesSize = 0;
lock (_changes)
{
foreach (var srcEntry in srcList)
{
if (!destList.TryGetValue(srcEntry.Path, out var destEntry))
{
destEntry = FsEntry.Empty;
}
// Skip changed srcEntry
if (!_changes.ContainsKey(srcEntry.Path))
{
// add to changes (no replace)
if (!srcEntry.Equals(destEntry))
{
itemsCount++;
if (!srcEntry.IsDirectory)
{
changesSize += srcEntry.Length;
}
AddChange(FsSenderChange.CreateChange(srcEntry), false);
}
}
if (!destEntry.IsEmpty)
{
destList.Remove(destEntry.Path);
}
}
// add deletes
foreach (var destEntry in destList.Values)
{
// Skip changed destEntry
if (!_changes.ContainsKey(destEntry.Path))
{
itemsCount++;
AddChange(FsSenderChange.CreateRemove(destEntry.Path), false);
}
}
}
_needToScan = false;
_logger.Log(
$"Scanned in {sw.ElapsedMilliseconds} ms, {itemsCount} items, {PrettySize(changesSize)} to send");
UpdateHasWork();
}
public bool WriteFsChangeBody(string filename, FsSenderChange fsSenderChange)
{
long written = 0;
FileStream fs = null;
try
{
try
{
fs = new FileStream(filename, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite);
}
catch (Exception ex)
{
if (ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
// file vanished
fsSenderChange.Vanished = true;
}
else
{
// something else
_logger.Log(ex.Message, LogLevel.Error);
}
// cannot read file (sender error)
WriteFsChange(fsSenderChange);
WriteInt(-1);
return(false);
}
// length resolved
fsSenderChange.Length = fs.Length;
WriteFsChange(fsSenderChange);
fsSenderChange.Opened = true;
int read;
do
{
if (fsSenderChange.Expired)
{
// file change is expired -> stop
WriteInt(-1);
return(false);
}
try
{
read = fs.Read(_buffer, 0, BufferLength);
if (read <= 0)
{
break;
}
}
catch (Exception ex)
{
// file read error (sender error)
_logger.Log(ex.Message, LogLevel.Error);
WriteInt(-1);
return(false);
}
WriteInt(read);
BinaryWriter.Write(_buffer, 0, read);
written += read;
} while (read == BufferLength);
// check written
if (written != fsSenderChange.Length)
{
// file length mismatch
WriteInt(-1);
return(false);
}
WriteInt(0);
return(true);
}
finally
{
fs?.Dispose();
}
}
private void AddChange(FsSenderChange fsSenderChange, bool notifyHasWork = true, bool withSubdirectories = false)
{
if (_logger.IsDebug)
{
_logger.Log($"AddChange {fsSenderChange}", LogLevel.Debug);
}
// ignore empty path
if (string.IsNullOrEmpty(fsSenderChange.Path))
{
return;
}
lock (_changes)
{
if (_changes.TryGetValue(fsSenderChange.Path, out var oldFsChange))
{
oldFsChange.Expired = true;
_changes.Remove(oldFsChange.Path);
// Rename + Change -> Combine
if (fsSenderChange.IsChange && oldFsChange.IsRename)
{
fsSenderChange.Combine(oldFsChange);
}
}
_changes.Add(fsSenderChange.Path, fsSenderChange);
if (fsSenderChange.IsRename)
{
// replace changes inside OldPath to Path
var oldPathPrefix = fsSenderChange.OldPath + "/";
var insideOldPathFsChanges = _changes.Values
.Where(fsChange => !fsChange.Opened &&
(((fsChange.IsChange || fsChange.IsRename) && fsChange.Path.StartsWith(oldPathPrefix)) ||
(fsChange.IsRename && fsChange.OldPath.StartsWith(oldPathPrefix)))
)
.ToList();
var index = oldPathPrefix.Length - 1;
foreach (var insideOldPathFsChange in insideOldPathFsChanges)
{
insideOldPathFsChange.Expired = true;
_changes.Remove(insideOldPathFsChange.Path);
// update Path
var newPath = insideOldPathFsChange.Path.StartsWith(oldPathPrefix)
? string.Concat(fsSenderChange.Path, insideOldPathFsChange.Path.AsSpan(index))
: insideOldPathFsChange.Path;
var newChange = FsSenderChange.CreateWithPath(insideOldPathFsChange, newPath);
// update OldPath
if (newChange.IsRename && newChange.OldPath.StartsWith(oldPathPrefix))
{
newChange.OldPath = string.Concat(fsSenderChange.Path, newChange.OldPath.AsSpan(index));
}
AddChange(newChange);
}
// expire and combine change with OldPath
if (_changes.TryGetValue(fsSenderChange.OldPath, out var oldPathFsChange) &&
!oldPathFsChange.Opened && oldPathFsChange.IsChange)
{
oldPathFsChange.Expired = true;
// Change + Rename -> Combine
fsSenderChange.Combine(oldPathFsChange);
}
}
}
if (withSubdirectories)
{
_pathScanner.Add(fsSenderChange.Path);
}
if (notifyHasWork)
{
UpdateHasWork();
}
}
