public FsChange ReadFsChange()
{
var changeType = (FsChangeType)ReadByte();
if (changeType == FsChangeType.EmptyMarker)
{
return(FsChange.Empty);
}
var fsChange = new FsChange(changeType, ReadString());
switch (fsChange.ChangeType)
{
case FsChangeType.Change:
fsChange.Length = ReadLong();
fsChange.IsDirectory = fsChange.Length == -1;
fsChange.LastWriteTime = ReadDateTime();
break;
case FsChangeType.Rename:
fsChange.OldPath = ReadString();
break;
}
return(fsChange);
}
// read and apply changes
public IEnumerable <FsChangeResult> ReadAndApplyChanges(FileMaskList excludeList)
{
while (true)
{
var fsChange = Reader.ReadFsChange();
if (fsChange.IsEmpty)
{
break;
}
var fsChangeResult = ApplyFsChange(fsChange, excludeList);
if (fsChangeResult.ResultCode != FsChangeResultCode.Ok && fsChange.ChangeType == FsChangeType.Change)
{
var path = Path.Combine(BasePath, fsChange.Path);
if (fsChange.IsDirectory && File.Exists(path) ||
!fsChange.IsDirectory && Directory.Exists(path))
{
// delete and retry
var fsRemoveChangeResult = ApplyFsChange(
FsChange.CreateRemove(fsChange.Path),
excludeList);
if (fsRemoveChangeResult.ResultCode == FsChangeResultCode.Ok)
{
fsChangeResult = ApplyFsChange(fsChange, excludeList);
}
}
}
// TODO: skip ok codes (sender do not use them at the moment)
if (fsChangeResult.ResultCode != FsChangeResultCode.Ok)
{
yield return(fsChangeResult);
}
}
}
private void AddDirectoryContents(string path)
{
try
{
var scanDirectory = new ScanDirectory(_sender._logger, _sender._excludeList,
false, _cancellationTokenSource.Token);
foreach (var srcEntry in scanDirectory.ScanPath(_sender._srcPath, path))
{
_sender.AddChange(FsChange.CreateChange(srcEntry.Path));
}
}
catch (OperationCanceledException)
{
}
}
public void WriteFsChange(FsChange fsChange)
{
WriteByte((byte)fsChange.ChangeType);
if (fsChange.IsEmpty)
{
return;
}
WriteString(fsChange.Path);
if (fsChange.IsChange)
{
WriteLong(fsChange.IsDirectory ? -1 : fsChange.Length);
WriteDateTime(fsChange.LastWriteTime);
}
if (fsChange.IsRename)
{
WriteString(fsChange.OldPath);
}
}
private void OnWatcherRenamed(object source, RenamedEventArgs e)
{
// ignore event for srcPath (don't know why it occurs rarely)
if (e.FullPath == _srcPath || e.OldFullPath == _srcPath)
{
return;
}
var path = GetPath(e.FullPath);
var oldPath = GetPath(e.OldFullPath);
if (_gitIsBusy && oldPath == GIT_INDEX_LOCK_FILENAME)
{
SetGitIsBusy(false);
}
// is new file excluded?
if (_excludeList.IsMatch(path))
{
// old file is not excluded -> delete it
if (!_excludeList.IsMatch(oldPath))
{
AddChange(FsChange.CreateRemove(oldPath));
}
// both files are excluded -> do nothing
}
else // new file is not excluded
{
// old file is excluded -> send change with withSubdirectories
if (_excludeList.IsMatch(oldPath))
{
AddChange(FsChange.CreateChange(path), withSubdirectories: true);
}
else
{
// both files are not excluded -> send rename
AddChange(FsChange.CreateRename(path, oldPath));
}
}
}
private void OnWatcherChanged(object source, FileSystemEventArgs e)
{
// ignore event for srcPath (don't know why it occurs rarely)
if (e.FullPath == _srcPath)
{
return;
}
var path = GetPath(e.FullPath);
if (!_gitIsBusy && e.ChangeType == WatcherChangeTypes.Created && path == GIT_INDEX_LOCK_FILENAME)
{
SetGitIsBusy(true);
}
if (_excludeList.IsMatch(path))
{
return;
}
AddChange(FsChange.CreateChange(path), withSubdirectories: e.ChangeType == WatcherChangeTypes.Created);
}
private void AddChange(FsChange fsChange, bool notifyHasWork = true, bool withSubdirectories = false)
{
lock (_changes)
{
if (_changes.TryGetValue(fsChange.Path, out var oldFsChange))
{
oldFsChange.Expired = true;
}
_changes[fsChange.Path] = fsChange;
}
if (withSubdirectories)
{
_pathScanner.Add(fsChange.Path);
}
if (notifyHasWork)
{
UpdateHasWork();
}
}
private void OnWatcherDeleted(object source, FileSystemEventArgs e)
{
// ignore event for srcPath (don't know why it occurs rarely)
if (e.FullPath == _srcPath)
{
return;
}
var path = GetPath(e.FullPath);
if (_gitIsBusy && path == GIT_INDEX_LOCK_FILENAME)
{
SetGitIsBusy(false);
}
if (_excludeList.IsMatch(path))
{
return;
}
AddChange(FsChange.CreateRemove(path));
}
public bool ReadFsChangeBody(string path, FsChange fsChange)
{
var shebangPosition = 0;
var makeExecutable = false;
string tempPath = null;
FileStream fs = null;
bool bodyReadSuccess = false;
try
{
long written = 0;
int chunkSize;
do
{
chunkSize = ReadInt();
if (chunkSize < 0)
{
// error occurred in sender
return(false);
}
var remain = chunkSize;
while (remain > 0)
{
var read = BinaryReader.Read(_buffer, 0, Math.Min(BUFFER_LENGTH, remain));
if (read <= 0)
{
throw new EndOfStreamException($"Premature end of stream {remain}, {chunkSize}, {read})");
}
if (written == 0)
{
var directoryName = Path.GetDirectoryName(path);
Directory.CreateDirectory(directoryName);
tempPath = Path.Combine(directoryName,
"." + Path.GetFileName(path) + "." + Path.GetRandomFileName());
fs = new FileStream(tempPath, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
}
if (PlatformHasChmod && shebangPosition < ShebangLength)
{
for (var i = 0; i < read;)
{
if (_buffer[i++] != ShebangBytes[shebangPosition++])
{
// no shebang
shebangPosition = int.MaxValue;
break;
}
if (shebangPosition == ShebangLength)
{
makeExecutable = true;
break;
}
}
}
fs?.Write(_buffer, 0, read);
written += read;
remain -= read;
}
} while (chunkSize > 0);
bodyReadSuccess = written == fsChange.Length;
if (bodyReadSuccess && makeExecutable)
{
// 0755, rwxr-xr-x
fs.ChangeMode(0b111_101_101);
}
// create empty file
if (bodyReadSuccess && fsChange.Length == 0)
{
var directoryName = Path.GetDirectoryName(path);
Directory.CreateDirectory(directoryName);
using (new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read))
{
}
File.SetLastWriteTime(path, fsChange.LastWriteTime);
}
return(bodyReadSuccess);
}
catch (Exception)
{
SkipFsChangeBody();
throw;
}
finally
{
if (fs != null)
{
fs.Dispose();
if (bodyReadSuccess)
{
try
{
File.SetLastWriteTime(tempPath, fsChange.LastWriteTime);
File.Move(tempPath, path, true);
}
catch (Exception)
{
FsHelper.TryDeleteFile(tempPath);
throw;
}
}
else
{
FsHelper.TryDeleteFile(tempPath);
}
}
}
}
public bool WriteFsChangeBody(string filename, FsChange fsChange)
{
long written = 0;
FileStream fs = null;
try
{
try
{
fs = new FileStream(filename, FileMode.Open, FileAccess.Read,
FileShare.Delete | FileShare.ReadWrite);
}
catch (Exception ex)
{
if (ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
// file vanished -> ignore it
fsChange.Expired = true;
}
else
{
// something else
_logger.Log(ex.Message, LogLevel.Error);
}
// cannot read file (sender error)
WriteFsChange(fsChange);
WriteInt(-1);
return(false);
}
// length resolved
fsChange.Length = fs.Length;
WriteFsChange(fsChange);
int read;
do
{
if (fsChange.Expired)
{
// file change is expired -> stop
WriteInt(-1);
return(false);
}
try
{
read = fs.Read(_buffer, 0, BUFFER_LENGTH);
if (read <= 0)
{
break;
}
}
catch (Exception ex)
{
// file read error (sender error)
_logger.Log(ex.Message, LogLevel.Error);
WriteInt(-1);
return(false);
}
WriteInt(read);
BinaryWriter.Write(_buffer, 0, read);
written += read;
} while (read == BUFFER_LENGTH);
// check written
if (written != fsChange.Length)
{
// file length mismatch
WriteInt(-1);
return(false);
}
WriteInt(0);
return(true);
}
finally
{
fs?.Dispose();
}
}
private FsChangeResult ApplyFsChange(FsChange fsChange, FileMaskList excludeList)
{
var path = Path.Combine(BasePath, fsChange.Path);
FsChangeResultCode resultCode = FsChangeResultCode.None;
string error = null;
// Change file | Remove | Rename + Change directory
if (fsChange.IsChange && !fsChange.IsDirectory)
{
try
{
if (fsChange.HasBody && Reader.ReadFsChangeBody(path, fsChange))
{
resultCode = FsChangeResultCode.Ok;
}
else
{
// error occurred in sender
error = "Sender error";
resultCode = FsChangeResultCode.SenderError;
}
}
catch (EndOfStreamException)
{
// end of data
throw;
}
catch (Exception ex)
{
resultCode = FsChangeResultCode.Error;
error = ex.Message;
}
}
else if (fsChange.IsRemove)
{
// directory
if (Directory.Exists(path))
{
var scanDirectory = new ScanDirectory(Logger, excludeList, false);
Exception exception = null;
foreach (var fsEntry in scanDirectory.ScanPath(BasePath, fsChange.Path))
{
var fsEntryPath = Path.Combine(BasePath, fsEntry.Path);
try
{
if (fsEntry.IsDirectory)
{
Directory.Delete(fsEntryPath, false);
}
else
{
File.Delete(fsEntryPath);
}
}
catch (Exception ex)
{
exception ??= ex;
Logger.Log($"Error deleting {fsEntryPath}: {ex.Message}", LogLevel.Warning);
}
}
try
{
Directory.Delete(path, false);
}
catch (Exception ex)
{
exception ??= ex;
Logger.Log($"Error deleting {path}: {ex.Message}", LogLevel.Warning);
}
if (exception == null)
{
resultCode = FsChangeResultCode.Ok;
}
else
{
// scan directory see any file -> error (handle excludes)
if (scanDirectory.ScanPath(BasePath, fsChange.Path).Any(x => !x.IsDirectory))
{
resultCode = FsChangeResultCode.Error;
error = exception.Message;
}
else
{
resultCode = FsChangeResultCode.Ok;
}
}
}
else if (File.Exists(path))
{
try
{
File.Delete(path);
resultCode = FsChangeResultCode.Ok;
}
catch (Exception ex)
{
resultCode = FsChangeResultCode.Error;
error = ex.Message;
}
}
else
{
resultCode = FsChangeResultCode.Ok;
}
}
else
{
if (fsChange.IsChange && fsChange.IsDirectory)
{
try
{
var directoryPath = fsChange.IsRename ? Path.Combine(BasePath, fsChange.OldPath) : path;
Directory.CreateDirectory(directoryPath);
Directory.SetLastWriteTime(directoryPath, fsChange.LastWriteTime);
resultCode = FsChangeResultCode.Ok;
}
catch (Exception ex)
{
error = ex.Message;
resultCode = FsChangeResultCode.Error;
}
}
if (resultCode != FsChangeResultCode.Error && fsChange.IsRename)
{
try
{
var oldPath = Path.Combine(BasePath, fsChange.OldPath);
if (Directory.Exists(oldPath))
{
Directory.Move(oldPath, path);
}
else
{
File.Move(oldPath, path, true);
}
resultCode = FsChangeResultCode.Ok;
}
catch (Exception ex)
{
error = ex.Message;
resultCode = FsChangeResultCode.Error;
}
}
}
if (resultCode == FsChangeResultCode.None)
{
resultCode = FsChangeResultCode.Error;
error = "Unknown change type";
}
return(new FsChangeResult
{
ChangeType = fsChange.ChangeType,
Path = fsChange.Path,
ResultCode = resultCode,
ErrorMessage = error
});
}
private bool SendChanges()
{
// fetch changes
lock (_changes)
{
if (_changes.Count == 0)
{
return(true);
}
_applyRequest.SetChanges(_changes.Values);
}
if (!_isSending)
{
_logger.Log("Sending");
_isSending = true;
}
var sw = Stopwatch.StartNew();
var response = _agentStarter.SendCommand <ApplyResponse>(_applyRequest);
var responseResult = response.Result.ToDictionary(x => x.Key, y => y);
bool hasErrors = false;
// process sent changes
lock (_changes)
{
foreach (var fsChange in _applyRequest.SentChanges)
{
if (!fsChange.Expired)
{
_changes.Remove(fsChange.Path);
if (responseResult.TryGetValue(fsChange.Path, out var fsChangeResult))
{
if (fsChangeResult.ResultCode != FsChangeResultCode.Ok)
{
var withSubdirectories = false;
// ignore sender errors: just resend
if (fsChangeResult.ResultCode != FsChangeResultCode.SenderError)
{
// if rename failed -> send change with withSubdirectories
if (fsChange.ChangeType == FsChangeType.Rename)
{
withSubdirectories = true;
}
else
{
hasErrors = true;
_logger.Log(
$"Change apply error {fsChange.ChangeType} {fsChange.Path}: {fsChangeResult.ErrorMessage ?? "-"}", LogLevel.Error);
}
}
AddChange(FsChange.CreateChange(fsChange.Path), false, withSubdirectories);
}
}
}
else
{
// remove expired
if (_changes.TryGetValue(fsChange.Path, out var oldFsChange) && oldFsChange.Expired)
{
_changes.Remove(fsChange.Path);
}
}
}
}
_sentReporter.Report(_applyRequest.SentChanges, _applyRequest.SentChangesSize, sw.Elapsed);
_applyRequest.ClearChanges();
UpdateHasWork();
return(!hasErrors);
}
private void Scan()
{
var sw = Stopwatch.StartNew();
List <FsEntry> srcList = null;
Dictionary <string, FsEntry> destList;
/*
* Start agent before scan source
*
* Old timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ................................. Finish
*
* New timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ....................... Finish
*
* A failed start could cause unnecessary scanning source in old timeline.
* No need to scan source before start in most cases because it is about as fast as the scan destination.
*/
_agentStarter.Start();
using (var tokenSource = new CancellationTokenSource())
{
var cancellationToken = tokenSource.Token;
// scan source
var task = Task.Run(() =>
{
try
{
var swScanSource = Stopwatch.StartNew();
var scanDirectory =
new ScanDirectory(_logger, _excludeList, cancellationToken: cancellationToken);
srcList = scanDirectory.ScanPath(_srcPath).ToList();
cancellationToken.ThrowIfCancellationRequested();
_logger.Log($"Scanned source {srcList.Count} items in {swScanSource.ElapsedMilliseconds} ms");
}
catch (OperationCanceledException)
{
srcList = null;
}
}, cancellationToken);
try
{
var swScanDestination = Stopwatch.StartNew();
// scan destination
var response = _agentStarter.SendCommand <ScanResponse>(new ScanRequest(_logger));
destList = response.FileList.ToDictionary(x => x.Path, y => y);
_logger.Log($"Scanned destination {destList.Count} items in {swScanDestination.ElapsedMilliseconds} ms");
task.Wait(cancellationToken);
}
catch (Exception)
{
tokenSource.Cancel();
throw;
}
}
// During scan, changes could come from file system events or from PathScanner, we should not overwrite them.
var itemsCount = 0;
long changesSize = 0;
lock (_changes)
{
foreach (var srcEntry in srcList)
{
if (!destList.TryGetValue(srcEntry.Path, out var destEntry))
{
destEntry = FsEntry.Empty;
}
// Skip changed srcEntry
if (!_changes.ContainsKey(srcEntry.Path))
{
// add to changes (no replace)
if (!srcEntry.Compare(destEntry))
{
itemsCount++;
if (!srcEntry.IsDirectory)
{
changesSize += srcEntry.Length;
}
AddChange(FsChange.CreateChange(srcEntry), false);
}
}
if (!destEntry.IsEmpty)
{
destList.Remove(destEntry.Path);
}
}
// add deletes
foreach (var destEntry in destList.Values)
{
// Skip changed destEntry
if (!_changes.ContainsKey(destEntry.Path))
{
itemsCount++;
AddChange(FsChange.CreateRemove(destEntry.Path), false);
}
}
}
_needToScan = false;
_logger.Log(
$"Scanned in {sw.ElapsedMilliseconds} ms, {itemsCount} items, {PrettySize(changesSize)} to send");
UpdateHasWork();
}
