public ChunkWriteStreamFlusher(Stream stream, ICompression compression)
{
_stream = stream;
_compression = compression;
_flushStopwatch = SlimStopwatch.Create();
if (_compression != null)
{
_chunkCompressedBytes = new byte[ChunkSize + LengthSize];
}
}
private void WaitForWork()
{
const int readyTimeout = 300;
var waitForReady = true;
var sw = SlimStopwatch.StartNew();
while (!HasWork)
{
var timeout = Timeout.Infinite;
var waitForGit = false;
if (waitForReady)
{
var elapsed = sw.ElapsedMilliseconds;
// no work for some time and git is not busy
if (elapsed >= readyTimeout)
{
if (!_gitIsBusy)
{
_sentReporter.Flush();
_logger.Log("Ready");
waitForReady = false;
_isSending = false;
}
else
{
waitForGit = true;
}
}
else
{
timeout = readyTimeout - (int)elapsed;
}
}
if (waitForGit)
{
WaitHandle.WaitAny(new WaitHandle[] { _hasWorkEvent, _gitIsReadyEvent }, timeout);
}
else
{
_hasWorkEvent.WaitOne(timeout);
}
}
}
public SentReporter(ILogger logger)
{
_logger = logger;
_stopwatch = SlimStopwatch.Create();
_timeSpan = TimeSpan.Zero;
}
private bool SendChanges()
{
// fetch changes
lock (_changes)
{
if (_changes.Count == 0)
{
return(true);
}
// filter not ready changes
_applyRequest.SetChanges(_changes.Values.Where(x => x.IsReady));
}
// nothing to send -> has not ready changes
if (!_applyRequest.HasChanges)
{
// waiting for change is getting ready or we get new ready changes
Thread.Sleep(FsSenderChange.WaitForReadyTimeoutMs);
return(true);
}
if (!_isSending)
{
_logger.Log("Sending");
_isSending = true;
}
var sw = SlimStopwatch.StartNew();
var response = _agentStarter.SendCommand <ApplyResponse>(_applyRequest);
var responseResult = response.Result.ToDictionary(x => x.Key, y => y);
bool hasErrors = false;
// process sent changes
lock (_changes)
{
foreach (var fsChange in _applyRequest.SentChanges)
{
if (!fsChange.Expired)
{
_changes.Remove(fsChange.Path);
if (responseResult.TryGetValue(fsChange.Path, out var fsChangeResult))
{
if (fsChangeResult.ResultCode != FsChangeResultCode.Ok)
{
var withSubdirectories = false;
// ignore sender errors: just resend
if (fsChangeResult.ResultCode != FsChangeResultCode.SenderError)
{
// if rename failed -> send change with withSubdirectories
if (fsChange.ChangeType == FsChangeType.Rename)
{
withSubdirectories = true;
}
else
{
hasErrors = true;
_logger.Log(
$"Change apply error {fsChange.ChangeType} {fsChange.Path}: {fsChangeResult.ErrorMessage ?? "-"}", LogLevel.Error);
}
}
AddChange(FsSenderChange.CreateChange(fsChange.Path), false, withSubdirectories);
}
}
}
else
{
// remove expired
if (_changes.TryGetValue(fsChange.Path, out var oldFsChange) && oldFsChange.Expired)
{
_changes.Remove(fsChange.Path);
}
}
}
}
_sentReporter.Report(_applyRequest.SentChanges, _applyRequest.SentChangesSize, sw.Elapsed);
_applyRequest.ClearChanges();
UpdateHasWork();
return(!hasErrors);
}
private void Scan()
{
var sw = SlimStopwatch.StartNew();
List <FsEntry> srcList = null;
Dictionary <string, FsEntry> destList;
/*
* Start agent before scan source
*
* Old timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ................................. Finish
*
* New timeline: [Main thread] Start ... Initialize ... Scan destination ... Finish
* [Secondary thread] Scan source ....................... Finish
*
* A failed start could cause unnecessary scanning source in old timeline.
* No need to scan source before start in most cases because it is about as fast as the scan destination.
*/
using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token))
{
var cancellationToken = tokenSource.Token;
cancellationToken.ThrowIfCancellationRequested();
_agentStarter.Start();
// scan source
var task = Task.Run(() =>
{
try
{
var swScanSource = SlimStopwatch.StartNew();
var scanDirectory =
new ScanDirectory(_logger, _excludeList, cancellationToken: cancellationToken);
srcList = scanDirectory.ScanPath(_srcPath).ToList();
cancellationToken.ThrowIfCancellationRequested();
_logger.Log($"Scanned source {srcList.Count} items in {swScanSource.ElapsedMilliseconds} ms");
}
catch (OperationCanceledException)
{
srcList = null;
}
}, cancellationToken);
try
{
var swScanDestination = SlimStopwatch.StartNew();
// scan destination
var response = _agentStarter.SendCommand <ScanResponse>(new ScanRequest(_logger));
destList = response.FileList.ToDictionary(x => x.Path, y => y);
_logger.Log(
$"Scanned destination {destList.Count} items in {swScanDestination.ElapsedMilliseconds} ms");
task.Wait(cancellationToken);
}
catch (Exception)
{
tokenSource.Cancel();
throw;
}
}
// During scan, changes could come from file system events or from PathScanner, we should not overwrite them.
var itemsCount = 0;
long changesSize = 0;
lock (_changes)
{
foreach (var srcEntry in srcList)
{
if (!destList.TryGetValue(srcEntry.Path, out var destEntry))
{
destEntry = FsEntry.Empty;
}
// Skip changed srcEntry
if (!_changes.ContainsKey(srcEntry.Path))
{
// add to changes (no replace)
if (!srcEntry.Equals(destEntry))
{
itemsCount++;
if (!srcEntry.IsDirectory)
{
changesSize += srcEntry.Length;
}
AddChange(FsSenderChange.CreateChange(srcEntry), false);
}
}
if (!destEntry.IsEmpty)
{
destList.Remove(destEntry.Path);
}
}
// add deletes
foreach (var destEntry in destList.Values)
{
// Skip changed destEntry
if (!_changes.ContainsKey(destEntry.Path))
{
itemsCount++;
AddChange(FsSenderChange.CreateRemove(destEntry.Path), false);
}
}
}
_needToScan = false;
_logger.Log(
$"Scanned in {sw.ElapsedMilliseconds} ms, {itemsCount} items, {PrettySize(changesSize)} to send");
UpdateHasWork();
}