예제 #1
0
            public ChunkWriteStreamFlusher(Stream stream, ICompression compression)
            {
                _stream         = stream;
                _compression    = compression;
                _flushStopwatch = SlimStopwatch.Create();

                if (_compression != null)
                {
                    _chunkCompressedBytes = new byte[ChunkSize + LengthSize];
                }
            }
예제 #2
0
        private void WaitForWork()
        {
            const int readyTimeout = 300;
            var       waitForReady = true;
            var       sw           = SlimStopwatch.StartNew();

            while (!HasWork)
            {
                var timeout    = Timeout.Infinite;
                var waitForGit = false;
                if (waitForReady)
                {
                    var elapsed = sw.ElapsedMilliseconds;
                    // no work for some time and git is not busy
                    if (elapsed >= readyTimeout)
                    {
                        if (!_gitIsBusy)
                        {
                            _sentReporter.Flush();
                            _logger.Log("Ready");
                            waitForReady = false;
                            _isSending   = false;
                        }
                        else
                        {
                            waitForGit = true;
                        }
                    }
                    else
                    {
                        timeout = readyTimeout - (int)elapsed;
                    }
                }

                if (waitForGit)
                {
                    WaitHandle.WaitAny(new WaitHandle[] { _hasWorkEvent, _gitIsReadyEvent }, timeout);
                }
                else
                {
                    _hasWorkEvent.WaitOne(timeout);
                }
            }
        }
예제 #3
0
 public SentReporter(ILogger logger)
 {
     _logger    = logger;
     _stopwatch = SlimStopwatch.Create();
     _timeSpan  = TimeSpan.Zero;
 }
예제 #4
0
        private bool SendChanges()
        {
            // fetch changes
            lock (_changes)
            {
                if (_changes.Count == 0)
                {
                    return(true);
                }

                // filter not ready changes
                _applyRequest.SetChanges(_changes.Values.Where(x => x.IsReady));
            }

            // nothing to send -> has not ready changes
            if (!_applyRequest.HasChanges)
            {
                // waiting for change is getting ready or we get new ready changes
                Thread.Sleep(FsSenderChange.WaitForReadyTimeoutMs);
                return(true);
            }

            if (!_isSending)
            {
                _logger.Log("Sending");
                _isSending = true;
            }

            var sw             = SlimStopwatch.StartNew();
            var response       = _agentStarter.SendCommand <ApplyResponse>(_applyRequest);
            var responseResult = response.Result.ToDictionary(x => x.Key, y => y);

            bool hasErrors = false;

            // process sent changes
            lock (_changes)
            {
                foreach (var fsChange in _applyRequest.SentChanges)
                {
                    if (!fsChange.Expired)
                    {
                        _changes.Remove(fsChange.Path);
                        if (responseResult.TryGetValue(fsChange.Path, out var fsChangeResult))
                        {
                            if (fsChangeResult.ResultCode != FsChangeResultCode.Ok)
                            {
                                var withSubdirectories = false;
                                // ignore sender errors: just resend
                                if (fsChangeResult.ResultCode != FsChangeResultCode.SenderError)
                                {
                                    // if rename failed -> send change with withSubdirectories
                                    if (fsChange.ChangeType == FsChangeType.Rename)
                                    {
                                        withSubdirectories = true;
                                    }
                                    else
                                    {
                                        hasErrors = true;
                                        _logger.Log(
                                            $"Change apply error {fsChange.ChangeType} {fsChange.Path}: {fsChangeResult.ErrorMessage ?? "-"}", LogLevel.Error);
                                    }
                                }
                                AddChange(FsSenderChange.CreateChange(fsChange.Path), false, withSubdirectories);
                            }
                        }
                    }
                    else
                    {
                        // remove expired
                        if (_changes.TryGetValue(fsChange.Path, out var oldFsChange) && oldFsChange.Expired)
                        {
                            _changes.Remove(fsChange.Path);
                        }
                    }
                }
            }

            _sentReporter.Report(_applyRequest.SentChanges, _applyRequest.SentChangesSize, sw.Elapsed);
            _applyRequest.ClearChanges();
            UpdateHasWork();
            return(!hasErrors);
        }
예제 #5
0
        private void Scan()
        {
            var            sw      = SlimStopwatch.StartNew();
            List <FsEntry> srcList = null;
            Dictionary <string, FsEntry> destList;

            /*
             * Start agent before scan source
             *
             * Old timeline: [Main thread]       Start ... Initialize ... Scan destination ... Finish
             *               [Secondary thread]  Scan source ................................. Finish
             *
             * New timeline: [Main thread]       Start ... Initialize ... Scan destination ... Finish
             *               [Secondary thread]            Scan source ....................... Finish
             *
             * A failed start could cause unnecessary scanning source in old timeline.
             * No need to scan source before start in most cases because it is about as fast as the scan destination.
             */
            using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token))
            {
                var cancellationToken = tokenSource.Token;
                cancellationToken.ThrowIfCancellationRequested();

                _agentStarter.Start();

                // scan source
                var task = Task.Run(() =>
                {
                    try
                    {
                        var swScanSource  = SlimStopwatch.StartNew();
                        var scanDirectory =
                            new ScanDirectory(_logger, _excludeList, cancellationToken: cancellationToken);
                        srcList = scanDirectory.ScanPath(_srcPath).ToList();
                        cancellationToken.ThrowIfCancellationRequested();
                        _logger.Log($"Scanned source {srcList.Count} items in {swScanSource.ElapsedMilliseconds} ms");
                    }
                    catch (OperationCanceledException)
                    {
                        srcList = null;
                    }
                }, cancellationToken);

                try
                {
                    var swScanDestination = SlimStopwatch.StartNew();
                    // scan destination
                    var response = _agentStarter.SendCommand <ScanResponse>(new ScanRequest(_logger));
                    destList = response.FileList.ToDictionary(x => x.Path, y => y);
                    _logger.Log(
                        $"Scanned destination {destList.Count} items in {swScanDestination.ElapsedMilliseconds} ms");
                    task.Wait(cancellationToken);
                }
                catch (Exception)
                {
                    tokenSource.Cancel();
                    throw;
                }
            }

            // During scan, changes could come from file system events or from PathScanner, we should not overwrite them.
            var  itemsCount  = 0;
            long changesSize = 0;

            lock (_changes)
            {
                foreach (var srcEntry in srcList)
                {
                    if (!destList.TryGetValue(srcEntry.Path, out var destEntry))
                    {
                        destEntry = FsEntry.Empty;
                    }

                    // Skip changed srcEntry
                    if (!_changes.ContainsKey(srcEntry.Path))
                    {
                        // add to changes (no replace)
                        if (!srcEntry.Equals(destEntry))
                        {
                            itemsCount++;
                            if (!srcEntry.IsDirectory)
                            {
                                changesSize += srcEntry.Length;
                            }
                            AddChange(FsSenderChange.CreateChange(srcEntry), false);
                        }
                    }

                    if (!destEntry.IsEmpty)
                    {
                        destList.Remove(destEntry.Path);
                    }
                }

                // add deletes
                foreach (var destEntry in destList.Values)
                {
                    // Skip changed destEntry
                    if (!_changes.ContainsKey(destEntry.Path))
                    {
                        itemsCount++;
                        AddChange(FsSenderChange.CreateRemove(destEntry.Path), false);
                    }
                }
            }

            _needToScan = false;
            _logger.Log(
                $"Scanned in {sw.ElapsedMilliseconds} ms, {itemsCount} items, {PrettySize(changesSize)} to send");
            UpdateHasWork();
        }