예제 #1
0
        public void TestReadTimeout()
        {
            var streamMock = new Mock <Stream>();

            streamMock.Setup(stream => stream.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
            .Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
            streamMock.Setup(stream => stream.ReadAsync(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>(),
                                                        It.IsAny <CancellationToken>()))
            .Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
            streamMock.Setup(stream => stream.CanRead)
            .Returns(true);

            var readers = new List <ColumnReader>();
            var sw      = new Stopwatch();

            sw.Start();
            Assert.That(
                () => LogParser.ParseStream(streamMock.Object, "env", "tgt", readers, CreateMetricsDict(), 200,
                                            CancellationToken.None), Throws.TypeOf <StreamStarvationException>());
            sw.Stop();
            Assert.That(sw.Elapsed.TotalSeconds, Is.LessThan(ReadSleepSeconds));
        }
예제 #2
0
        public void TestCancellation()
        {
            var streamMock = new Mock <Stream>();

            streamMock.Setup(stream => stream.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
            .Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
            streamMock.Setup(stream => stream.ReadAsync(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>(),
                                                        It.IsAny <CancellationToken>()))
            .Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
            streamMock.Setup(stream => stream.CanRead)
            .Returns(true);

            var readers     = new List <ColumnReader>();
            var tokenSource = new CancellationTokenSource();
            var task        = Task.Run(
                () => LogParser.ParseStream(streamMock.Object, "env", "tgt", readers, CreateMetricsDict(), int.MaxValue,
                                            tokenSource.Token), tokenSource.Token);

            Thread.Sleep(StepDelay);
            Assert.That(task.Status, Is.EqualTo(TaskStatus.Running));
            tokenSource.Cancel();
            Thread.Sleep(StepDelay);
            Assert.That(task.Status, Is.EqualTo(TaskStatus.RanToCompletion));
        }
예제 #3
0
        public void Run()
        {
            Logger.Info($"Scraper thread for {_filename} on {_host} became alive");
            var envHostDict = new LabelDict(_environment);

            envHostDict.Set("host", _host);
            var connected = MetricBase.Connected.WithLabels(envHostDict) as Gauge;

            Debug.Assert(connected != null);
            try
            {
                while (!CancellationTokenSource.IsCancellationRequested)
                {
                    try
                    {
                        connected.Set(0);
                        try
                        {
                            Logger.Info($"Trying to establish connection to {_host}");
                            using (var client = CreateClient())
                            {
                                client.Connect();
                                connected.Set(1);
                                Logger.Info($"Starting tailing {_filename} on {_host}");
                                var cmd = client.CreateCommand($"tail -n0 --follow=name \"{_filename}\" 2>/dev/null");
                                var tmp = cmd.BeginExecute();
                                ((PipeStream)cmd.OutputStream).BlockLastReadBuffer = true;
                                try
                                {
                                    LogParser.ParseStream(cmd.OutputStream, _environment, _host, _readers, _metrics,
                                                          _readTimeoutMs,
                                                          CancellationTokenSource.Token);

                                    cmd.EndExecute(tmp);

                                    if (cmd.ExitStatus != 0)
                                    {
                                        Logger.Warn($"Tail command failed with exit code {cmd.ExitStatus} on {_host}");
                                    }
                                    else
                                    {
                                        Logger.Info($"Tail command finished successfully on {_host}");
                                    }
                                }
                                catch (StreamStarvationException)
                                {
                                    Logger.Warn($"SSH stream starvation for {_filename} on {_host}");
                                    cmd.CancelAsync();
                                }
                            }
                        }
                        catch (SshOperationTimeoutException ex)
                        {
                            Logger.Error($"Timeout on {_host}: {ex.Message}");
                        }
                        catch (SshConnectionException ex)
                        {
                            Logger.Error($"Failed to connect to {_host}: {ex.Message}");
                        }
                        catch (SshAuthenticationException ex)
                        {
                            Logger.Error($"Failed to authenticate for {_host}: {ex.Message}");
                        }
                        catch (SocketException ex)
                        {
                            Logger.Error($"Error on socket for {_host} (check firewall?): {ex.Message}");
                        }
                        catch (Exception ex)
                        {
                            Logger.Fatal(ex, $"Unhandled exception on {_host}: {ex.Message}");
                        }

                        connected.Set(0);
                        Logger.Info($"Will retry connecting to {_host} in 30 seconds");
                        if (CancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(30)))
                        {
                            break;
                        }
                    }
                    finally
                    {
                        connected.Set(0);
                    }
                }
            }
            finally
            {
                connected.Drop();
            }
        }