public void TestReadTimeout()
{
var streamMock = new Mock <Stream>();
streamMock.Setup(stream => stream.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
.Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
streamMock.Setup(stream => stream.ReadAsync(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>(),
It.IsAny <CancellationToken>()))
.Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
streamMock.Setup(stream => stream.CanRead)
.Returns(true);
var readers = new List <ColumnReader>();
var sw = new Stopwatch();
sw.Start();
Assert.That(
() => LogParser.ParseStream(streamMock.Object, "env", "tgt", readers, CreateMetricsDict(), 200,
CancellationToken.None), Throws.TypeOf <StreamStarvationException>());
sw.Stop();
Assert.That(sw.Elapsed.TotalSeconds, Is.LessThan(ReadSleepSeconds));
}
public void TestCancellation()
{
var streamMock = new Mock <Stream>();
streamMock.Setup(stream => stream.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
.Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
streamMock.Setup(stream => stream.ReadAsync(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>(),
It.IsAny <CancellationToken>()))
.Callback(() => { Thread.Sleep(TimeSpan.FromSeconds(ReadSleepSeconds)); });
streamMock.Setup(stream => stream.CanRead)
.Returns(true);
var readers = new List <ColumnReader>();
var tokenSource = new CancellationTokenSource();
var task = Task.Run(
() => LogParser.ParseStream(streamMock.Object, "env", "tgt", readers, CreateMetricsDict(), int.MaxValue,
tokenSource.Token), tokenSource.Token);
Thread.Sleep(StepDelay);
Assert.That(task.Status, Is.EqualTo(TaskStatus.Running));
tokenSource.Cancel();
Thread.Sleep(StepDelay);
Assert.That(task.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
public void Run()
{
Logger.Info($"Scraper thread for {_filename} on {_host} became alive");
var envHostDict = new LabelDict(_environment);
envHostDict.Set("host", _host);
var connected = MetricBase.Connected.WithLabels(envHostDict) as Gauge;
Debug.Assert(connected != null);
try
{
while (!CancellationTokenSource.IsCancellationRequested)
{
try
{
connected.Set(0);
try
{
Logger.Info($"Trying to establish connection to {_host}");
using (var client = CreateClient())
{
client.Connect();
connected.Set(1);
Logger.Info($"Starting tailing {_filename} on {_host}");
var cmd = client.CreateCommand($"tail -n0 --follow=name \"{_filename}\" 2>/dev/null");
var tmp = cmd.BeginExecute();
((PipeStream)cmd.OutputStream).BlockLastReadBuffer = true;
try
{
LogParser.ParseStream(cmd.OutputStream, _environment, _host, _readers, _metrics,
_readTimeoutMs,
CancellationTokenSource.Token);
cmd.EndExecute(tmp);
if (cmd.ExitStatus != 0)
{
Logger.Warn($"Tail command failed with exit code {cmd.ExitStatus} on {_host}");
}
else
{
Logger.Info($"Tail command finished successfully on {_host}");
}
}
catch (StreamStarvationException)
{
Logger.Warn($"SSH stream starvation for {_filename} on {_host}");
cmd.CancelAsync();
}
}
}
catch (SshOperationTimeoutException ex)
{
Logger.Error($"Timeout on {_host}: {ex.Message}");
}
catch (SshConnectionException ex)
{
Logger.Error($"Failed to connect to {_host}: {ex.Message}");
}
catch (SshAuthenticationException ex)
{
Logger.Error($"Failed to authenticate for {_host}: {ex.Message}");
}
catch (SocketException ex)
{
Logger.Error($"Error on socket for {_host} (check firewall?): {ex.Message}");
}
catch (Exception ex)
{
Logger.Fatal(ex, $"Unhandled exception on {_host}: {ex.Message}");
}
connected.Set(0);
Logger.Info($"Will retry connecting to {_host} in 30 seconds");
if (CancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(30)))
{
break;
}
}
finally
{
connected.Set(0);
}
}
}
finally
{
connected.Drop();
}
}