public static void ParseStream(Stream stream, string environment, string target, IList <ColumnReader> readers,
IDictionary <string, MetricBase> metrics, int msTimeout, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(environment))
{
throw new ArgumentException("Environment must not be empty", nameof(environment));
}
var envDict = new LabelDict(environment);
var envTargetDict = new LabelDict(environment);
envTargetDict.Set("target", target);
foreach (var entry in new LogParser(stream, readers, environment).ReadAll(msTimeout, cancellationToken))
{
if (entry == null)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
MetricBase.ParserErrors.WithLabels(envDict).Add(1);
MetricBase.ParserErrorsPerTarget.WithLabels(envTargetDict).Add(1);
continue;
}
MetricBase.LinesParsed.WithLabels(entry.Labels).Add(1);
var labels = new LabelDict(entry.Labels);
labels.Set("target", target);
MetricBase.LinesParsedPerTarget.WithLabels(labels).Add(1);
foreach (var(name, amount) in entry.Metrics)
{
metrics[name].WithLabels(entry.Labels).Add(amount);
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
}
}
public void Run()
{
Logger.Info($"Scraper thread for {_filename} on {_host} became alive");
var envHostDict = new LabelDict(_environment);
envHostDict.Set("host", _host);
var connected = MetricBase.Connected.WithLabels(envHostDict) as Gauge;
Debug.Assert(connected != null);
try
{
while (!CancellationTokenSource.IsCancellationRequested)
{
try
{
connected.Set(0);
try
{
Logger.Info($"Trying to establish connection to {_host}");
using (var client = CreateClient())
{
client.Connect();
connected.Set(1);
Logger.Info($"Starting tailing {_filename} on {_host}");
var cmd = client.CreateCommand($"tail -n0 --follow=name \"{_filename}\" 2>/dev/null");
var tmp = cmd.BeginExecute();
((PipeStream)cmd.OutputStream).BlockLastReadBuffer = true;
try
{
LogParser.ParseStream(cmd.OutputStream, _environment, _host, _readers, _metrics,
_readTimeoutMs,
CancellationTokenSource.Token);
cmd.EndExecute(tmp);
if (cmd.ExitStatus != 0)
{
Logger.Warn($"Tail command failed with exit code {cmd.ExitStatus} on {_host}");
}
else
{
Logger.Info($"Tail command finished successfully on {_host}");
}
}
catch (StreamStarvationException)
{
Logger.Warn($"SSH stream starvation for {_filename} on {_host}");
cmd.CancelAsync();
}
}
}
catch (SshOperationTimeoutException ex)
{
Logger.Error($"Timeout on {_host}: {ex.Message}");
}
catch (SshConnectionException ex)
{
Logger.Error($"Failed to connect to {_host}: {ex.Message}");
}
catch (SshAuthenticationException ex)
{
Logger.Error($"Failed to authenticate for {_host}: {ex.Message}");
}
catch (SocketException ex)
{
Logger.Error($"Error on socket for {_host} (check firewall?): {ex.Message}");
}
catch (Exception ex)
{
Logger.Fatal(ex, $"Unhandled exception on {_host}: {ex.Message}");
}
connected.Set(0);
Logger.Info($"Will retry connecting to {_host} in 30 seconds");
if (CancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(30)))
{
break;
}
}
finally
{
connected.Set(0);
}
}
}
finally
{
connected.Drop();
}
}