public async Task SendRecord_MetricIsPublished()
{
var records = new LogEnvelope <string>("msg", DateTime.UtcNow, "msg", null, 0, 1);
var sent = new List <string>();
var mock = new Mock <IAmazonCloudWatchLogs>();
var metrics = new InMemoryMetricsSource();
using var cts = new CancellationTokenSource();
var sink = new AsyncCloudWatchLogsSink(nameof(LogGroupStreamVariable_ShouldResolve), null, "group", "logStream", mock.Object, new NoopAppDataFileProvider(),
NullLogger.Instance, metrics, _mockBm, new NetworkStatus(new AlwaysAvailableNetworkProvider()), _cloudWatchOptions);
await sink.StartAsync(cts.Token);
// 7 metrics after start-up
Assert.Equal(7, metrics.Metrics.Count);
// send a record
sink.OnNext(records);
await Task.Delay(1000);
// 13 metrics after sending data
Assert.Equal(13, metrics.Metrics.Count);
cts.Cancel();
await sink.StopAsync(default);
public async Task ParseRecordsAsync(LogContext context, IList <IEnvelope <JObject> > output,
int recordCount, CancellationToken stopToken = default)
{
using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
stream.Position = context.Position;
using (var reader = new LineReader(stream, _encoding))
{
var linesCount = 0;
while (linesCount < recordCount)
{
stopToken.ThrowIfCancellationRequested();
var(line, consumed) = await reader.ReadAsync(stopToken);
_logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed);
context.Position += consumed;
if (line is null)
{
break;
}
try
{
var jObject = JObject.Parse(line);
var timestamp = _timestampExtrator is null
? DateTime.Now
: _timestampExtrator.GetTimestamp(jObject);
var envelope = new LogEnvelope <JObject>(jObject,
timestamp,
line,
context.FilePath,
context.Position,
context.LineNumber);
output.Add(envelope);
linesCount++;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing record '{0}'", line);
continue;
}
}
}
}
}
private async Task ParseRfc3164Log(LineReader reader, LogContext context, IList <IEnvelope <SyslogData> > output,
int recordCount, CancellationToken stopToken)
{
var packet = new Rfc3164Packet();
var parser = new Rfc3164Parser(new Rfc3164ParserOptions
{
RequirePri = false,
DefaultYear = DateTime.Now.Year
});
var linesCount = 0;
while (linesCount < recordCount)
{
stopToken.ThrowIfCancellationRequested();
var(line, consumed) = await reader.ReadAsync(stopToken);
_logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed);
context.Position += consumed;
if (line is null)
{
break;
}
context.LineNumber++;
var valid = parser.ParseString(line, ref packet);
if (!valid)
{
_logger.LogWarning($"Unable to parse record at line {context.LineNumber} in file {context.FilePath}. Record may be in invalid format");
}
var record = new SyslogData(
packet.TimeStamp ?? DateTimeOffset.Now,
packet.HostName,
packet.Tag,
packet.Content);
var envelope = new LogEnvelope <SyslogData>(
record,
record.Timestamp.UtcDateTime,
line,
context.FilePath,
context.Position,
context.LineNumber);
output.Add(envelope);
linesCount++;
}
}
/// <inheritdoc/>
public async Task ParseRecordsAsync(LogContext context, IList <IEnvelope <string> > output,
int recordCount, CancellationToken stopToken)
{
using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
stream.Position = context.Position;
using (var reader = new LineReader(stream, _encoding, _bufferSize))
{
string line;
int consumed;
var linesCount = 0;
while (linesCount < recordCount)
{
stopToken.ThrowIfCancellationRequested();
(line, consumed) = await reader.ReadAsync(stopToken);
_logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed);
context.Position += consumed;
if (line is null)
{
break;
}
if (context.LineNumber++ < _skipLines)
{
continue;
}
var envelope = new LogEnvelope <string>(line,
DateTime.UtcNow,
line,
context.FilePath,
context.Position,
context.LineNumber);
output.Add(envelope);
linesCount++;
}
}
}
}
public void EmitLogMessage(LogMessage logMessage)
{
using (var client = new UdpClient())
{
client.Connect(loggeratorAddress, port);
var envelope = new LogEnvelope
{
log_message = logMessage,
routing_key = logMessage.app_id,
signature = Encrypt(secret, DigestBytes(logMessage.message))
};
using (var stream = new MemoryStream())
{
Serializer.Serialize(stream, envelope);
stream.Flush();
byte[] writebuffer = stream.ToArray();
client.Send(writebuffer, writebuffer.Length);
}
}
}
/// <inheritdoc/>
public async Task ParseRecordsAsync(DelimitedTextLogContext context, IList <IEnvelope <TData> > output,
int recordCount, CancellationToken stopToken = default)
{
if (context.Fields is null)
{
context.Fields = await TryGetHeaderFields(context, stopToken);
}
var count = 0;
using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
stream.Position = context.Position;
using (var reader = new LineReader(stream, _encoding, _bufferSize))
{
while (count < recordCount)
{
stopToken.ThrowIfCancellationRequested();
var(line, consumed) = await reader.ReadAsync(stopToken);
_logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed);
context.Position += consumed;
if (line is null)
{
break;
}
if (ShouldStopAndRollback(line, context))
{
context.Position -= consumed;
return;
}
context.LineNumber++;
if (IsHeaders(line, context.LineNumber))
{
context.Fields = ParseHeadersLine(line);
continue;
}
else if (IsComment(line))
{
continue;
}
try
{
// 'ParseDataFragments' and 'CreateRecord' might throw error, so we need to catch it and skip the record
var fragments = ParseDataFragments(line);
if (context.Fields is null)
{
_logger.LogWarning("Unknown field mapping, skipping line {0}", context.LineNumber);
continue;
}
var dict = new Dictionary <string, string>();
for (var i = 0; i < context.Fields.Length; i++)
{
if (i >= fragments.Length)
{
break;
}
var(key, val) = KeyValueSelector(context.Fields[i], fragments[i]);
dict[key] = val;
}
var record = CreateRecord(context, dict);
var envelope = new LogEnvelope <TData>(record, record.Timestamp, line, context.FilePath, context.Position, context.LineNumber);
output.Add(envelope);
count++;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error processing record '{0}'", line);
continue;
}
}
}
}
}
