/// <summary> /// When overriden, return the header fields for the log file with a given <paramref name="context"/>, /// or 'null' if none is found /// </summary> protected virtual async Task <string[]> TryGetHeaderFields(DelimitedTextLogContext context, CancellationToken stopToken) { var position = 0; long lineNumber = 0; using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var reader = new LineReader(stream, _encoding, _bufferSize)) { while (position < context.Position) { stopToken.ThrowIfCancellationRequested(); var(line, consumed) = await reader.ReadAsync(stopToken); if (line is null) { break; } lineNumber++; position += consumed; if (IsHeaders(line, lineNumber)) { return(ParseHeadersLine(line)); } } } return(null); }
public async Task ParseRecordsAsync(LogContext context, IList <IEnvelope <JObject> > output, int recordCount, CancellationToken stopToken = default) { using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { stream.Position = context.Position; using (var reader = new LineReader(stream, _encoding)) { var linesCount = 0; while (linesCount < recordCount) { stopToken.ThrowIfCancellationRequested(); var(line, consumed) = await reader.ReadAsync(stopToken); _logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed); context.Position += consumed; if (line is null) { break; } try { var jObject = JObject.Parse(line); var timestamp = _timestampExtrator is null ? DateTime.Now : _timestampExtrator.GetTimestamp(jObject); var envelope = new LogEnvelope <JObject>(jObject, timestamp, line, context.FilePath, context.Position, context.LineNumber); output.Add(envelope); linesCount++; } catch (Exception ex) { _logger.LogError(ex, "Error processing record '{0}'", line); continue; } } } } }
private async Task ParseRfc3164Log(LineReader reader, LogContext context, IList <IEnvelope <SyslogData> > output, int recordCount, CancellationToken stopToken) { var packet = new Rfc3164Packet(); var parser = new Rfc3164Parser(new Rfc3164ParserOptions { RequirePri = false, DefaultYear = DateTime.Now.Year }); var linesCount = 0; while (linesCount < recordCount) { stopToken.ThrowIfCancellationRequested(); var(line, consumed) = await reader.ReadAsync(stopToken); _logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed); context.Position += consumed; if (line is null) { break; } context.LineNumber++; var valid = parser.ParseString(line, ref packet); if (!valid) { _logger.LogWarning($"Unable to parse record at line {context.LineNumber} in file {context.FilePath}. Record may be in invalid format"); } var record = new SyslogData( packet.TimeStamp ?? DateTimeOffset.Now, packet.HostName, packet.Tag, packet.Content); var envelope = new LogEnvelope <SyslogData>( record, record.Timestamp.UtcDateTime, line, context.FilePath, context.Position, context.LineNumber); output.Add(envelope); linesCount++; } }
/// <inheritdoc/> public async Task ParseRecordsAsync(LogContext context, IList <IEnvelope <string> > output, int recordCount, CancellationToken stopToken) { using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { stream.Position = context.Position; using (var reader = new LineReader(stream, _encoding, _bufferSize)) { string line; int consumed; var linesCount = 0; while (linesCount < recordCount) { stopToken.ThrowIfCancellationRequested(); (line, consumed) = await reader.ReadAsync(stopToken); _logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed); context.Position += consumed; if (line is null) { break; } if (context.LineNumber++ < _skipLines) { continue; } var envelope = new LogEnvelope <string>(line, DateTime.UtcNow, line, context.FilePath, context.Position, context.LineNumber); output.Add(envelope); linesCount++; } } } }
public async Task ParseRecordsAsync(RegexLogContext context, IList <IEnvelope <IDictionary <string, string> > > output, int recordCount, CancellationToken stopToken = default) { var count = 0; using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { stream.Position = context.Position; using (var reader = new LineReader(stream, _encoding, _bufferSize)) { while (count < recordCount) { stopToken.ThrowIfCancellationRequested(); var(line, consumed) = await reader.ReadAsync(stopToken); _logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed); if (line is null) { // end-of-file var record = CreateRecord(context); if (record is not null) { output.Add(record); } context.RecordBuilder.Clear(); context.MatchedLineNumber = -1; context.MatchedLineTimestamp = null; break; } context.LineNumber++; context.Position += consumed; var match = _patternRegex.Match(line); if (match.Success) { _logger.LogDebug("Regex matched."); // this is start of a new record, get the last record var record = CreateRecord(context); if (record is not null) { output.Add(record); count++; } // remember the new record's first line context.RecordBuilder.Clear(); context.RecordBuilder.Append(line); context.MatchedLineNumber = context.LineNumber; // if the matched line has a 'Timestamp' match group, use it as the record's timestamp context.MatchedLineTimestamp = GetTimeStamp(match); } else { _logger.LogDebug("Regex NOT matched."); // line is not the beginning of a new record if (context.RecordBuilder.Length != 0) { // some lines have been added to the new record context.RecordBuilder.AppendLine(); context.RecordBuilder.Append(line); } else if (_removeUnmatchedRecord) { _logger.LogWarning("Line discarded: {0}", line); } else { // start a new record // TODO figure out what's the use case scenario for this context.RecordBuilder.Append(line); context.MatchedLineNumber = context.LineNumber; } } } } } }
/// <inheritdoc/> public async Task ParseRecordsAsync(DelimitedTextLogContext context, IList <IEnvelope <TData> > output, int recordCount, CancellationToken stopToken = default) { if (context.Fields is null) { context.Fields = await TryGetHeaderFields(context, stopToken); } var count = 0; using (var stream = new FileStream(context.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { stream.Position = context.Position; using (var reader = new LineReader(stream, _encoding, _bufferSize)) { while (count < recordCount) { stopToken.ThrowIfCancellationRequested(); var(line, consumed) = await reader.ReadAsync(stopToken); _logger.LogTrace("File: '{0}', line: '{1}', bytes: {2}", context.FilePath, line, consumed); context.Position += consumed; if (line is null) { break; } if (ShouldStopAndRollback(line, context)) { context.Position -= consumed; return; } context.LineNumber++; if (IsHeaders(line, context.LineNumber)) { context.Fields = ParseHeadersLine(line); continue; } else if (IsComment(line)) { continue; } try { // 'ParseDataFragments' and 'CreateRecord' might throw error, so we need to catch it and skip the record var fragments = ParseDataFragments(line); if (context.Fields is null) { _logger.LogWarning("Unknown field mapping, skipping line {0}", context.LineNumber); continue; } var dict = new Dictionary <string, string>(); for (var i = 0; i < context.Fields.Length; i++) { if (i >= fragments.Length) { break; } var(key, val) = KeyValueSelector(context.Fields[i], fragments[i]); dict[key] = val; } var record = CreateRecord(context, dict); var envelope = new LogEnvelope <TData>(record, record.Timestamp, line, context.FilePath, context.Position, context.LineNumber); output.Add(envelope); count++; } catch (Exception ex) { _logger.LogError(ex, "Error processing record '{0}'", line); continue; } } } } }