public async Task TimestampOnSecondLine(string timestampFormat)
{
var timestamp = DateTime.Now.ToString(timestampFormat, CultureInfo.InvariantCulture);
var lines = new string[]
{
"Header: Record",
$"Time: {timestamp}"
};
await File.WriteAllLinesAsync(_testFile, lines);
var records = new List <IEnvelope <IDictionary <string, string> > >();
var regexTextParser = new RegexLogParser(NullLogger.Instance, "^Header:", new RegexParserOptions
{
ExtractionPattern = $@"^Header: (?<Title>.*)({Environment.NewLine})Time: (?<Timestamp>.*)$",
TimeZoneKind = DateTimeKind.Local,
TimestampFormat = timestampFormat
}, null, 1024);
await regexTextParser.ParseRecordsAsync(new RegexLogContext
{
FilePath = _testFile
}, records, 100);
Assert.Single(records);
var record = records[0];
Assert.Equal("Record", record.Data["Title"]);
Assert.Equal(DateTime.Parse(timestamp, CultureInfo.InvariantCulture), records[0].Timestamp);
}
public async Task ParseMultiLinesRecords(int recordCount)
{
var recordLines = new List <string>();
var noOfLines = 0;
for (var i = 0; i < recordCount; i++)
{
// vary the # of lines from 0-9
noOfLines = (noOfLines + 1) % 10;
recordLines.Add($"Header: Record {i}");
for (var j = 0; j < noOfLines; j++)
{
recordLines.Add($"Line {j}");
}
}
await File.WriteAllLinesAsync(_testFile, recordLines);
var records = new List <IEnvelope <IDictionary <string, string> > >();
var regexTextParser = new RegexLogParser(NullLogger.Instance, "^Header:", new RegexParserOptions
{
TimeZoneKind = DateTimeKind.Utc
}, null, 1024);
await regexTextParser.ParseRecordsAsync(new RegexLogContext
{
FilePath = _testFile
}, records, recordCount * 2);
Assert.Equal(recordCount, records.Count);
}
public async Task PrintLog()
{
await File.WriteAllLinesAsync(_testFile, _printLogs);
var regexTextParser = new RegexLogParser(NullLogger.Instance, @"^\[\w+\]\[(?<TimeStamp>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]", new RegexParserOptions
{
TimestampFormat = "yyyy/MM/dd HH:mm:ss.fff",
TimeZoneKind = DateTimeKind.Utc
}, null, 1024);
var records = new List <IEnvelope <IDictionary <string, string> > >();
await regexTextParser.ParseRecordsAsync(new RegexLogContext
{
FilePath = _testFile
}, records, 100);
Assert.Equal("[FATAL][2017/05/03 21:31:00.534][0x00003ca8][0000059c][][EQCASLicensingSubSystem][eqGetLicenseForSystemID][0] 'EQException.File: EQCASLicensingSubSystem.cpp'", records[0].GetMessage(null));
Assert.Equal(new DateTime(2017, 5, 3, 21, 31, 0, 534, DateTimeKind.Utc), records[0].Timestamp);
Assert.Equal("[FATAL][2017/05/03 21:31:00.535][0x00003ca8][0000059c][][EQCASLicensingSubSystem][eqGetLicenseForSystemID][0] 'EQException.Line: 3999'", records[1].GetMessage(null));
Assert.Equal(new DateTime(2017, 5, 3, 21, 31, 0, 535, DateTimeKind.Utc), records[1].Timestamp);
var envelope = (ILogEnvelope)records[1];
Assert.Equal(2, envelope.LineNumber);
}
/// <summary>
/// Valdiate regex
/// </summary>
/// <param name="directory"></param>
/// <param name="logName"></param>
/// <param name="config"></param>
/// <param name="sourceSection"></param>
/// <param name="curId"></param>
/// <param name="messages"></param>
/// <returns>True iff the logs matches the regex pattern in the configuration</returns>
private bool ValidateRegex(string directory, string logName, IConfigurationRoot config, IConfigurationSection sourceSection,
string curId, IList <string> messages)
{
string pattern = config[$"{sourceSection.Path}:{"Pattern"}"];
string timestampFormat = config[$"{sourceSection.Path}:{"TimestampFormat"}"];
string extractionPattern = config[$"{sourceSection.Path}:{"ExtrationPattern"}"];
string extractionRegexOptions = config[$"{sourceSection.Path}:{"ExtractionRegexOptions"}"];
var records = new List <IEnvelope <IDictionary <string, string> > >();
var parser = new RegexLogParser(NullLogger.Instance, pattern, new RegexParserOptions
{
TimestampFormat = timestampFormat,
ExtractionPattern = extractionPattern,
ExtractionRegexOptions = extractionRegexOptions,
TimeZoneKind = DateTimeKind.Utc
}, Encoding.UTF8, 1024);
parser.ParseRecordsAsync(new RegexLogContext {
FilePath = Path.Combine(directory, logName)
}, records, 100).GetAwaiter().GetResult();
if (records.Count == 1)
{
messages.Add("Invalid Regex at source ID: " + curId);
return(false);
}
else
{
messages.Add("Valid Regex at source ID: " + curId);
return(true);
}
}
public async Task PrintLog_RegExtractor()
{
await File.WriteAllLinesAsync(_testFile, _printLogs);
var xtractPattern = @"^\[(?<Severity>\w+)\]" +
@"\[(?<TimeStamp>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]" +
@"\[[^]]*\]" +
@"\[[^]]*\]" +
@"\[[^]]*\]" +
@"\[(?<SubSystem>\w+)\]" +
@"\[(?<Module>\w+)\]" +
@"\[[^]]*\]" +
@" '(?<Message>.*)'$";
var regexTextParser = new RegexLogParser(NullLogger.Instance, @"^\[\w+\]\[(?<TimeStamp>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\]", new RegexParserOptions
{
ExtractionPattern = xtractPattern,
TimeZoneKind = DateTimeKind.Utc,
TimestampFormat = "yyyy/MM/dd HH:mm:ss.fff"
}, null, 1024);
var records = new List <IEnvelope <IDictionary <string, string> > >();
await regexTextParser.ParseRecordsAsync(new RegexLogContext
{
FilePath = _testFile
}, records, 100);
Assert.Equal("FATAL", records[0].Data["Severity"]);
Assert.Equal("2017/05/03 21:31:00.534", records[0].Data["TimeStamp"]);
Assert.Equal("EQCASLicensingSubSystem", records[0].Data["SubSystem"]);
Assert.Equal("eqGetLicenseForSystemID", records[0].Data["Module"]);
Assert.Equal("EQException.File: EQCASLicensingSubSystem.cpp", records[0].Data["Message"]);
Assert.Equal("2017/05/03 21:31:00.535", records[1].Data["TimeStamp"]);
Assert.Equal("EQException.Line: 3999", records[1].Data["Message"]);
var envelope = (ILogEnvelope)records[1];
Assert.Equal(2, envelope.LineNumber);
}
