public async Task TestSQLLog()
{
var log = @"2019-02-13 04:50:33.89 Server Microsoft SQL Server 2014 (SP2-GDR) (KB4057120) - 12.0.5214.6 (X64)
Jan 9 2018 15:03:12
Copyright (c) Microsoft Corporation
Enterprise Edition (64-bit) on Windows NT 6.3 <X64> (Build 9600: ) (Hypervisor)
2019-02-13 04:50:34.00 Server UTC adjustment: 0:00";
await File.WriteAllLinesAsync(_testFile, new string[] { log });
var records = new List <IEnvelope <IDictionary <string, string> > >();
var parser = new TimestampLogParser(NullLogger.Instance, new RegexParserOptions
{
TimestampFormat = "yyyy-MM-dd HH:mm:ss.ff",
TimeZoneKind = DateTimeKind.Utc,
ExtractionPattern = "^\\s*(?<TimeStamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{2})\\s(?<Source>\\w+)\\s+(?<Message>.*)$",
ExtractionRegexOptions = nameof(RegexOptions.Singleline)
}, null, 1024);
await parser.ParseRecordsAsync(new RegexLogContext { FilePath = _testFile }, records, 100);
Assert.Equal(2, records.Count);
Assert.Equal("2019-02-13 04:50:33.89", records[0].Data["TimeStamp"]);
Assert.Equal("Server", records[0].Data["Source"]);
Assert.True(records[0].Data["Message"].Length > 0);
}
public async Task TestSSMLog(DateTimeKind dateTimeKind)
{
var logs = new string[]
{
"2017-03-31 10:06:20 ERROR [instanceID=i-0ad46e850f00b20ba] [MessageProcessor] error when calling AWS APIs. error details - GetMessages Error: AccessDeniedException: User: arn:aws:sts::266928793956:assumed-role/ds2amazonli/i-0ad46e850f00b20ba is not authorized to perform: ec2messages:GetMessages on resource: *",
"status code: 400, request id: afe4d9d7-15f9-11e7-8eb7-193e481a50d1",
"2017-03-31 10:06:21 INFO [instanceID=i-0ad46e850f00b20ba] [MessageProcessor] increasing error count by 1"
};
await File.WriteAllLinesAsync(_testFile, logs);
var records = new List <IEnvelope <IDictionary <string, string> > >();
var parser = new TimestampLogParser(NullLogger.Instance, new RegexParserOptions
{
TimestampFormat = "yyyy-MM-dd HH:mm:ss",
TimeZoneKind = dateTimeKind
}, null, 1024);
await parser.ParseRecordsAsync(new RegexLogContext { FilePath = _testFile }, records, 100);
Assert.Equal("2017-03-31 10:06:20 ERROR [instanceID=i-0ad46e850f00b20ba] [MessageProcessor] error when calling AWS APIs. error details - GetMessages Error: AccessDeniedException: User: arn:aws:sts::266928793956:assumed-role/ds2amazonli/i-0ad46e850f00b20ba is not authorized to perform: ec2messages:GetMessages on resource: *" +
Environment.NewLine + "status code: 400, request id: afe4d9d7-15f9-11e7-8eb7-193e481a50d1", records[0].GetMessage(null));
var timestamp0 = new DateTime(2017, 3, 31, 10, 6, 20, dateTimeKind);
Assert.Equal(timestamp0, records[0].Timestamp);
Assert.Equal(timestamp0.Kind, records[0].Timestamp.Kind);
Assert.Equal("2017-03-31 10:06:21 INFO [instanceID=i-0ad46e850f00b20ba] [MessageProcessor] increasing error count by 1", records[1].GetMessage(null));
var timestamp1 = new DateTime(2017, 3, 31, 10, 6, 21, dateTimeKind);
Assert.Equal(timestamp1, records[1].Timestamp);
Assert.Equal(timestamp1.Kind, records[1].Timestamp.Kind);
}
/// <summary>
/// Validate timestamp
/// </summary>
/// <param name="directory"></param>
/// <param name="logName"></param>
/// <param name="config"></param>
/// <param name="sourceSection"></param>
/// <param name="curId"></param>
/// <param name="messages"></param>
/// <returns>True iff the logs has valid timestamps</returns>
private bool ValidateTimeStamp(string directory, string logName, IConfigurationRoot config, IConfigurationSection sourceSection, string curId, IList <string> messages)
{
string timestampFormat = config[$"{sourceSection.Path}:TimestampFormat"];
var parser = new TimestampLogParser(NullLogger.Instance, new RegexParserOptions
{
TimestampFormat = timestampFormat,
TimeZoneKind = DateTimeKind.Utc
}, null, 1024);
var records = new List <IEnvelope <IDictionary <string, string> > >();
parser.ParseRecordsAsync(new RegexLogContext
{
FilePath = Path.Combine(directory, logName)
}, records, 2).GetAwaiter().GetResult();
if (records.Count == 1)
{
messages.Add("Invalid Timestamp format at source ID: " + curId);
return(false);
}
else
{
messages.Add("Valid Timestamp format at source ID: " + curId);
return(true);
}
}
private async Task TestTimestampLogInternal(DateTime expectedTime1, DateTime expectedTime2, DateTimeKind timeZoneKind)
{
var logs = new string[]
{
"2017-05-18 00:00:28.0665 Quartz.Core.QuartzSchedulerThread DEBUG Batch acquisition of 0 triggers",
"2017-05-18 00:00:56.8128 Quartz.Core.QuartzSchedulerThread DEBUG Batch acquisition of 0 triggers"
};
await File.WriteAllLinesAsync(_testFile, logs);
var records = new List <IEnvelope <IDictionary <string, string> > >();
var parser = new TimestampLogParser(NullLogger.Instance, new RegexParserOptions
{
TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffff",
TimeZoneKind = timeZoneKind
}, null, 1024);
await parser.ParseRecordsAsync(new RegexLogContext { FilePath = _testFile }, records, 100);
Assert.Equal("2017-05-18 00:00:28.0665 Quartz.Core.QuartzSchedulerThread DEBUG Batch acquisition of 0 triggers", records[0].GetMessage(null));
Assert.Equal(expectedTime1, records[0].Timestamp);
Assert.Equal("2017-05-18 00:00:56.8128 Quartz.Core.QuartzSchedulerThread DEBUG Batch acquisition of 0 triggers", records[1].GetMessage(null));
Assert.Equal(expectedTime2, records[1].Timestamp);
}