private void ProcessLine(Grok grok, string line, List <Dictionary <string, object> > result)
{
if (string.IsNullOrWhiteSpace(line))
{
if (!IgnoreEmptyLines.IsPresent)
{
result.Add(null);
}
return;
}
GrokResult grokResult = grok.Parse(line);
if (grokResult.Count == 0)
{
if (!IgnoreUnmatched.IsPresent)
{
result.Add(null);
}
return;
}
var dictionary = new Dictionary <string, object>();
foreach (GrokItem item in grokResult)
{
dictionary[item.Key] = item.Value;
}
result.Add(dictionary);
}
public void LogLocal()
{
Grok grokLogLocal = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
_ = grokLogLocal.Parse(@"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------");
}
public List <Dictionary <string, string> > readLogFile()
{
List <Dictionary <string, string> > dicArray = new List <Dictionary <string, string> >();
string text = System.IO.File.ReadAllText(@".\Files\apache_logs.txt");
string[] array = text.Split(new char[] { '\n' });
Grok grok = new Grok("%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request}HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}");
foreach (var line in array)
{
var response = grok.Parse(line);
Dictionary <string, string> dic = new Dictionary <string, string>();
foreach (var match in response)
{
dic.Add(match.Key, match.Value?.ToString());
}
dicArray.Add(dic);
}
return(dicArray);
}
public void Parse_MonthDay_Pattern()
{
// Arrange
const string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
const string month = "06";
const string day = "21";
const string year = "19";
var sut = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
// Act
GrokResult grokResult = sut.Parse(logs);
// Assert
Assert.Equal("month", grokResult[0].Key);
Assert.Equal(month, grokResult[0].Value);
Assert.Equal("day", grokResult[1].Key);
Assert.Equal(day, grokResult[1].Value);
Assert.Equal("year", grokResult[2].Key);
Assert.Equal(year, grokResult[2].Value);
Assert.Equal("month", grokResult[8].Key);
Assert.Equal(month, grokResult[8].Value);
Assert.Equal("day", grokResult[9].Key);
Assert.Equal(day, grokResult[9].Value);
Assert.Equal("year", grokResult[10].Key);
Assert.Equal(year, grokResult[10].Value);
}
/// <summary>
/// Write all the logs consistently into the database table Logs
/// </summary>
/// <param name="proc"> The process to be executed </param>
private void SaveLogs(Process proc)
{
proc.Start();
while (!proc.StandardOutput.EndOfStream && !_token.IsCancellationRequested)
{
string line = proc.StandardOutput.ReadLine();
if (!string.IsNullOrEmpty(line) && !line.StartsWith("---------"))
{
line = _device.Serial + " " + _device.Name + " " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff") + " " + line;
try
{
var grokResult = _grok.Parse(line);
_tableLogs.InsertValues(grokResult[0].Value.ToString(), grokResult[1].Value.ToString(),
Convert.ToDateTime(grokResult[2].Value), Convert.ToDateTime(grokResult[3].Value),
Convert.ToInt32(grokResult[4].Value), Convert.ToInt32(grokResult[5].Value),
grokResult[6].Value.ToString(), grokResult[7].Value.ToString(), grokResult[8].Value.ToString());
}
catch (Exception)
{
Console.WriteLine("Error while logging ... continuing");
}
}
}
}
public void ParseEmptyTest()
{
Grok act = new Grok("");
GrokResult result = act.Parse("");
Assert.AreNotEqual(null, result);
Assert.AreEqual(0, result.Count);
}
public void Exception_When_Parsing_NullGrokPattern()
{
// Arrange
const string grokPattern = null;
const string logs = "";
var sut = new Grok(grokPattern);
// Act, Assert
Assert.Throws <ArgumentNullException>(() => sut.Parse(logs));
}
public void TimePatternTest()
{
Grok act = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
GrokResult grokResult = act.Parse(logs);
Assert.Equal("21:00:13:589241", grokResult[3].Value);
Assert.Equal("21:00:13:589265", grokResult[11].Value);
}
public void ParseEmptyTest()
{
string grokPattern = "";
Grok act = new Grok(grokPattern);
string logs = "";
GrokResult grokResult = act.Parse(logs);
Assert.NotNull(grokResult);
Assert.Empty(grokResult);
}
public void PatternCountTest()
{
string grokPattern = "%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}";
Grok act = new Grok(grokPattern);
string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
const int elements = 16;
GrokResult grokResult = act.Parse(logs);
Assert.Equal(elements, grokResult.Count);
}
public void MacPatternTest(string macAddress)
{
const string timestamp = "21:00:13:589241";
string logs = $"{macAddress}:{timestamp}";
string grokPattern = "%{MAC:mac}:%{TIME:timestamp}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(macAddress, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
public void HostnamePatternTest(string hostName)
{
const string timestamp = "21:00:13:589241";
string logs = $"{hostName}:{timestamp}";
string grokPattern = "%{HOSTNAME:host}:%{TIME:timestamp}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(hostName, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
[InlineData("::")] // Default route
public void IPv6PatternTest(string ipAddress)
{
const string comment = "Free as in Free Beer";
string logs = $"{ipAddress}:{comment}";
string grokPattern = "%{IPV6:IP}:%{GREEDYDATA:comment}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(ipAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
}
public void EmailPatternTest()
{
Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}");
string logs = @"[email protected]:Free as in Free Beer";
GrokResult grokResult = act.Parse(logs);
Assert.Equal("*****@*****.**", grokResult[0].Value);
Assert.Equal("Free as in Free Beer", grokResult[1].Value);
Assert.Equal("email", grokResult[0].Key);
Assert.Equal("comment", grokResult[1].Key);
}
public void Parse_With_Type_Parse_Exception_Should_Ignore_Type(string regex, string parse, string toType)
{
// Arrange
var sut = new Grok($"%{{{regex}:{nameof(parse)}:{toType}}}", ReadCustomFile());
// Act
GrokResult grokResult = sut.Parse($"{parse}");
// Assert
Assert.Equal(nameof(parse), grokResult[0].Key);
Assert.Equal(parse, grokResult[0].Value);
}
public void Parse_Hostname_Pattern(string hostName)
{
// Arrange
const string timestamp = "21:00:13:589241";
var sut = new Grok("%{HOSTNAME:host}:%{TIME:timestamp}");
// Act
GrokResult grokResult = sut.Parse($"{hostName}:{timestamp}");
// Assert
Assert.Equal(hostName, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
public void Parse_Mac_Pattern(string macAddress)
{
// Arrange
const string timestamp = "21:00:13:589241";
var sut = new Grok("%{MAC:mac}:%{TIME:timestamp}");
// Act
GrokResult grokResult = sut.Parse($"{macAddress}:{timestamp}");
// Assert
Assert.Equal(macAddress, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
[InlineData("::")] // Default route
public void Parse_IPv6_Pattern(string ipAddress)
{
// Arrange
const string comment = "Free as in Free Beer";
var sut = new Grok("%{IPV6:IP}:%{GREEDYDATA:comment}");
// Act
GrokResult grokResult = sut.Parse($"{ipAddress}:{comment}");
// Assert
Assert.Equal(ipAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
}
public void EmailPatternTest()
{
Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}");
const string emailAddress = "*****@*****.**";
const string comment = "Free as in Free Beer";
string logs = emailAddress + ":" + comment;
GrokResult grokResult = act.Parse(logs);
Assert.Equal(emailAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
Assert.Equal("email", grokResult[0].Key);
Assert.Equal("comment", grokResult[1].Key);
}
public void Load_Custom_Patterns(string zipcode)
{
// Arrange
const string email = "*****@*****.**";
var sut = new Grok("%{ZIPCODE:zipcode}:%{EMAILADDRESS:email}", ReadCustomFile());
// Act
GrokResult grokResult = sut.Parse($"{zipcode}:{email}");
// Assert
Assert.Equal(zipcode, grokResult[0].Value);
Assert.Equal(email, grokResult[1].Value);
}
public void Parse_Empty_Logs_Not_Throws()
{
// Arrange
const string grokPattern = "";
const string logs = "";
var sut = new Grok(grokPattern);
// Act
GrokResult grokResult = sut.Parse(logs);
// Assert
Assert.NotNull(grokResult);
Assert.Empty(grokResult);
}
public void Expected_Elements_Count()
{
// Arrange
const string grokPattern = "%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}";
const string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
var sut = new Grok(grokPattern);
// Act
GrokResult grokResult = sut.Parse(logs);
// Assert
Assert.NotNull(grokResult);
Assert.Equal(16, grokResult.Count);
}
public void Parse_Email_Pattern()
{
// Arrange
const string emailAddress = "*****@*****.**";
const string comment = "Free as in Free Beer";
var sut = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}");
// Act
GrokResult grokResult = sut.Parse($"{emailAddress}:{comment}");
// Assert
Assert.Equal(emailAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
Assert.Equal("email", grokResult[0].Key);
Assert.Equal("comment", grokResult[1].Key);
}
public void Load_Custom_Patterns_From_IDictionary(string zipcode)
{
// Arrange
var customPatterns = new Dictionary <string, string>
{
{ "ZIPCODE", "[1-9]{1}[0-9]{2}\\s{0,1}[0-9]{3}" },
{ "FLOAT", "[+-]?([0-9]*[.,]}?[0-9]+)" }
};
const string email = "*****@*****.**";
var sut = new Grok("%{ZIPCODE:zipcode}:%{EMAILADDRESS:email}", customPatterns);
// Act
GrokResult grokResult = sut.Parse($"{zipcode}:{email}");
// Assert
Assert.Equal(zipcode, grokResult[0].Value);
Assert.Equal(email, grokResult[1].Value);
}
public void CustomLocal()
{
Grok grokCustomLocal = new Grok("%{ZIPCODE:zipcode}:%{EMAILADDRESS:email}");
_ = grokCustomLocal.Parse("06590:[email protected]");
}
public void EmptyLocal()
{
Grok grokEmptyLocal = new Grok("");
_ = grokEmptyLocal.Parse("");
}
public void Log()
{
_ = _grokLog.Parse(@"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------");
}
public void Custom()
{
_ = _grokCustom.Parse("06590:[email protected]");
}
public void Empty()
{
_ = _grokEmpty.Parse("");
}