private void ProcessLine(Grok grok, string line, List <Dictionary <string, object> > result)
{
if (string.IsNullOrWhiteSpace(line))
{
if (!IgnoreEmptyLines.IsPresent)
{
result.Add(null);
}
return;
}
GrokResult grokResult = grok.Parse(line);
if (grokResult.Count == 0)
{
if (!IgnoreUnmatched.IsPresent)
{
result.Add(null);
}
return;
}
var dictionary = new Dictionary <string, object>();
foreach (GrokItem item in grokResult)
{
dictionary[item.Key] = item.Value;
}
result.Add(dictionary);
}
public void LogLocal()
{
Grok grokLogLocal = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
_ = grokLogLocal.Parse(@"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------");
}
public void TestMatchCount()
{
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
],
""remove_tag"":[
""tag1""
]
}
}]
}
}";
Configuration c = Configuration.FromString(grokJson);
Grok grok = c.Filters.First() as Grok;
Assert.AreEqual(2, grok.Match.Length);
}
void NoteUserSpeech(string key, DataStore.IValue value)
{
string input = value.ToString().Trim();
if (string.IsNullOrEmpty(input))
{
return;
}
// Initialize a parser with the raw input
var parser = new Parser();
ParseState st = parser.InitState(input);
// Apply parsing rules until we can do no more
int rule;
while ((rule = parser.NextStep(st)) > 0)
{
//Print(st.ToString() + " {" + rule + "}");
}
SetValue("user:parse", st.TreeForm(), "parsed: " + input);
// Now, attempt to grok the input (convert it to a Communication)
Communication comm = Grok.GrokInput(input, st);
SetValue("user:communication", new CommunicationValue(comm), comm.ToString());
}
public List <Dictionary <string, string> > readLogFile()
{
List <Dictionary <string, string> > dicArray = new List <Dictionary <string, string> >();
string text = System.IO.File.ReadAllText(@".\Files\apache_logs.txt");
string[] array = text.Split(new char[] { '\n' });
Grok grok = new Grok("%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request}HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}");
foreach (var line in array)
{
var response = grok.Parse(line);
Dictionary <string, string> dic = new Dictionary <string, string>();
foreach (var match in response)
{
dic.Add(match.Key, match.Value?.ToString());
}
dicArray.Add(dic);
}
return(dicArray);
}
public void TestAddFieldCount()
{
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
],
""remove_tag"":[
""tag1""
],
""add_field"":[
""host"",
""%{ComputerName}""
]
}
}]
}
}";
Configuration c = Configuration.FromString(grokJson);
Grok grok = c.Filters.First() as Grok;
Assert.IsTrue(grok.AddField.Length % 2 == 0);
}
public void Parse_Pattern_With_Type_Should_Parse_To_Specified_Type()
{
// Arrange
const int intValue = 28;
const double floatValue = 3000.5F;
var dateTime = new DateTime(2010, 10, 10);
var sut = new Grok("%{INT:int_value:int}:%{DATESTAMP:date_time:datetime}:%{FLOAT:float_value:float}", ReadCustomFile());
// Act
GrokResult grokResult = sut.Parse($"{intValue}:{dateTime:dd-MM-yyyy HH:mm:ss}:{floatValue}");
// Assert
Assert.Equal("int_value", grokResult[0].Key);
Assert.Equal(intValue, grokResult[0].Value);
Assert.IsType <int>(grokResult[0].Value);
Assert.Equal("date_time", grokResult[1].Key);
Assert.Equal(dateTime, grokResult[1].Value);
Assert.IsType <DateTime>(grokResult[1].Value);
Assert.Equal("float_value", grokResult[2].Key);
Assert.Equal(floatValue, grokResult[2].Value);
Assert.IsType <double>(grokResult[2].Value); // Float converts to double actually
}
public void TestAddTagCount()
{
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
],
""add_tag"":[
""rn_%{RecordNumber}"",
""bar""
]
}
}]
}
}";
Configuration c = Configuration.FromString(grokJson);
Grok grok = c.Filters.First() as Grok;
Assert.IsTrue(grok.AddTag.Length >= 1);
}
public void Parse_MonthDay_Pattern()
{
// Arrange
const string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
const string month = "06";
const string day = "21";
const string year = "19";
var sut = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
// Act
GrokResult grokResult = sut.Parse(logs);
// Assert
Assert.Equal("month", grokResult[0].Key);
Assert.Equal(month, grokResult[0].Value);
Assert.Equal("day", grokResult[1].Key);
Assert.Equal(day, grokResult[1].Value);
Assert.Equal("year", grokResult[2].Key);
Assert.Equal(year, grokResult[2].Value);
Assert.Equal("month", grokResult[8].Key);
Assert.Equal(month, grokResult[8].Value);
Assert.Equal("day", grokResult[9].Key);
Assert.Equal(day, grokResult[9].Value);
Assert.Equal("year", grokResult[10].Key);
Assert.Equal(year, grokResult[10].Value);
}
public void NegativeResult()
{
var grok = new Grok("%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}");
var response = grok.ParseLine("gary mail \"never quit\"");
Assert.Equal(0, response.Captures.Count);
}
public void ParseEmptyTest()
{
Grok act = new Grok("");
GrokResult result = act.Parse("");
Assert.AreNotEqual(null, result);
Assert.AreEqual(0, result.Count);
}
public void TypedResult()
{
var grok = new Grok("%{WORD:name} %{INT:age:int} %{QUOTEDSTRING:motto}");
var response = grok.ParseLine("gary 1 \"never quit\"");
Assert.Equal(3, response.Captures.Count);
Assert.Equal("age", response.Captures[1].Item1);
Assert.Equal(typeof(int).ToString(), response.Captures[1].Item2.GetType().ToString());
}
public void SingleResults(string grokString, string line, string extractName, string extractData)
{
var grok = new Grok(grokString);
var response = grok.ParseLine(line);
Assert.Equal(1, response.Captures.Count);
Assert.Equal(extractName, response.Captures[0].Item1);
Assert.Equal(extractData, response.Captures[0].Item2.ToString());
}
public void Exception_When_Parsing_NullGrokPattern()
{
// Arrange
const string grokPattern = null;
const string logs = "";
var sut = new Grok(grokPattern);
// Act, Assert
Assert.Throws <ArgumentNullException>(() => sut.Parse(logs));
}
public void TimePatternTest()
{
Grok act = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}");
string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
GrokResult grokResult = act.Parse(logs);
Assert.Equal("21:00:13:589241", grokResult[3].Value);
Assert.Equal("21:00:13:589265", grokResult[11].Value);
}
public void ParseEmptyTest()
{
string grokPattern = "";
Grok act = new Grok(grokPattern);
string logs = "";
GrokResult grokResult = act.Parse(logs);
Assert.NotNull(grokResult);
Assert.Empty(grokResult);
}
private void ProcessString(List <Dictionary <string, object> > result)
{
Grok grok = GrokInstance();
var lines = Input.Split(new[] { Environment.NewLine },
IgnoreEmptyLines.IsPresent ? StringSplitOptions.RemoveEmptyEntries : StringSplitOptions.None);
foreach (string line in lines)
{
ProcessLine(grok, line, result);
}
}
public void PatternCountTest()
{
string grokPattern = "%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}";
Grok act = new Grok(grokPattern);
string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165
06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------";
const int elements = 16;
GrokResult grokResult = act.Parse(logs);
Assert.Equal(elements, grokResult.Count);
}
public void MacPatternTest(string macAddress)
{
const string timestamp = "21:00:13:589241";
string logs = $"{macAddress}:{timestamp}";
string grokPattern = "%{MAC:mac}:%{TIME:timestamp}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(macAddress, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
public void HostnamePatternTest(string hostName)
{
const string timestamp = "21:00:13:589241";
string logs = $"{hostName}:{timestamp}";
string grokPattern = "%{HOSTNAME:host}:%{TIME:timestamp}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(hostName, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
[InlineData("::")] // Default route
public void IPv6PatternTest(string ipAddress)
{
const string comment = "Free as in Free Beer";
string logs = $"{ipAddress}:{comment}";
string grokPattern = "%{IPV6:IP}:%{GREEDYDATA:comment}";
Grok act = new Grok(grokPattern);
GrokResult grokResult = act.Parse(logs);
Assert.Equal(ipAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
}
public void Parse_With_Type_Parse_Exception_Should_Ignore_Type(string regex, string parse, string toType)
{
// Arrange
var sut = new Grok($"%{{{regex}:{nameof(parse)}:{toType}}}", ReadCustomFile());
// Act
GrokResult grokResult = sut.Parse($"{parse}");
// Assert
Assert.Equal(nameof(parse), grokResult[0].Key);
Assert.Equal(parse, grokResult[0].Value);
}
public void EmailPatternTest()
{
Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}");
string logs = @"[email protected]:Free as in Free Beer";
GrokResult grokResult = act.Parse(logs);
Assert.Equal("*****@*****.**", grokResult[0].Value);
Assert.Equal("Free as in Free Beer", grokResult[1].Value);
Assert.Equal("email", grokResult[0].Key);
Assert.Equal("comment", grokResult[1].Key);
}
public void MulipleResults()
{
var grok = new Grok("%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}");
var response = grok.ParseLine("gary 25 \"never quit\"");
Assert.Equal(3, response.Captures.Count);
Assert.Equal("name", response.Captures[0].Item1);
Assert.Equal("gary", response.Captures[0].Item2.ToString());
Assert.Equal("age", response.Captures[1].Item1);
Assert.Equal("25", response.Captures[1].Item2.ToString());
Assert.Equal("motto", response.Captures[2].Item1);
Assert.Equal("\"never quit\"", response.Captures[2].Item2.ToString());
}
[InlineData("::")] // Default route
public void Parse_IPv6_Pattern(string ipAddress)
{
// Arrange
const string comment = "Free as in Free Beer";
var sut = new Grok("%{IPV6:IP}:%{GREEDYDATA:comment}");
// Act
GrokResult grokResult = sut.Parse($"{ipAddress}:{comment}");
// Assert
Assert.Equal(ipAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
}
public void Parse_Hostname_Pattern(string hostName)
{
// Arrange
const string timestamp = "21:00:13:589241";
var sut = new Grok("%{HOSTNAME:host}:%{TIME:timestamp}");
// Act
GrokResult grokResult = sut.Parse($"{hostName}:{timestamp}");
// Assert
Assert.Equal(hostName, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
public void Parse_Mac_Pattern(string macAddress)
{
// Arrange
const string timestamp = "21:00:13:589241";
var sut = new Grok("%{MAC:mac}:%{TIME:timestamp}");
// Act
GrokResult grokResult = sut.Parse($"{macAddress}:{timestamp}");
// Assert
Assert.Equal(macAddress, grokResult[0].Value);
Assert.Equal(timestamp, grokResult[1].Value);
}
public void EmailPatternTest()
{
Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}");
const string emailAddress = "*****@*****.**";
const string comment = "Free as in Free Beer";
string logs = emailAddress + ":" + comment;
GrokResult grokResult = act.Parse(logs);
Assert.Equal(emailAddress, grokResult[0].Value);
Assert.Equal(comment, grokResult[1].Value);
Assert.Equal("email", grokResult[0].Key);
Assert.Equal("comment", grokResult[1].Key);
}
private void ProcessFile(List <Dictionary <string, object> > result)
{
Grok grok = GrokInstance();
using (FileStream fileStream = File.OpenRead(Path))
using (var streamReader = new StreamReader(fileStream, Encoding.UTF8, true))
{
string line;
while ((line = streamReader.ReadLine()) != null)
{
ProcessLine(grok, line, result);
}
}
}
public void Parse_Empty_Logs_Not_Throws()
{
// Arrange
const string grokPattern = "";
const string logs = "";
var sut = new Grok(grokPattern);
// Act
GrokResult grokResult = sut.Parse(logs);
// Assert
Assert.NotNull(grokResult);
Assert.Empty(grokResult);
}
