private void ProcessLine(Grok grok, string line, List <Dictionary <string, object> > result) { if (string.IsNullOrWhiteSpace(line)) { if (!IgnoreEmptyLines.IsPresent) { result.Add(null); } return; } GrokResult grokResult = grok.Parse(line); if (grokResult.Count == 0) { if (!IgnoreUnmatched.IsPresent) { result.Add(null); } return; } var dictionary = new Dictionary <string, object>(); foreach (GrokItem item in grokResult) { dictionary[item.Key] = item.Value; } result.Add(dictionary); }
public void LogLocal() { Grok grokLogLocal = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}"); _ = grokLogLocal.Parse(@"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165 06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------"); }
public void TestMatchCount() { string grokJson = @"{ ""TimberWinR"":{ ""Filters"":[ { ""grok"":{ ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" ], ""remove_tag"":[ ""tag1"" ] } }] } }"; Configuration c = Configuration.FromString(grokJson); Grok grok = c.Filters.First() as Grok; Assert.AreEqual(2, grok.Match.Length); }
void NoteUserSpeech(string key, DataStore.IValue value) { string input = value.ToString().Trim(); if (string.IsNullOrEmpty(input)) { return; } // Initialize a parser with the raw input var parser = new Parser(); ParseState st = parser.InitState(input); // Apply parsing rules until we can do no more int rule; while ((rule = parser.NextStep(st)) > 0) { //Print(st.ToString() + " {" + rule + "}"); } SetValue("user:parse", st.TreeForm(), "parsed: " + input); // Now, attempt to grok the input (convert it to a Communication) Communication comm = Grok.GrokInput(input, st); SetValue("user:communication", new CommunicationValue(comm), comm.ToString()); }
public List <Dictionary <string, string> > readLogFile() { List <Dictionary <string, string> > dicArray = new List <Dictionary <string, string> >(); string text = System.IO.File.ReadAllText(@".\Files\apache_logs.txt"); string[] array = text.Split(new char[] { '\n' }); Grok grok = new Grok("%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request}HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"); foreach (var line in array) { var response = grok.Parse(line); Dictionary <string, string> dic = new Dictionary <string, string>(); foreach (var match in response) { dic.Add(match.Key, match.Value?.ToString()); } dicArray.Add(dic); } return(dicArray); }
public void TestAddFieldCount() { string grokJson = @"{ ""TimberWinR"":{ ""Filters"":[ { ""grok"":{ ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" ], ""remove_tag"":[ ""tag1"" ], ""add_field"":[ ""host"", ""%{ComputerName}"" ] } }] } }"; Configuration c = Configuration.FromString(grokJson); Grok grok = c.Filters.First() as Grok; Assert.IsTrue(grok.AddField.Length % 2 == 0); }
public void Parse_Pattern_With_Type_Should_Parse_To_Specified_Type() { // Arrange const int intValue = 28; const double floatValue = 3000.5F; var dateTime = new DateTime(2010, 10, 10); var sut = new Grok("%{INT:int_value:int}:%{DATESTAMP:date_time:datetime}:%{FLOAT:float_value:float}", ReadCustomFile()); // Act GrokResult grokResult = sut.Parse($"{intValue}:{dateTime:dd-MM-yyyy HH:mm:ss}:{floatValue}"); // Assert Assert.Equal("int_value", grokResult[0].Key); Assert.Equal(intValue, grokResult[0].Value); Assert.IsType <int>(grokResult[0].Value); Assert.Equal("date_time", grokResult[1].Key); Assert.Equal(dateTime, grokResult[1].Value); Assert.IsType <DateTime>(grokResult[1].Value); Assert.Equal("float_value", grokResult[2].Key); Assert.Equal(floatValue, grokResult[2].Value); Assert.IsType <double>(grokResult[2].Value); // Float converts to double actually }
public void TestAddTagCount() { string grokJson = @"{ ""TimberWinR"":{ ""Filters"":[ { ""grok"":{ ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" ], ""add_tag"":[ ""rn_%{RecordNumber}"", ""bar"" ] } }] } }"; Configuration c = Configuration.FromString(grokJson); Grok grok = c.Filters.First() as Grok; Assert.IsTrue(grok.AddTag.Length >= 1); }
public void Parse_MonthDay_Pattern() { // Arrange const string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165 06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------"; const string month = "06"; const string day = "21"; const string year = "19"; var sut = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}"); // Act GrokResult grokResult = sut.Parse(logs); // Assert Assert.Equal("month", grokResult[0].Key); Assert.Equal(month, grokResult[0].Value); Assert.Equal("day", grokResult[1].Key); Assert.Equal(day, grokResult[1].Value); Assert.Equal("year", grokResult[2].Key); Assert.Equal(year, grokResult[2].Value); Assert.Equal("month", grokResult[8].Key); Assert.Equal(month, grokResult[8].Value); Assert.Equal("day", grokResult[9].Key); Assert.Equal(day, grokResult[9].Value); Assert.Equal("year", grokResult[10].Key); Assert.Equal(year, grokResult[10].Value); }
public void NegativeResult() { var grok = new Grok("%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}"); var response = grok.ParseLine("gary mail \"never quit\""); Assert.Equal(0, response.Captures.Count); }
public void ParseEmptyTest() { Grok act = new Grok(""); GrokResult result = act.Parse(""); Assert.AreNotEqual(null, result); Assert.AreEqual(0, result.Count); }
public void TypedResult() { var grok = new Grok("%{WORD:name} %{INT:age:int} %{QUOTEDSTRING:motto}"); var response = grok.ParseLine("gary 1 \"never quit\""); Assert.Equal(3, response.Captures.Count); Assert.Equal("age", response.Captures[1].Item1); Assert.Equal(typeof(int).ToString(), response.Captures[1].Item2.GetType().ToString()); }
public void SingleResults(string grokString, string line, string extractName, string extractData) { var grok = new Grok(grokString); var response = grok.ParseLine(line); Assert.Equal(1, response.Captures.Count); Assert.Equal(extractName, response.Captures[0].Item1); Assert.Equal(extractData, response.Captures[0].Item2.ToString()); }
public void Exception_When_Parsing_NullGrokPattern() { // Arrange const string grokPattern = null; const string logs = ""; var sut = new Grok(grokPattern); // Act, Assert Assert.Throws <ArgumentNullException>(() => sut.Parse(logs)); }
public void TimePatternTest() { Grok act = new Grok("%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}"); string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165 06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------"; GrokResult grokResult = act.Parse(logs); Assert.Equal("21:00:13:589241", grokResult[3].Value); Assert.Equal("21:00:13:589265", grokResult[11].Value); }
public void ParseEmptyTest() { string grokPattern = ""; Grok act = new Grok(grokPattern); string logs = ""; GrokResult grokResult = act.Parse(logs); Assert.NotNull(grokResult); Assert.Empty(grokResult); }
private void ProcessString(List <Dictionary <string, object> > result) { Grok grok = GrokInstance(); var lines = Input.Split(new[] { Environment.NewLine }, IgnoreEmptyLines.IsPresent ? StringSplitOptions.RemoveEmptyEntries : StringSplitOptions.None); foreach (string line in lines) { ProcessLine(grok, line, result); } }
public void PatternCountTest() { string grokPattern = "%{MONTHDAY:month}-%{MONTHDAY:day}-%{MONTHDAY:year} %{TIME:timestamp};%{WORD:id};%{LOGLEVEL:loglevel};%{WORD:func};%{GREEDYDATA:msg}"; Grok act = new Grok(grokPattern); string logs = @"06-21-19 21:00:13:589241;15;INFO;main;DECODED: 775233900043 DECODED BY: 18500738 DISTANCE: 1.5165 06-21-19 21:00:13:589265;156;WARN;main;DECODED: 775233900043 EMPTY DISTANCE: --------"; const int elements = 16; GrokResult grokResult = act.Parse(logs); Assert.Equal(elements, grokResult.Count); }
public void MacPatternTest(string macAddress) { const string timestamp = "21:00:13:589241"; string logs = $"{macAddress}:{timestamp}"; string grokPattern = "%{MAC:mac}:%{TIME:timestamp}"; Grok act = new Grok(grokPattern); GrokResult grokResult = act.Parse(logs); Assert.Equal(macAddress, grokResult[0].Value); Assert.Equal(timestamp, grokResult[1].Value); }
public void HostnamePatternTest(string hostName) { const string timestamp = "21:00:13:589241"; string logs = $"{hostName}:{timestamp}"; string grokPattern = "%{HOSTNAME:host}:%{TIME:timestamp}"; Grok act = new Grok(grokPattern); GrokResult grokResult = act.Parse(logs); Assert.Equal(hostName, grokResult[0].Value); Assert.Equal(timestamp, grokResult[1].Value); }
[InlineData("::")] // Default route public void IPv6PatternTest(string ipAddress) { const string comment = "Free as in Free Beer"; string logs = $"{ipAddress}:{comment}"; string grokPattern = "%{IPV6:IP}:%{GREEDYDATA:comment}"; Grok act = new Grok(grokPattern); GrokResult grokResult = act.Parse(logs); Assert.Equal(ipAddress, grokResult[0].Value); Assert.Equal(comment, grokResult[1].Value); }
public void Parse_With_Type_Parse_Exception_Should_Ignore_Type(string regex, string parse, string toType) { // Arrange var sut = new Grok($"%{{{regex}:{nameof(parse)}:{toType}}}", ReadCustomFile()); // Act GrokResult grokResult = sut.Parse($"{parse}"); // Assert Assert.Equal(nameof(parse), grokResult[0].Key); Assert.Equal(parse, grokResult[0].Value); }
public void EmailPatternTest() { Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}"); string logs = @"[email protected]:Free as in Free Beer"; GrokResult grokResult = act.Parse(logs); Assert.Equal("*****@*****.**", grokResult[0].Value); Assert.Equal("Free as in Free Beer", grokResult[1].Value); Assert.Equal("email", grokResult[0].Key); Assert.Equal("comment", grokResult[1].Key); }
public void MulipleResults() { var grok = new Grok("%{WORD:name} %{INT:age} %{QUOTEDSTRING:motto}"); var response = grok.ParseLine("gary 25 \"never quit\""); Assert.Equal(3, response.Captures.Count); Assert.Equal("name", response.Captures[0].Item1); Assert.Equal("gary", response.Captures[0].Item2.ToString()); Assert.Equal("age", response.Captures[1].Item1); Assert.Equal("25", response.Captures[1].Item2.ToString()); Assert.Equal("motto", response.Captures[2].Item1); Assert.Equal("\"never quit\"", response.Captures[2].Item2.ToString()); }
[InlineData("::")] // Default route public void Parse_IPv6_Pattern(string ipAddress) { // Arrange const string comment = "Free as in Free Beer"; var sut = new Grok("%{IPV6:IP}:%{GREEDYDATA:comment}"); // Act GrokResult grokResult = sut.Parse($"{ipAddress}:{comment}"); // Assert Assert.Equal(ipAddress, grokResult[0].Value); Assert.Equal(comment, grokResult[1].Value); }
public void Parse_Hostname_Pattern(string hostName) { // Arrange const string timestamp = "21:00:13:589241"; var sut = new Grok("%{HOSTNAME:host}:%{TIME:timestamp}"); // Act GrokResult grokResult = sut.Parse($"{hostName}:{timestamp}"); // Assert Assert.Equal(hostName, grokResult[0].Value); Assert.Equal(timestamp, grokResult[1].Value); }
public void Parse_Mac_Pattern(string macAddress) { // Arrange const string timestamp = "21:00:13:589241"; var sut = new Grok("%{MAC:mac}:%{TIME:timestamp}"); // Act GrokResult grokResult = sut.Parse($"{macAddress}:{timestamp}"); // Assert Assert.Equal(macAddress, grokResult[0].Value); Assert.Equal(timestamp, grokResult[1].Value); }
public void EmailPatternTest() { Grok act = new Grok("%{EMAILADDRESS:email}:%{GREEDYDATA:comment}"); const string emailAddress = "*****@*****.**"; const string comment = "Free as in Free Beer"; string logs = emailAddress + ":" + comment; GrokResult grokResult = act.Parse(logs); Assert.Equal(emailAddress, grokResult[0].Value); Assert.Equal(comment, grokResult[1].Value); Assert.Equal("email", grokResult[0].Key); Assert.Equal("comment", grokResult[1].Key); }
private void ProcessFile(List <Dictionary <string, object> > result) { Grok grok = GrokInstance(); using (FileStream fileStream = File.OpenRead(Path)) using (var streamReader = new StreamReader(fileStream, Encoding.UTF8, true)) { string line; while ((line = streamReader.ReadLine()) != null) { ProcessLine(grok, line, result); } } }
public void Parse_Empty_Logs_Not_Throws() { // Arrange const string grokPattern = ""; const string logs = ""; var sut = new Grok(grokPattern); // Act GrokResult grokResult = sut.Parse(logs); // Assert Assert.NotNull(grokResult); Assert.Empty(grokResult); }