public void ValidateIndexedFieldReference() { KqlNode node = new KqlNode(); // Get the sample data for 1116 AntiMalware var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); string eventXmlOf1116AntiMalware = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "1116_AntiMalware.xml")); string comment = "/\'DDID 543: SIM-00035: GLOBAL - 1116-1119 - Microsoft Antimalware\', folder = @\'DetectionFunctionBuildout/WEC/Production/Global\')"; string query = "cluster(\"CDOC\").database(\"WEC\").SecurityLog\r\n | where Provider == \"Microsoft Antimalware\" and EventId in (1116, 1117, 1118, 1119)\r\n | extend FilePath = EventData.[\"22\"], SourceProcessName = EventData.[\"19\"], SourceUserName = EventData.[\"20\"], ThreatName = EventData.[\"08\"] \r\n | where not(ThreatName endswith \"EICAR_Test_File\" and Computer startswith \"WU2SGRPVT\")\r\n\t| where not(ThreatName == \"Trojan:Win32/Peals.F!cl\")\r\n\t| where not(SourceProcessName contains \":\\\\data\\\\Perf\\\\\" or FilePath contains \":\\\\data\\\\Perf\\\\\")\r\n | where not((FilePath contains \"C:\\\\Users\\\\L4Test\" or FilePath contains \"D:\\\\Users\\\\L4Test\") and SourceUserName startswith \"L4TestUser\")\r\n | where not(FilePath contains \"NGFMuploads\" and Computer in (\"CO1MSSDTMLFS11.phx.gbl\", \"CO1MSSDTMLFS12.phx.gbl\", \"CO1MSSDTMLFS13.phx.gbl\", \"CO1MSSDTMLFS14.phx.gbl\", \"DB3MSSDTMLFS11.phx.gbl\", \"DB3MSSDTMLFS12.phx.gbl\", \"DB3MSSDTMLFS13.phx.gbl\", \"DB3MSSDTMLFS14.phx.gbl\", \"SG2MSSDTMLFS11.phx.gbl\", \"SG2MSSDTMLFS12.phx.gbl\", \"SG2MSSDTMLFS13.phx.gbl\", \"SG2MSSDTMLFS14.phx.gbl\"))\r\n | where not(FilePath startswith \"file:_D:\\\\http\\\\security\\\\encyclopedia\\\\en-us\\\\entries\\\\\" or FilePath startswith \"file:_E:\\\\Services\\\\HostIDS\\\\\" or FilePath startswith \"file:_E:\\\\AzureMAStore\\\\\" or FilePath startswith \"file:_E:\\\\Services\\\\WLS_Colorado\\\\\" or FilePath startswith \"file:_C:\\\\Windows\\\\System32\\\\config\\\\systemprofile\\\\AppData\\\\Local\\\\Microsoft\\\\Windows\\\\\")\r\n | where not(SourceProcessName endswith \"agent\\\\MonAgentCore.exe\" or SourceProcessName endswith \"service\\\\MonAgentCore.exe\")\r\n | where not((FilePath contains \":\\\\Temp\\\\Website\\\\\" and Computer contains \"MSDN\") or FilePath contains \"\\\\Users\\\\YarnppNMUser\\\\AppData\\\\Local\\\\Temp\\\\\")\r\n"; List<KqlQuery> newDetections = new List<KqlQuery>(); newDetections.Add(new KqlQuery { Comment = comment, Query = query }); node.AddKqlQueryList(newDetections, true); // Subscribe to the sucessful detections. var list = new List<object>(); node.Subscribe(evt => { list.Add(evt); }); // Add the detections. var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf1116AntiMalware); node.OnNext(eventDynamic); }
public void DateTimeNumericExpressionTests() { string jsonText = " {\r\n \"FirstDate\": \"2018-12-10T13:45:00.000Z\",\r\n \"SecondDate\": \"2018-12-10T14:45:00.000Z\"\r\n }"; Dictionary <string, object> jsonDictionary = JsonConvert.DeserializeObject <Dictionary <string, object> >(jsonText); KqlNode node = new KqlNode(); List <KqlQuery> newDetections = new List <KqlQuery>(); newDetections.Add(new KqlQuery { Comment = "Determine Subnet", Query = "SIEMfx " + "| extend TimeDiff = SecondDate - FirstDate" + "| where TimeDiff == \"01:00:00\"" }); node.AddKqlQueryList(newDetections, true); // Subscribe to the successful detections. var results = new List <IDictionary <string, object> >(); node.Subscribe(evt => { results.Add(evt.Output); }); // Add the execute on the simple IP. node.OnNext(jsonDictionary); // Make sure ALL of the above KqlQuery objects return detections Assert.IsTrue(results.Count == node.KqlQueryList.Count); }
public void DetectionInfoQueries() { KqlNode node = new KqlNode(); // deserialize JSON to the runtime type, and iterate. var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); List<KqlQuery> newDetectionInfos = new List<KqlQuery>(); newDetectionInfos.Add(new KqlQuery { Comment = "// GLOBAL - 1102 - Audit Log Cleared - SIM-00014", Query = "SecurityLog | where EventId == 1102 | extend SubjectUserName = EventData.SubjectUserName, SubjectDomainName = EventData.SubjectDomainName | project TimeCreated, Computer, SubjectUserName, SubjectDomainName" }); node.AddKqlQueryList(newDetectionInfos, true); // Subscribe to the sucessful detections. var list = new List<object>(); node.Subscribe(evt => { list.Add(evt); }); string eventXmlOf1102 = "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event' xml:lang='en-US'><System><Provider Name='Microsoft-Windows-Eventlog' Guid='{fc65ddd8-d6ef-4962-83d5-6e5cfe9ce148}'/><EventID>1102</EventID><Version>0</Version><Level>4</Level><Task>104</Task><Opcode>0</Opcode><Keywords>0x4020000000000000</Keywords><TimeCreated SystemTime='2017-08-03T17:11:29.255592600Z'/><EventRecordID>36837151</EventRecordID><Correlation/><Execution ProcessID='996' ThreadID='11180'/><Channel>Security</Channel><Computer>GFTVMHostDev.redmond.corp.microsoft.com</Computer><Security/></System><UserData><LogFileCleared xmlns='http://manifests.microsoft.com/win/2004/08/windows/eventlog'><SubjectUserSid>S-1-5-21-2127521184-1604012920-1887927527-9916173</SubjectUserSid><SubjectUserName>rbiles</SubjectUserName><SubjectDomainName>REDMOND</SubjectDomainName><SubjectLogonId>0x34d1b1eb</SubjectLogonId></LogFileCleared></UserData></Event>"; // Add the detections. var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf1102); node.OnNext(eventDynamic); }
public void NumericExpressionTests() { string jsonText = "{\r\n \"SrcIp\": \"12.34.45.245\"\r\n}"; Dictionary <string, object> jsonDictionary = JsonConvert.DeserializeObject <Dictionary <string, object> >(jsonText); KqlNode node = new KqlNode(); List <KqlQuery> newDetections = new List <KqlQuery>(); newDetections.Add(new KqlQuery { Comment = "Determine Subnet", Query = "SIEMfx " + "| extend SrcSubnet = substring(SrcIp, 0, strlen(SrcIp) - indexof(reverse(SrcIp), \".\")) " + "| where SrcSubnet == \"12.34.45.\"" }); newDetections.Add(new KqlQuery { Comment = "Determine Subnet another way with extend statement calculation cascade", Query = "SIEMfx " + "| extend three = toint(3) " + "| extend four = 4 " + "| extend SrcSubnet = substring(SrcIp, 0, (three * four) - three) " + "| where SrcSubnet == \"12.34.45.\"" }); newDetections.Add(new KqlQuery { Comment = "Correctly calculate results based on valid PEMDAS", Query = "SIEMfx " + "| extend five = 5 " + "| extend two = 2 " + "| extend three = five - two " + "| extend fortyeight = 8 / 2 * (2+2) * three " + "| where fortyeight == 48" }); node.AddKqlQueryList(newDetections, true); // Subscribe to the sucessful detections. var results = new List <IDictionary <string, object> >(); node.Subscribe(evt => { results.Add(evt.Output); }); // Add the execute on the simple IP. node.OnNext(jsonDictionary); // Make sure ALL of the above KqlQuery objects return detections Assert.IsTrue(results.Count == node.KqlQueryList.Count); }
public void ProjectExtendValidation() { KqlNode node = new KqlNode(); // Get the sample data for 4688 Process Create var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml")); List <KqlQuery> newDetections = new List <KqlQuery>(); newDetections.Add(new KqlQuery { Comment = "This works", Query = "Security | where EventId == 4688 | extend ProcessName = EventData.NewProcessName | extend UserName = EventData.SubjectUserName | project ProcessName,UserName" }); newDetections.Add(new KqlQuery { Comment = "This also should work", Query = "Security | where EventId == 4688 | project ProcessName = EventData.NewProcessName, UserName = EventData.SubjectUserName" }); node.AddKqlQueryList(newDetections, true); // Subscribe to the sucessful detections. var results = new List <IDictionary <string, object> >(); node.Subscribe(evt => { results.Add(evt.Output); }); // Add the detections. var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688); node.OnNext((IDictionary <string, object>)eventDynamic); Assert.IsTrue(results.Count == 2); // Make sure BOTH return the same values var x = results.ToArray(); string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe"; string userNameValue = "RUSSELLHPDEV$"; Assert.AreEqual(x[0]["ProcessName"], processNameValue); Assert.AreEqual(x[0]["UserName"], userNameValue); Assert.AreEqual(x[1]["ProcessName"], processNameValue); Assert.AreEqual(x[1]["UserName"], userNameValue); }
public void ProjectExtendChainedValidation() { KqlNode node = new KqlNode(); // Get the sample data for 4688 Process Create var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml")); List <KqlQuery> newDetections = new List <KqlQuery>(); newDetections.Add(new KqlQuery { Comment = "An extended projected ordeal...", Query = "Security " + "| where EventId == 4688 " + "| extend processName = EventData.NewProcessName" + "| project newName = processName " + "| extend extendedName = newName" + "| project finalProcessNameResult = extendedName" }); node.AddKqlQueryList(newDetections, true); // Subscribe to the sucessful detections. var results = new List <IDictionary <string, object> >(); node.Subscribe(evt => { results.Add(evt.Output); }); // Add the detections. var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688); node.OnNext(eventDynamic); Assert.IsTrue(results.Count == 1); // Make sure BOTH return the same values var x = results.ToArray(); string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe"; Assert.AreEqual(x[0]["finalProcessNameResult"], processNameValue); }
protected List <dynamic> RunAtomicQueryWithKqlNode(dynamic eventDynamic, string query) { var result = new List <dynamic>(); KqlNode node = new KqlNode(); node.AddKqlQuery(new KqlQuery { Comment = "TestQuery", Query = query }); // Subscribe to the sucessful detections. node.Subscribe(evt => { result.Add(evt.Output); }); // Add the detections. node.OnNext((IDictionary <string, object>)eventDynamic); return(result); }
public void SimpleQueries() { KqlNode node = new KqlNode(); // deserialize JSON to the runtime type, and iterate. var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); node.AddCslFile(Path.Combine(directory, "SimpleQueries.csl")); // Subscribe to the sucessful detections. var list = new List<object>(); node.Subscribe(evt => { list.Add(evt); }); // Add the detections. for (int i = 0; i < 10; i++) { dynamic evt = new ExpandoObject(); evt.Seq = i; node.OnNext((IDictionary<string, object>) evt); } }
public void FunctionQueries() { KqlNode node = new KqlNode(); // deserialize JSON to the runtime type, and iterate. var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); node.AddCslFile(Path.Combine(directory, "KqlFunctionTestFiles", "Rule_4720_UsrAcctCreation_WecExtract.csl")); Debug.Assert(GlobalFunctions.KqlFunctions.Count == 3, "Rx.Kql FILTER Functions are not loading correctly from CSL files!"); Debug.Assert(node.KqlQueryList.Count > 0 || node.FailedKqlQueryList.Count == 0, "Kql query failed to load. There is an Rx.Kql parsing bug!"); string evt4720 = "<Event xmlns=\'http://schemas.microsoft.com/win/2004/08/events/event\' xml:lang=\'en-US\'><System><Provider Name=\'Microsoft-Windows-Security-Auditing\' Guid=\'{54849625-5478-4994-A5BA-3E3B0328C30D}\'/><EventID>4720</EventID><Version>0</Version><Level>0</Level><Task>13824</Task><Opcode>0</Opcode><Keywords>0x8020000000000000</Keywords><TimeCreated SystemTime=\'2017-08-31T19:38:21.509585500Z\'/><EventRecordID>2079336</EventRecordID><Correlation/><Execution ProcessID=\'2092\' ThreadID=\'42656\'/><Channel>Security</Channel><Computer>SN2SCH101140124.phx.gbl</Computer><Security/></System><EventData><Data Name=\'TargetUserName\'>QTU-bs_el_idsv-7</Data><Data Name=\'TargetDomainName\'>SN2SCH101140124</Data><Data Name=\'TargetSid\'>S-1-5-21-1266794097-2621680504-1140025688-1442</Data><Data Name=\'SubjectUserSid\'>S-1-5-21-606747145-1563985344-839522115-25776942</Data><Data Name=\'SubjectUserName\'>_qcloud1</Data><Data Name=\'SubjectDomainName\'>PHX</Data><Data Name=\'SubjectLogonId\'>0x21a3d239e</Data><Data Name=\'PrivilegeList\'>-</Data><Data Name=\'SamAccountName\'>QTU-bs_el_idsv-7</Data><Data Name=\'DisplayName\'>%%1793</Data><Data Name=\'UserPrincipalName\'>-</Data><Data Name=\'HomeDirectory\'>%%1793</Data><Data Name=\'HomePath\'>%%1793</Data><Data Name=\'ScriptPath\'>%%1793</Data><Data Name=\'ProfilePath\'>%%1793</Data><Data Name=\'UserWorkstations\'>%%1793</Data><Data Name=\'PasswordLastSet\'>%%1794</Data><Data Name=\'AccountExpires\'>%%1794</Data><Data Name=\'PrimaryGroupId\'>513</Data><Data Name=\'AllowedToDelegateTo\'>-</Data><Data Name=\'OldUacValue\'>0x0</Data><Data Name=\'NewUacValue\'>0x15</Data><Data Name=\'UserAccountControl\'>\r\n\t\t%%2080\r\n\t\t%%2082\r\n\t\t%%2084</Data><Data Name=\'UserParameters\'>%%1793</Data><Data Name=\'SidHistory\'>-</Data><Data Name=\'LogonHours\'>%%1797</Data></EventData></Event>"; dynamic eventDynamic = EvtxExtensions.Deserialize(evt4720); // Subscribe to the sucessful detections. var list = new List<object>(); node.Subscribe(evt => { list.Add(evt); }); node.OnNext((IDictionary<string, object>) eventDynamic); }
private static bool Test( string kql, Func <IEnumerable <StockQuote>, IEnumerable <Summary> > linq, StockQuote[] quotes) { KqlNode node = new KqlNode(); node.AddKqlQuery(new KqlQuery { Comment = "Blank Comment", Query = kql }); Console.WriteLine(kql); Console.WriteLine(); var linqResult = linq(quotes) .Select(a => string.Format("{0} {1} {2}", a.Time, a.Symbol, a.Result)) .ToArray(); List <string> kqlResult = new List <string>(); var obs = quotes.ToObservable() .ToDynamic(e => e); KqlNode kqlNode = new KqlNode(); List <KqlQuery> kustoQueryUserInput = new List <KqlQuery> { new KqlQuery { Comment = "Blank Comment", Query = kql.Trim() } }; kqlNode.AddKqlQueryList(kustoQueryUserInput, true); ManualResetEvent completedEvent = new ManualResetEvent(false); kqlNode.Subscribe(evt => { kqlResult.Add(string.Format("{0} {1} {2}", ((DateTime)evt.Output["Time"]), evt.Output["Symbol"], evt.Output["r"])); }, () => { completedEvent.Set(); }); using (obs.Subscribe(kqlNode)) { completedEvent.WaitOne(); kqlNode.OnCompleted(); } Console.WriteLine(" LINQ Rx.KQL"); for (int i = 0; i < linqResult.Length; i++) { Console.WriteLine("{0}\t{1}", linqResult[i], kqlResult[i]); } return(linqResult.Length == kqlResult.Count); }
public void ProjectWithConstants() { KqlNode node = new KqlNode(); // Get the sample data for 4688 Process Create var path = Assembly.GetExecutingAssembly().Location; var directory = Path.GetDirectoryName(path); string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml")); List <KqlQuery> newDetections = new List <KqlQuery>(); newDetections.Add(new KqlQuery { Comment = "An extended project with constants...", Query = "Security " + "| where EventId == 4688 " + "| extend processName = EventData.NewProcessName" + "| project Computer, TimeCreated, A = 1, B = true, \"two\", \"two\", \"two\", \"two\", \"five\"" }); newDetections.Add(new KqlQuery { Comment = "An project with constants and a function...", Query = "Security " + "| where EventId == 4688 " + "| extend processName = EventData.NewProcessName" + "| project Computer, TimeCreated, \"one\", \"two\", \"three\", \"four\", \"five\", X1 = tolower(\"SIX\")" }); newDetections.Add(new KqlQuery { Comment = "An project with only constants...", Query = "Security " + "| where EventId == 4688 " + "| extend processName = EventData.NewProcessName" + "| project \"one\", \"two\", \"three\", 1024" }); node.AddKqlQueryList(newDetections, true); // Subscribe to the sucessful detections. var results = new List <IDictionary <string, object> >(); node.Subscribe(evt => { results.Add(evt.Output); }); // Add the detections. var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688); node.OnNext(eventDynamic); Assert.IsTrue(results.Count == 3); // Make sure BOTH return the same values var x = results.ToArray(); string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe"; Assert.AreEqual(x[0]["A"], 1L); Assert.AreEqual(x[0]["B"], true); Assert.AreEqual(x[0]["Column2"], "two"); Assert.AreEqual(x[0]["Column4"], "two"); Assert.AreEqual(x[0]["Column5"], "five"); Assert.AreEqual(x[1]["Computer"], "RussellHPDev.redmond.corp.microsoft.com"); Assert.AreEqual(x[1]["Column1"], "one"); Assert.AreEqual(x[1]["Column2"], "two"); Assert.AreEqual(x[1]["Column3"], "three"); Assert.AreEqual(x[1]["Column4"], "four"); Assert.AreEqual(x[1]["Column5"], "five"); Assert.AreEqual(x[1]["X1"], "six"); Assert.AreEqual(x[2]["Column1"], "one"); Assert.AreEqual(x[2]["Column2"], "two"); Assert.AreEqual(x[2]["Column3"], "three"); Assert.AreEqual(x[2]["Column4"], 1024L); }