public void ValidateIndexedFieldReference()
{
KqlNode node = new KqlNode();
// Get the sample data for 1116 AntiMalware
var path = Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(path);
string eventXmlOf1116AntiMalware = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "1116_AntiMalware.xml"));
string comment =
"/\'DDID 543: SIM-00035: GLOBAL - 1116-1119 - Microsoft Antimalware\', folder = @\'DetectionFunctionBuildout/WEC/Production/Global\')";
string query =
"cluster(\"CDOC\").database(\"WEC\").SecurityLog\r\n | where Provider == \"Microsoft Antimalware\" and EventId in (1116, 1117, 1118, 1119)\r\n | extend FilePath = EventData.[\"22\"], SourceProcessName = EventData.[\"19\"], SourceUserName = EventData.[\"20\"], ThreatName = EventData.[\"08\"] \r\n | where not(ThreatName endswith \"EICAR_Test_File\" and Computer startswith \"WU2SGRPVT\")\r\n\t| where not(ThreatName == \"Trojan:Win32/Peals.F!cl\")\r\n\t| where not(SourceProcessName contains \":\\\\data\\\\Perf\\\\\" or FilePath contains \":\\\\data\\\\Perf\\\\\")\r\n | where not((FilePath contains \"C:\\\\Users\\\\L4Test\" or FilePath contains \"D:\\\\Users\\\\L4Test\") and SourceUserName startswith \"L4TestUser\")\r\n | where not(FilePath contains \"NGFMuploads\" and Computer in (\"CO1MSSDTMLFS11.phx.gbl\", \"CO1MSSDTMLFS12.phx.gbl\", \"CO1MSSDTMLFS13.phx.gbl\", \"CO1MSSDTMLFS14.phx.gbl\", \"DB3MSSDTMLFS11.phx.gbl\", \"DB3MSSDTMLFS12.phx.gbl\", \"DB3MSSDTMLFS13.phx.gbl\", \"DB3MSSDTMLFS14.phx.gbl\", \"SG2MSSDTMLFS11.phx.gbl\", \"SG2MSSDTMLFS12.phx.gbl\", \"SG2MSSDTMLFS13.phx.gbl\", \"SG2MSSDTMLFS14.phx.gbl\"))\r\n | where not(FilePath startswith \"file:_D:\\\\http\\\\security\\\\encyclopedia\\\\en-us\\\\entries\\\\\" or FilePath startswith \"file:_E:\\\\Services\\\\HostIDS\\\\\" or FilePath startswith \"file:_E:\\\\AzureMAStore\\\\\" or FilePath startswith \"file:_E:\\\\Services\\\\WLS_Colorado\\\\\" or FilePath startswith \"file:_C:\\\\Windows\\\\System32\\\\config\\\\systemprofile\\\\AppData\\\\Local\\\\Microsoft\\\\Windows\\\\\")\r\n | where not(SourceProcessName endswith \"agent\\\\MonAgentCore.exe\" or SourceProcessName endswith \"service\\\\MonAgentCore.exe\")\r\n | where not((FilePath contains \":\\\\Temp\\\\Website\\\\\" and Computer contains \"MSDN\") or FilePath contains \"\\\\Users\\\\YarnppNMUser\\\\AppData\\\\Local\\\\Temp\\\\\")\r\n";
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = comment,
Query = query
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the sucessful detections.
var list = new List <object>();
node.Subscribe(evt => { list.Add(evt); });
// Add the detections.
var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf1116AntiMalware);
node.OnNext(eventDynamic);
}
public void DateTimeNumericExpressionTests()
{
string jsonText = " {\r\n \"FirstDate\": \"2018-12-10T13:45:00.000Z\",\r\n \"SecondDate\": \"2018-12-10T14:45:00.000Z\"\r\n }";
Dictionary <string, object> jsonDictionary = JsonConvert.DeserializeObject <Dictionary <string, object> >(jsonText);
KqlNode node = new KqlNode();
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = "Determine Subnet",
Query = "SIEMfx " +
"| extend TimeDiff = SecondDate - FirstDate" +
"| where TimeDiff == \"01:00:00\""
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the successful detections.
var results = new List <IDictionary <string, object> >();
node.Subscribe(evt => { results.Add(evt.Output); });
// Add the execute on the simple IP.
node.OnNext(jsonDictionary);
// Make sure ALL of the above KqlQuery objects return detections
Assert.IsTrue(results.Count == node.KqlQueryList.Count);
}
public void DetectionInfoQueries()
{
KqlNode node = new KqlNode();
// deserialize JSON to the runtime type, and iterate.
var path = Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(path);
List <KqlQuery> newDetectionInfos = new List <KqlQuery>();
newDetectionInfos.Add(new KqlQuery
{
Comment = "// GLOBAL - 1102 - Audit Log Cleared - SIM-00014",
Query =
"SecurityLog | where EventId == 1102 | extend SubjectUserName = EventData.SubjectUserName, SubjectDomainName = EventData.SubjectDomainName | project TimeCreated, Computer, SubjectUserName, SubjectDomainName"
});
node.AddKqlQueryList(newDetectionInfos, true);
// Subscribe to the sucessful detections.
var list = new List <object>();
node.Subscribe(evt => { list.Add(evt); });
string eventXmlOf1102 =
"<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event' xml:lang='en-US'><System><Provider Name='Microsoft-Windows-Eventlog' Guid='{fc65ddd8-d6ef-4962-83d5-6e5cfe9ce148}'/><EventID>1102</EventID><Version>0</Version><Level>4</Level><Task>104</Task><Opcode>0</Opcode><Keywords>0x4020000000000000</Keywords><TimeCreated SystemTime='2017-08-03T17:11:29.255592600Z'/><EventRecordID>36837151</EventRecordID><Correlation/><Execution ProcessID='996' ThreadID='11180'/><Channel>Security</Channel><Computer>GFTVMHostDev.redmond.corp.microsoft.com</Computer><Security/></System><UserData><LogFileCleared xmlns='http://manifests.microsoft.com/win/2004/08/windows/eventlog'><SubjectUserSid>S-1-5-21-2127521184-1604012920-1887927527-9916173</SubjectUserSid><SubjectUserName>rbiles</SubjectUserName><SubjectDomainName>REDMOND</SubjectDomainName><SubjectLogonId>0x34d1b1eb</SubjectLogonId></LogFileCleared></UserData></Event>";
// Add the detections.
var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf1102);
node.OnNext(eventDynamic);
}
public void NumericExpressionTests()
{
string jsonText = "{\r\n \"SrcIp\": \"12.34.45.245\"\r\n}";
Dictionary <string, object> jsonDictionary = JsonConvert.DeserializeObject <Dictionary <string, object> >(jsonText);
KqlNode node = new KqlNode();
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = "Determine Subnet",
Query = "SIEMfx " +
"| extend SrcSubnet = substring(SrcIp, 0, strlen(SrcIp) - indexof(reverse(SrcIp), \".\")) " +
"| where SrcSubnet == \"12.34.45.\""
});
newDetections.Add(new KqlQuery
{
Comment = "Determine Subnet another way with extend statement calculation cascade",
Query = "SIEMfx " +
"| extend three = toint(3) " +
"| extend four = 4 " +
"| extend SrcSubnet = substring(SrcIp, 0, (three * four) - three) " +
"| where SrcSubnet == \"12.34.45.\""
});
newDetections.Add(new KqlQuery
{
Comment = "Correctly calculate results based on valid PEMDAS",
Query = "SIEMfx " +
"| extend five = 5 " +
"| extend two = 2 " +
"| extend three = five - two " +
"| extend fortyeight = 8 / 2 * (2+2) * three " +
"| where fortyeight == 48"
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the sucessful detections.
var results = new List <IDictionary <string, object> >();
node.Subscribe(evt => { results.Add(evt.Output); });
// Add the execute on the simple IP.
node.OnNext(jsonDictionary);
// Make sure ALL of the above KqlQuery objects return detections
Assert.IsTrue(results.Count == node.KqlQueryList.Count);
}
public void ProjectExtendValidation()
{
KqlNode node = new KqlNode();
// Get the sample data for 4688 Process Create
var path = Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(path);
string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml"));
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = "This works",
Query = "Security | where EventId == 4688 | extend ProcessName = EventData.NewProcessName | extend UserName = EventData.SubjectUserName | project ProcessName,UserName"
});
newDetections.Add(new KqlQuery
{
Comment = "This also should work",
Query = "Security | where EventId == 4688 | project ProcessName = EventData.NewProcessName, UserName = EventData.SubjectUserName"
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the sucessful detections.
var results = new List <IDictionary <string, object> >();
node.Subscribe(evt =>
{
results.Add(evt.Output);
});
// Add the detections.
var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688);
node.OnNext((IDictionary <string, object>)eventDynamic);
Assert.IsTrue(results.Count == 2);
// Make sure BOTH return the same values
var x = results.ToArray();
string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe";
string userNameValue = "RUSSELLHPDEV$";
Assert.AreEqual(x[0]["ProcessName"], processNameValue);
Assert.AreEqual(x[0]["UserName"], userNameValue);
Assert.AreEqual(x[1]["ProcessName"], processNameValue);
Assert.AreEqual(x[1]["UserName"], userNameValue);
}
public void RandomQueryDetectionInfo()
{
KqlNode node = new KqlNode();
KqlQuery detectionItem = new KqlQuery
{
Comment = "// GLOBAL - 1102 - Audit Log Cleared - SIM-00014",
Query =
"cluster('CDOCC').database('WEC').SecurityLog | where Provider == 'Microsoft-Windows-Security-Auditing' and EventId == 4728 | extend MemberName = EventData.MemberName, MemberSid = EventData.MemberSid, TargetUserName = EventData.TargetUserName | where MemberName contains 'Domain Computers' or (MemberSid startswith 'S-1-5-21-' and MemberSid endswith '-515') or MemberName contains 'Domain Users' or (MemberSid startswith 'S-1-5-21-' and MemberSid endswith '-513')"
};
Query q = new Query(detectionItem.Query);
List<KqlQuery> newDetectionInfos = new List<KqlQuery>();
newDetectionInfos.Add(detectionItem);
node.AddKqlQueryList(newDetectionInfos, true);
}
public void ProjectExtendChainedValidation()
{
KqlNode node = new KqlNode();
// Get the sample data for 4688 Process Create
var path = Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(path);
string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml"));
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = "An extended projected ordeal...",
Query = "Security " +
"| where EventId == 4688 " +
"| extend processName = EventData.NewProcessName" +
"| project newName = processName " +
"| extend extendedName = newName" +
"| project finalProcessNameResult = extendedName"
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the sucessful detections.
var results = new List <IDictionary <string, object> >();
node.Subscribe(evt =>
{
results.Add(evt.Output);
});
// Add the detections.
var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688);
node.OnNext(eventDynamic);
Assert.IsTrue(results.Count == 1);
// Make sure BOTH return the same values
var x = results.ToArray();
string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe";
Assert.AreEqual(x[0]["finalProcessNameResult"], processNameValue);
}
public void AddingDetectionMultipleItems()
{
KqlNode node = new KqlNode();
List<KqlQuery> newDetectionInfos = new List<KqlQuery>();
newDetectionInfos.Add(new KqlQuery
{
Comment = "// GLOBAL - 1102 - Audit Log Cleared - SIM-00014",
Query =
"SecurityLog | where EventId == 1102 | extend SubjectUserName = EventData.SubjectUserName, SubjectDomainName = EventData.SubjectDomainName | project TimeCreated, Computer, SubjectUserName, SubjectDomainName"
});
newDetectionInfos.Add(new KqlQuery
{
Comment = "// GLOBAL - 1103 ",
Query = "SecurityLog | where EventId == 1103"
});
node.AddKqlQueryList(newDetectionInfos, true);
if (node.KqlQueryList.Count != 2)
{
throw new Exception();
}
}
private static bool Test(
string kql,
Func <IEnumerable <StockQuote>, IEnumerable <Summary> > linq,
StockQuote[] quotes)
{
KqlNode node = new KqlNode();
node.AddKqlQuery(new KqlQuery
{
Comment = "Blank Comment",
Query = kql
});
Console.WriteLine(kql);
Console.WriteLine();
var linqResult = linq(quotes)
.Select(a => string.Format("{0} {1} {2}", a.Time, a.Symbol, a.Result))
.ToArray();
List <string> kqlResult = new List <string>();
var obs = quotes.ToObservable()
.ToDynamic(e => e);
KqlNode kqlNode = new KqlNode();
List <KqlQuery> kustoQueryUserInput = new List <KqlQuery>
{
new KqlQuery
{
Comment = "Blank Comment",
Query = kql.Trim()
}
};
kqlNode.AddKqlQueryList(kustoQueryUserInput, true);
ManualResetEvent completedEvent = new ManualResetEvent(false);
kqlNode.Subscribe(evt =>
{
kqlResult.Add(string.Format("{0} {1} {2}",
((DateTime)evt.Output["Time"]),
evt.Output["Symbol"],
evt.Output["r"]));
},
() =>
{
completedEvent.Set();
});
using (obs.Subscribe(kqlNode))
{
completedEvent.WaitOne();
kqlNode.OnCompleted();
}
Console.WriteLine(" LINQ Rx.KQL");
for (int i = 0; i < linqResult.Length; i++)
{
Console.WriteLine("{0}\t{1}", linqResult[i], kqlResult[i]);
}
return(linqResult.Length == kqlResult.Count);
}
public void ProjectWithConstants()
{
KqlNode node = new KqlNode();
// Get the sample data for 4688 Process Create
var path = Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(path);
string eventXmlOf4688 = File.ReadAllText(Path.Combine(directory, "ExampleEventXml", "Sample4688.xml"));
List <KqlQuery> newDetections = new List <KqlQuery>();
newDetections.Add(new KqlQuery
{
Comment = "An extended project with constants...",
Query = "Security " +
"| where EventId == 4688 " +
"| extend processName = EventData.NewProcessName" +
"| project Computer, TimeCreated, A = 1, B = true, \"two\", \"two\", \"two\", \"two\", \"five\""
});
newDetections.Add(new KqlQuery
{
Comment = "An project with constants and a function...",
Query = "Security " +
"| where EventId == 4688 " +
"| extend processName = EventData.NewProcessName" +
"| project Computer, TimeCreated, \"one\", \"two\", \"three\", \"four\", \"five\", X1 = tolower(\"SIX\")"
});
newDetections.Add(new KqlQuery
{
Comment = "An project with only constants...",
Query = "Security " +
"| where EventId == 4688 " +
"| extend processName = EventData.NewProcessName" +
"| project \"one\", \"two\", \"three\", 1024"
});
node.AddKqlQueryList(newDetections, true);
// Subscribe to the sucessful detections.
var results = new List <IDictionary <string, object> >();
node.Subscribe(evt => { results.Add(evt.Output); });
// Add the detections.
var eventDynamic = EvtxExtensions.Deserialize(eventXmlOf4688);
node.OnNext(eventDynamic);
Assert.IsTrue(results.Count == 3);
// Make sure BOTH return the same values
var x = results.ToArray();
string processNameValue = "C:\\Windows\\System32\\backgroundTaskHost.exe";
Assert.AreEqual(x[0]["A"], 1L);
Assert.AreEqual(x[0]["B"], true);
Assert.AreEqual(x[0]["Column2"], "two");
Assert.AreEqual(x[0]["Column4"], "two");
Assert.AreEqual(x[0]["Column5"], "five");
Assert.AreEqual(x[1]["Computer"], "RussellHPDev.redmond.corp.microsoft.com");
Assert.AreEqual(x[1]["Column1"], "one");
Assert.AreEqual(x[1]["Column2"], "two");
Assert.AreEqual(x[1]["Column3"], "three");
Assert.AreEqual(x[1]["Column4"], "four");
Assert.AreEqual(x[1]["Column5"], "five");
Assert.AreEqual(x[1]["X1"], "six");
Assert.AreEqual(x[2]["Column1"], "one");
Assert.AreEqual(x[2]["Column2"], "two");
Assert.AreEqual(x[2]["Column3"], "three");
Assert.AreEqual(x[2]["Column4"], 1024L);
}