public static async Task CefFilesToSentinelProcessor() { //X509Certificate2 cert; GlobalLog.WriteToStringBuilderLog("Attempting to load CEF Files ", 14001); // Update to LINQ query to prevent attempting to load ALL files during an iteration. var directoryInfo = new DirectoryInfo(SentinelApiConfig.EnabledSentinelUploads.CefFileFolderToUpload); var orderedFileList = directoryInfo.EnumerateFiles("CefToSentinel*.json", SearchOption.TopDirectoryOnly) .OrderBy(d => d.LastAccessTime) .Select(d => d.FullName) .Take(25) .ToList(); if (orderedFileList.Count > 0) { foreach (string file in orderedFileList) { string jsonFinalString = File.ReadAllText(file); UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2); SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{file}] LogManagement.CommonSecurityLog messages to Sentinel.", ConsoleColor.Green); File.Delete(file); } } }
public static async Task SyslogToCustomLog() { string jsonString = JsonConvert.SerializeObject(syslogToSentinelProcessor.CustomLogDictionary); await LogAnalyticsPublicApi.SendEventsToLogAnalytics(jsonString, SentinelApiConfig, sentinalAuthWorkspaceKey); if (SentinelApiConfig.StoreDataToBlobStorage) { await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonString, "CustomLog"); } SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.CustomLogDictionary.Count}] Syslog Custom Logs messages to Sentinel.", ConsoleColor.Cyan); }
public static async Task SyslogToLinuxSyslogJson() { Dictionary <string, object> jsonLinuxSyslogDictionary = new Dictionary <string, object>(); jsonLinuxSyslogDictionary.Add("DataType", "LINUX_SYSLOGS_BLOB"); jsonLinuxSyslogDictionary.Add("IPName", "logmanagement"); jsonLinuxSyslogDictionary.Add("DataItems", syslogToSentinelProcessor.SyslogDictionary); string jsonFinalString = JsonConvert.SerializeObject(jsonLinuxSyslogDictionary); UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2); if (SentinelApiConfig.StoreDataToBlobStorage) { await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonFinalString, "LinuxSyslog"); } SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.SyslogDictionary.Count}] LogManagement.Syslog messages to Sentinel.", ConsoleColor.Magenta); }
public static async Task SyslogToCefSyslogJson() { Dictionary <string, object> jsonLinuxSyslogDictionary = new Dictionary <string, object>(); jsonLinuxSyslogDictionary.Add("DataType", "SECURITY_CEF_BLOB"); jsonLinuxSyslogDictionary.Add("IPName", "Security"); jsonLinuxSyslogDictionary.Add("DataItems", syslogToSentinelProcessor.CefDictionary); string jsonFinalString = JsonConvert.SerializeObject(jsonLinuxSyslogDictionary, Formatting.Indented); File.WriteAllText(@"c:\temp\CefJson.json", jsonFinalString); UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2); if (SentinelApiConfig.StoreDataToBlobStorage) { await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonFinalString, "CefSyslog"); } SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.CefDictionary.Count}] LogManagement.CommonSecurityLog messages to Sentinel.", ConsoleColor.Blue); }
public void GetNextBatchOfRecords() { // Initialize local dictionaries for this iteration. CefDictionary = new List <Dictionary <string, object> >(); CustomLogDictionary = new List <Dictionary <string, object> >(); SyslogDictionary = new List <Dictionary <string, object> >(); Stopwatch queryTimer = Stopwatch.StartNew(); // Query file information FileInfo fileInfo = new FileInfo(Path.Combine(SentinelWorkspacePoc.GetExecutionPath(), $"SyslogToSentinel.kql")); string textOfKustoTemplate = File.ReadAllText(fileInfo.FullName); // Create a single connection to be used for all queries. string connectionString = $"Data Source=https://{SentinelApiConfig.KustoDataSourceConfig.ClusterUri}:443;Initial Catalog={SentinelApiConfig.KustoDataSourceConfig.Database};AAD Federated Security=True"; var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(connectionString); // Use the KustoTemplate functionality QueryTemplate template = new QueryTemplate(textOfKustoTemplate); List <Dictionary <string, object> > result = template.ExecuteForDictionary(cslQueryProvider, null); queryTimer.Stop(); SentinelWorkspacePoc.PrintCustomMessage($"{fileInfo.Name} returned {result.Count} records in {queryTimer.Elapsed.TotalSeconds:N3} seconds.", ConsoleColor.Yellow); // Massage the Syslog Dictionary foreach (Dictionary <string, object> syslogRecordDictionary in result) { Dictionary <string, object> linuxSyslogRecord = new Dictionary <string, object>(); linuxSyslogRecord.Add("TimeStamp", syslogRecordDictionary["DeviceTimestamp"]); linuxSyslogRecord.Add("Host", syslogRecordDictionary["HostName"]); linuxSyslogRecord.Add("HostIp", syslogRecordDictionary["SourceIpAddress"]); linuxSyslogRecord.Add("ProcessId", syslogRecordDictionary["ProcId"]); linuxSyslogRecord.Add("Facility", syslogRecordDictionary["Facility"]); linuxSyslogRecord.Add("Severity", syslogRecordDictionary["Severity"]); linuxSyslogRecord.Add("Message", syslogRecordDictionary["Payload"]); linuxSyslogRecord.Add("AppName", syslogRecordDictionary["AppName"]); linuxSyslogRecord.Add("MsgId", syslogRecordDictionary["MsgId"]); SyslogDictionary.Add(linuxSyslogRecord); } Random random = new Random(); SyslogToCef syslogToCef = new SyslogToCef(); // Massage the CEF Dictionary foreach (Dictionary <string, object> cefRecordDictionary in result) { Dictionary <string, object> currentRecord = syslogToCef.ConvertSyslogToCef(cefRecordDictionary); //CefDictionary.Add(currentRecord); Dictionary <string, object> cefRecord = new Dictionary <string, object>(); cefRecord.Add("Timestamp", $"{DateTime.UtcNow:yyyy-MM-ddTHH:mm:ss.fffZ}"); cefRecord.Add("EventTime", cefRecordDictionary["DeviceTimestamp"]); cefRecord.Add("Host", cefRecordDictionary["HostName"]); cefRecord.Add("HostIP", cefRecordDictionary["SourceIpAddress"]); cefRecord.Add("ident", "CEF"); cefRecord.Add("Facility", cefRecordDictionary["Facility"]); cefRecord.Add("Severity", currentRecord["Severity"]); cefRecord.Add("Message", currentRecord["Message"]); CefDictionary.Add(cefRecord); } // Massage the CustomLog dictionary foreach (Dictionary <string, object> customLogRecordsDictionary in result) { customLogRecordsDictionary["ExtractedData"] = JsonConvert.SerializeObject(customLogRecordsDictionary["ExtractedData"]); customLogRecordsDictionary["LogFileLineage"] = JsonConvert.SerializeObject(customLogRecordsDictionary["LogFileLineage"]); CustomLogDictionary.Add(customLogRecordsDictionary); } }