public static async Task CefFilesToSentinelProcessor()
{
//X509Certificate2 cert;
GlobalLog.WriteToStringBuilderLog("Attempting to load CEF Files ", 14001);
// Update to LINQ query to prevent attempting to load ALL files during an iteration.
var directoryInfo = new DirectoryInfo(SentinelApiConfig.EnabledSentinelUploads.CefFileFolderToUpload);
var orderedFileList =
directoryInfo.EnumerateFiles("CefToSentinel*.json", SearchOption.TopDirectoryOnly)
.OrderBy(d => d.LastAccessTime)
.Select(d => d.FullName)
.Take(25)
.ToList();
if (orderedFileList.Count > 0)
{
foreach (string file in orderedFileList)
{
string jsonFinalString = File.ReadAllText(file);
UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2);
SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{file}] LogManagement.CommonSecurityLog messages to Sentinel.", ConsoleColor.Green);
File.Delete(file);
}
}
}
public static async Task SyslogToCustomLog()
{
string jsonString = JsonConvert.SerializeObject(syslogToSentinelProcessor.CustomLogDictionary);
await LogAnalyticsPublicApi.SendEventsToLogAnalytics(jsonString, SentinelApiConfig, sentinalAuthWorkspaceKey);
if (SentinelApiConfig.StoreDataToBlobStorage)
{
await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonString, "CustomLog");
}
SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.CustomLogDictionary.Count}] Syslog Custom Logs messages to Sentinel.", ConsoleColor.Cyan);
}
public static async Task SyslogToLinuxSyslogJson()
{
Dictionary <string, object> jsonLinuxSyslogDictionary = new Dictionary <string, object>();
jsonLinuxSyslogDictionary.Add("DataType", "LINUX_SYSLOGS_BLOB");
jsonLinuxSyslogDictionary.Add("IPName", "logmanagement");
jsonLinuxSyslogDictionary.Add("DataItems", syslogToSentinelProcessor.SyslogDictionary);
string jsonFinalString = JsonConvert.SerializeObject(jsonLinuxSyslogDictionary);
UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2);
if (SentinelApiConfig.StoreDataToBlobStorage)
{
await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonFinalString, "LinuxSyslog");
}
SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.SyslogDictionary.Count}] LogManagement.Syslog messages to Sentinel.", ConsoleColor.Magenta);
}
public static async Task SyslogToCefSyslogJson()
{
Dictionary <string, object> jsonLinuxSyslogDictionary = new Dictionary <string, object>();
jsonLinuxSyslogDictionary.Add("DataType", "SECURITY_CEF_BLOB");
jsonLinuxSyslogDictionary.Add("IPName", "Security");
jsonLinuxSyslogDictionary.Add("DataItems", syslogToSentinelProcessor.CefDictionary);
string jsonFinalString = JsonConvert.SerializeObject(jsonLinuxSyslogDictionary, Formatting.Indented);
File.WriteAllText(@"c:\temp\CefJson.json", jsonFinalString);
UploadBatchWithSelfSignedJson(jsonFinalString, AuthX509Certificate2);
if (SentinelApiConfig.StoreDataToBlobStorage)
{
await syslogToAzureBlob.UploadFileToBlobStorageAsync(jsonFinalString, "CefSyslog");
}
SentinelWorkspacePoc.PrintCustomMessage($"Uploading [{syslogToSentinelProcessor.CefDictionary.Count}] LogManagement.CommonSecurityLog messages to Sentinel.", ConsoleColor.Blue);
}
public void GetNextBatchOfRecords()
{
// Initialize local dictionaries for this iteration.
CefDictionary = new List <Dictionary <string, object> >();
CustomLogDictionary = new List <Dictionary <string, object> >();
SyslogDictionary = new List <Dictionary <string, object> >();
Stopwatch queryTimer = Stopwatch.StartNew();
// Query file information
FileInfo fileInfo = new FileInfo(Path.Combine(SentinelWorkspacePoc.GetExecutionPath(), $"SyslogToSentinel.kql"));
string textOfKustoTemplate = File.ReadAllText(fileInfo.FullName);
// Create a single connection to be used for all queries.
string connectionString =
$"Data Source=https://{SentinelApiConfig.KustoDataSourceConfig.ClusterUri}:443;Initial Catalog={SentinelApiConfig.KustoDataSourceConfig.Database};AAD Federated Security=True";
var cslQueryProvider = KustoClientFactory.CreateCslQueryProvider(connectionString);
// Use the KustoTemplate functionality
QueryTemplate template = new QueryTemplate(textOfKustoTemplate);
List <Dictionary <string, object> > result = template.ExecuteForDictionary(cslQueryProvider, null);
queryTimer.Stop();
SentinelWorkspacePoc.PrintCustomMessage($"{fileInfo.Name} returned {result.Count} records in {queryTimer.Elapsed.TotalSeconds:N3} seconds.", ConsoleColor.Yellow);
// Massage the Syslog Dictionary
foreach (Dictionary <string, object> syslogRecordDictionary in result)
{
Dictionary <string, object> linuxSyslogRecord = new Dictionary <string, object>();
linuxSyslogRecord.Add("TimeStamp", syslogRecordDictionary["DeviceTimestamp"]);
linuxSyslogRecord.Add("Host", syslogRecordDictionary["HostName"]);
linuxSyslogRecord.Add("HostIp", syslogRecordDictionary["SourceIpAddress"]);
linuxSyslogRecord.Add("ProcessId", syslogRecordDictionary["ProcId"]);
linuxSyslogRecord.Add("Facility", syslogRecordDictionary["Facility"]);
linuxSyslogRecord.Add("Severity", syslogRecordDictionary["Severity"]);
linuxSyslogRecord.Add("Message", syslogRecordDictionary["Payload"]);
linuxSyslogRecord.Add("AppName", syslogRecordDictionary["AppName"]);
linuxSyslogRecord.Add("MsgId", syslogRecordDictionary["MsgId"]);
SyslogDictionary.Add(linuxSyslogRecord);
}
Random random = new Random();
SyslogToCef syslogToCef = new SyslogToCef();
// Massage the CEF Dictionary
foreach (Dictionary <string, object> cefRecordDictionary in result)
{
Dictionary <string, object> currentRecord = syslogToCef.ConvertSyslogToCef(cefRecordDictionary);
//CefDictionary.Add(currentRecord);
Dictionary <string, object> cefRecord = new Dictionary <string, object>();
cefRecord.Add("Timestamp", $"{DateTime.UtcNow:yyyy-MM-ddTHH:mm:ss.fffZ}");
cefRecord.Add("EventTime", cefRecordDictionary["DeviceTimestamp"]);
cefRecord.Add("Host", cefRecordDictionary["HostName"]);
cefRecord.Add("HostIP", cefRecordDictionary["SourceIpAddress"]);
cefRecord.Add("ident", "CEF");
cefRecord.Add("Facility", cefRecordDictionary["Facility"]);
cefRecord.Add("Severity", currentRecord["Severity"]);
cefRecord.Add("Message", currentRecord["Message"]);
CefDictionary.Add(cefRecord);
}
// Massage the CustomLog dictionary
foreach (Dictionary <string, object> customLogRecordsDictionary in result)
{
customLogRecordsDictionary["ExtractedData"] = JsonConvert.SerializeObject(customLogRecordsDictionary["ExtractedData"]);
customLogRecordsDictionary["LogFileLineage"] = JsonConvert.SerializeObject(customLogRecordsDictionary["LogFileLineage"]);
CustomLogDictionary.Add(customLogRecordsDictionary);
}
}