/// <summary> /// Collect event logs on macOS using the 'log' utility /// </summary> public void ExecuteMacOs() { // New log entries start with a timestamp like so: // 2019-09-25 20:38:53.784594-0700 0xdbf47 Error 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy Regex MacLogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)"); EventLogObject curObject = null; using var process = new Process() { StartInfo = new ProcessStartInfo { FileName = "log", Arguments = (GatherVerboseLogs) ? "show" : "show --predicate \"messageType == 16 || messageType == 17\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WindowStyle = ProcessWindowStyle.Hidden } }; var stdError = new StringBuilder(); process.ErrorDataReceived += (sender, args) => stdError.AppendLine(args.Data); try { process.Start(); //Throw away header process.StandardOutput.ReadLine(); while (!process.StandardOutput.EndOfStream) { var evt = process.StandardOutput.ReadLine(); if (MacLogHeader.IsMatch(evt)) { DatabaseManager.Write(curObject, RunId); curObject = new EventLogObject() { Event = evt, Level = MacLogHeader.Matches(evt).Single().Groups[2].Value, Summary = $"{MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value}:{MacLogHeader.Matches(evt).Single().Groups[5].Captures[0].Value}", Timestamp = MacLogHeader.Matches(evt).Single().Groups[1].Captures[0].Value, Source = MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value, }; } else { curObject.Data.Append(evt); } } process.WaitForExit(); DatabaseManager.Write(curObject, RunId); } catch (Exception e) { Log.Debug(e, "Failed to gather event logs on Mac OS. {0}", stdError); } }
/// <summary> /// Collect event logs on macOS using the 'log' utility /// </summary> public void ExecuteMacOs() { _ = DatabaseManager.Transaction; var outputPath = Path.Combine(Directory.GetCurrentDirectory(), "events"); var file = (GatherVerboseLogs)? ExternalCommandRunner.RunExternalCommand("log", "show") : ExternalCommandRunner.RunExternalCommand("log", "show --predicate \"messageType == 16 || messageType == 17\""); // New log entries start with a timestamp like so: // 2019-09-25 20:38:53.784594-0700 0xdbf47 Error 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy Regex LogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)"); List <string> data = null; string previousLine = null; foreach (var line in file.Split('\n')) { if (LogHeader.IsMatch(line)) { if (previousLine != null) { var obj = new EventLogObject() { Data = (data.Count > 0) ? data : null, Event = previousLine, Level = LogHeader.Matches(previousLine).Single().Groups[2].Value, Summary = string.Format("{0}:{1}", LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value, LogHeader.Matches(previousLine).Single().Groups[5].Captures[0].Value), Timestamp = LogHeader.Matches(previousLine).Single().Groups[1].Captures[0].Value, Source = LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value }; DatabaseManager.Write(obj, runId); } previousLine = line; data = new List <string>(); } else { if (previousLine != null) { data.Add(line); } } } if (previousLine != null) { var obj = new EventLogObject() { Data = (data.Count > 0) ? data:null, Event = previousLine, Level = LogHeader.Matches(previousLine).Single().Groups[2].Value, Summary = string.Format("{0}:{1}", LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value, LogHeader.Matches(previousLine).Single().Groups[5].Captures[0].Value), Timestamp = LogHeader.Matches(previousLine).Single().Groups[1].Captures[0].Value, Source = LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value }; DatabaseManager.Write(obj, runId); } DatabaseManager.Commit(); }
/// <summary> /// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity) /// </summary> public void ExecuteLinux() { try { Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)"); string[] authLog = File.ReadAllLines("/var/log/auth.log"); foreach (var entry in authLog) { // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject() { Event = entry, Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value, Timestamp = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, Source = "/var/log/auth.log", Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, }; DatabaseManager.Write(obj, runId); } // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed systemd[1]: Reloading } string[] sysLog = File.ReadAllLines("/var/log/syslog"); foreach (var entry in sysLog) { // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed systemd[1]: Reloading if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject() { Event = entry, Summary = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, Timestamp = LogHeader.Matches(entry).Single().Groups[0].Captures[0].Value, Source = "/var/log/syslog", Process = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, }; DatabaseManager.Write(obj, runId); } } } catch (Exception e) { Log.Debug(e, "Failed to parse /var/log/auth.log"); } }
/// <summary> /// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity) /// </summary> public void ExecuteLinux() { Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)", RegexOptions.Compiled); void ParseLinuxLog(string path) { try { string[] log = File.ReadAllLines(path); foreach (var entry in log) { // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject(entry) { Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value, Source = path, Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, }; if (DateTime.TryParse(LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, out DateTime Timestamp)) { obj.Timestamp = Timestamp; } DatabaseManager.Write(obj, RunId); } } } catch (Exception e) when( e is ArgumentException || e is ArgumentNullException || e is DirectoryNotFoundException || e is PathTooLongException || e is FileNotFoundException || e is IOException || e is NotSupportedException || e is System.Security.SecurityException || e is UnauthorizedAccessException) { Log.Debug("Failed to parse {0}", path); } } ParseLinuxLog("/var/log/auth.log"); ParseLinuxLog("/var/log/syslog"); }
public void ExecuteWindows() { EventLog[] logs = EventLog.GetEventLogs(); foreach (var log in logs) { try { EventLogEntryCollection coll = log.Entries; foreach (EventLogEntry?entry in coll) { if (entry != null) { if (GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error") { var sentences = entry.Message.Split('.'); //Let's add the periods back. for (var i = 0; i < sentences.Length; i++) { sentences[i] = string.Concat(sentences[i], "."); } EventLogObject obj = new EventLogObject($"{entry.TimeGenerated.ToString("o", CultureInfo.InvariantCulture)} {entry.EntryType.ToString()} {entry.Message}") { Level = entry.EntryType.ToString(), Summary = sentences[0], Source = string.IsNullOrEmpty(entry.Source) ? null : entry.Source, Timestamp = entry.TimeGenerated, Data = new List <string>() { entry.Message } }; StallIfHighMemoryUsageAndLowMemoryModeEnabled(); Results.Push(obj); } } } } catch (Exception e) { Log.Debug(e, "Error parsing log {0}", log.Source); } } }
public void TestAddedInMemory() { var elo2 = new EventLogObject("Entry") { Timestamp = DateTime.Now.AddYears(1) }; BaseCompare bc = new BaseCompare(); bc.Compare(new List <CollectObject>() { }, new List <CollectObject>() { elo2 }, "FirstRun", "SecondRun"); var results = bc.Results; Assert.IsTrue(results[(RESULT_TYPE.LOG, CHANGE_TYPE.CREATED)].Any(x => x.Compare is EventLogObject));
public void TestAddedInDatabase() { var elo2 = new EventLogObject("Entry") { Timestamp = DateTime.Now.AddYears(1) }; DatabaseManager.Write(elo2, "SecondRun"); // Let Database Finish Writing Thread.Sleep(1); BaseCompare bc = new BaseCompare(); bc.Compare("FirstRun", "SecondRun"); var results = bc.Results; Assert.IsTrue(results[(RESULT_TYPE.LOG, CHANGE_TYPE.CREATED)].Any(x => x.Compare is EventLogObject));
/// <summary> /// Collect event logs on Windows using System.Diagnostics.EventLog /// </summary> public void ExecuteWindows() { EventLog[] logs = EventLog.GetEventLogs(); foreach (var log in logs) { try { EventLogEntryCollection coll = log.Entries; foreach (EventLogEntry entry in coll) { if (GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error") { var sentences = entry.Message.Split('.'); //Let's add the periods back. for (var i = 0; i < sentences.Length; i++) { sentences[i] = string.Concat(sentences[i], "."); } EventLogObject obj = new EventLogObject() { Level = entry.EntryType.ToString(), Summary = sentences[0], Source = string.IsNullOrEmpty(entry.Source) ? null:entry.Source, Data = new List <string>() { entry.Message }, Timestamp = entry.TimeGenerated.ToString(), Event = string.Format("{0} {1} {2}", entry.TimeGenerated.ToString(), entry.EntryType.ToString(), entry.Message) }; DatabaseManager.Write(obj, runId); } } } catch (Exception e) { Log.Debug(e, "Error parsing log {0}", log.Source); } } }
public void TestSerializeAndDeserializeEventLogObject() { var elo = new EventLogObject("Disk"); Assert.IsTrue(elo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(elo), RESULT_TYPE.LOG)?.RowKey)); }
/// <summary> /// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity) /// </summary> public void ExecuteLinux(CancellationToken cancellationToken) { Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)", RegexOptions.Compiled); void HandleLinuxEvent(string entry, string path) { // New log entries start with a timestamp like so: Sep 7 02:16:16 testbed sudo: // pam_unix(sudo:session):session opened for user root if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject(entry) { Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value, Source = path, Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, }; if (DateTime.TryParse(LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, out DateTime Timestamp)) { obj.Timestamp = Timestamp; } HandleChange(obj); } } void ParseLinuxLog(string path) { try { string[] log = File.ReadAllLines(path); if (opts.SingleThread) { foreach (var entry in log) { if (cancellationToken.IsCancellationRequested) { break; } HandleLinuxEvent(entry, path); } } else { ParallelOptions po = new ParallelOptions() { CancellationToken = cancellationToken }; Parallel.ForEach(log, po, entry => HandleLinuxEvent(entry, path)); } } catch (Exception e) when( e is ArgumentException || e is ArgumentNullException || e is DirectoryNotFoundException || e is PathTooLongException || e is FileNotFoundException || e is IOException || e is NotSupportedException || e is System.Security.SecurityException || e is UnauthorizedAccessException) { Log.Debug("Failed to parse {0}", path); } } ParseLinuxLog("/var/log/auth.log"); ParseLinuxLog("/var/log/syslog"); }
public void ExecuteWindows(CancellationToken cancellationToken) { void ParseWindowsLog(EventLogEntry entry) { if (opts.GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error") { var sentences = entry.Message.Split('.'); //Let's add the periods back. for (var i = 0; i < sentences.Length; i++) { sentences[i] = string.Concat(sentences[i], "."); } EventLogObject obj = new EventLogObject($"{entry.TimeGenerated.ToString("o", CultureInfo.InvariantCulture)} {entry.EntryType.ToString()} {entry.Message}") { Level = entry.EntryType.ToString(), Summary = sentences[0], Source = string.IsNullOrEmpty(entry.Source) ? null : entry.Source, Timestamp = entry.TimeGenerated, Data = new List <string>() { entry.Message } }; HandleChange(obj); } } EventLog[] logs = EventLog.GetEventLogs(); foreach (var log in logs) { if (cancellationToken.IsCancellationRequested) { break; } try { EventLogEntryCollection coll = log.Entries; if (opts.SingleThread) { foreach (EventLogEntry?entry in coll) { if (cancellationToken.IsCancellationRequested) { break; } if (entry != null) { ParseWindowsLog(entry); } } } else { List <EventLogEntry> coll2 = new List <EventLogEntry>(); ParallelOptions po = new ParallelOptions(); po.CancellationToken = cancellationToken; foreach (EventLogEntry?entry in coll) { if (entry != null) { coll2.Add(entry); } } Parallel.ForEach(coll2, po, entry => ParseWindowsLog(entry)); } } catch (Exception e) { Log.Debug(e, "Error parsing log {0}", log.Source); } } }
/// <summary> /// Collect event logs on macOS using the 'log' utility /// </summary> public void ExecuteMacOs(CancellationToken cancellationToken) { // New log entries start with a timestamp like so: 2019-09-25 20:38:53.784594-0700 0xdbf47 Error // 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy Regex MacLogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)", RegexOptions.Compiled); EventLogObject?curObject = null; using var process = new Process() { StartInfo = new ProcessStartInfo { FileName = "log", Arguments = opts.GatherVerboseLogs ? "show" : "show --predicate \"messageType == 16 || messageType == 17\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true, WindowStyle = ProcessWindowStyle.Hidden } }; var stdError = new StringBuilder(); process.ErrorDataReceived += (sender, args) => stdError.AppendLine(args.Data); try { process.Start(); //Throw away header process.StandardOutput.ReadLine(); while (!process.StandardOutput.EndOfStream) { if (cancellationToken.IsCancellationRequested) { break; } var evt = process.StandardOutput.ReadLine(); if (evt != null && MacLogHeader.IsMatch(evt)) { if (curObject != null) { HandleChange(curObject); } curObject = new EventLogObject(evt) { Level = MacLogHeader.Matches(evt).Single().Groups[2].Value, Summary = $"{MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value}:{MacLogHeader.Matches(evt).Single().Groups[5].Captures[0].Value}", Source = MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value, }; if (DateTime.TryParse(MacLogHeader.Matches(evt).Single().Groups[1].Captures[0].Value, out DateTime Timestamp)) { curObject.Timestamp = Timestamp; } } else { if (curObject != null) { if (curObject.Data == null) { curObject.Data = new List <string>(); } curObject.Data.Append(evt); } } } process.WaitForExit(); if (curObject != null) { HandleChange(curObject); } } catch (Exception e) { Log.Debug(e, "Failed to gather event logs on Mac OS. {0}", stdError); } }
/// <summary> /// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity) /// </summary> public void ExecuteLinux() { Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)"); try { string[] authLog = File.ReadAllLines("/var/log/auth.log"); foreach (var entry in authLog) { // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject() { Event = entry, Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value, Timestamp = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, Source = "/var/log/auth.log", Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, }; DatabaseManager.Write(obj, RunId); } // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed systemd[1]: Reloading } } catch (Exception e) when( e is ArgumentException || e is ArgumentNullException || e is DirectoryNotFoundException || e is PathTooLongException || e is FileNotFoundException || e is IOException || e is NotSupportedException || e is System.Security.SecurityException || e is UnauthorizedAccessException) { Log.Debug("Failed to parse /var/auth/auth.log"); } try { string[] sysLog = File.ReadAllLines("/var/log/syslog"); foreach (var entry in sysLog) { // New log entries start with a timestamp like so: // Sep 7 02:16:16 testbed systemd[1]: Reloading if (LogHeader.IsMatch(entry)) { var obj = new EventLogObject() { Event = entry, Summary = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value, Timestamp = LogHeader.Matches(entry).Single().Groups[0].Captures[0].Value, Source = "/var/log/syslog", Process = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, }; DatabaseManager.Write(obj, RunId); } } } catch (Exception e) when( e is ArgumentException || e is ArgumentNullException || e is DirectoryNotFoundException || e is PathTooLongException || e is FileNotFoundException || e is IOException || e is NotSupportedException || e is System.Security.SecurityException || e is UnauthorizedAccessException) { Log.Debug("Failed to parse /var/log/syslog"); } }