/// <summary>
/// Collect event logs on macOS using the 'log' utility
/// </summary>
public void ExecuteMacOs()
{
// New log entries start with a timestamp like so:
// 2019-09-25 20:38:53.784594-0700 0xdbf47 Error 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy
Regex MacLogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)");
EventLogObject curObject = null;
using var process = new Process()
{
StartInfo = new ProcessStartInfo
{
FileName = "log",
Arguments = (GatherVerboseLogs) ? "show" : "show --predicate \"messageType == 16 || messageType == 17\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden
}
};
var stdError = new StringBuilder();
process.ErrorDataReceived += (sender, args) => stdError.AppendLine(args.Data);
try
{
process.Start();
//Throw away header
process.StandardOutput.ReadLine();
while (!process.StandardOutput.EndOfStream)
{
var evt = process.StandardOutput.ReadLine();
if (MacLogHeader.IsMatch(evt))
{
DatabaseManager.Write(curObject, RunId);
curObject = new EventLogObject()
{
Event = evt,
Level = MacLogHeader.Matches(evt).Single().Groups[2].Value,
Summary = $"{MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value}:{MacLogHeader.Matches(evt).Single().Groups[5].Captures[0].Value}",
Timestamp = MacLogHeader.Matches(evt).Single().Groups[1].Captures[0].Value,
Source = MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value,
};
}
else
{
curObject.Data.Append(evt);
}
}
process.WaitForExit();
DatabaseManager.Write(curObject, RunId);
}
catch (Exception e)
{
Log.Debug(e, "Failed to gather event logs on Mac OS. {0}", stdError);
}
}
/// <summary>
/// Collect event logs on macOS using the 'log' utility
/// </summary>
public void ExecuteMacOs()
{
_ = DatabaseManager.Transaction;
var outputPath = Path.Combine(Directory.GetCurrentDirectory(), "events");
var file = (GatherVerboseLogs)? ExternalCommandRunner.RunExternalCommand("log", "show") : ExternalCommandRunner.RunExternalCommand("log", "show --predicate \"messageType == 16 || messageType == 17\"");
// New log entries start with a timestamp like so:
// 2019-09-25 20:38:53.784594-0700 0xdbf47 Error 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy
Regex LogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)");
List <string> data = null;
string previousLine = null;
foreach (var line in file.Split('\n'))
{
if (LogHeader.IsMatch(line))
{
if (previousLine != null)
{
var obj = new EventLogObject()
{
Data = (data.Count > 0) ? data : null,
Event = previousLine,
Level = LogHeader.Matches(previousLine).Single().Groups[2].Value,
Summary = string.Format("{0}:{1}", LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value, LogHeader.Matches(previousLine).Single().Groups[5].Captures[0].Value),
Timestamp = LogHeader.Matches(previousLine).Single().Groups[1].Captures[0].Value,
Source = LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value
};
DatabaseManager.Write(obj, runId);
}
previousLine = line;
data = new List <string>();
}
else
{
if (previousLine != null)
{
data.Add(line);
}
}
}
if (previousLine != null)
{
var obj = new EventLogObject()
{
Data = (data.Count > 0) ? data:null,
Event = previousLine,
Level = LogHeader.Matches(previousLine).Single().Groups[2].Value,
Summary = string.Format("{0}:{1}", LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value, LogHeader.Matches(previousLine).Single().Groups[5].Captures[0].Value),
Timestamp = LogHeader.Matches(previousLine).Single().Groups[1].Captures[0].Value,
Source = LogHeader.Matches(previousLine).Single().Groups[4].Captures[0].Value
};
DatabaseManager.Write(obj, runId);
}
DatabaseManager.Commit();
}
/// <summary>
/// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity)
/// </summary>
public void ExecuteLinux()
{
try
{
Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)");
string[] authLog = File.ReadAllLines("/var/log/auth.log");
foreach (var entry in authLog)
{
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject()
{
Event = entry,
Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value,
Timestamp = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value,
Source = "/var/log/auth.log",
Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
};
DatabaseManager.Write(obj, runId);
}
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed systemd[1]: Reloading
}
string[] sysLog = File.ReadAllLines("/var/log/syslog");
foreach (var entry in sysLog)
{
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed systemd[1]: Reloading
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject()
{
Event = entry,
Summary = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
Timestamp = LogHeader.Matches(entry).Single().Groups[0].Captures[0].Value,
Source = "/var/log/syslog",
Process = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value,
};
DatabaseManager.Write(obj, runId);
}
}
}
catch (Exception e)
{
Log.Debug(e, "Failed to parse /var/log/auth.log");
}
}
/// <summary>
/// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity)
/// </summary>
public void ExecuteLinux()
{
Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)", RegexOptions.Compiled);
void ParseLinuxLog(string path)
{
try
{
string[] log = File.ReadAllLines(path);
foreach (var entry in log)
{
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject(entry)
{
Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value,
Source = path,
Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
};
if (DateTime.TryParse(LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, out DateTime Timestamp))
{
obj.Timestamp = Timestamp;
}
DatabaseManager.Write(obj, RunId);
}
}
}
catch (Exception e) when(
e is ArgumentException ||
e is ArgumentNullException ||
e is DirectoryNotFoundException ||
e is PathTooLongException ||
e is FileNotFoundException ||
e is IOException ||
e is NotSupportedException ||
e is System.Security.SecurityException ||
e is UnauthorizedAccessException)
{
Log.Debug("Failed to parse {0}", path);
}
}
ParseLinuxLog("/var/log/auth.log");
ParseLinuxLog("/var/log/syslog");
}
public void ExecuteWindows()
{
EventLog[] logs = EventLog.GetEventLogs();
foreach (var log in logs)
{
try
{
EventLogEntryCollection coll = log.Entries;
foreach (EventLogEntry?entry in coll)
{
if (entry != null)
{
if (GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error")
{
var sentences = entry.Message.Split('.');
//Let's add the periods back.
for (var i = 0; i < sentences.Length; i++)
{
sentences[i] = string.Concat(sentences[i], ".");
}
EventLogObject obj = new EventLogObject($"{entry.TimeGenerated.ToString("o", CultureInfo.InvariantCulture)} {entry.EntryType.ToString()} {entry.Message}")
{
Level = entry.EntryType.ToString(),
Summary = sentences[0],
Source = string.IsNullOrEmpty(entry.Source) ? null : entry.Source,
Timestamp = entry.TimeGenerated,
Data = new List <string>()
{
entry.Message
}
};
StallIfHighMemoryUsageAndLowMemoryModeEnabled();
Results.Push(obj);
}
}
}
}
catch (Exception e)
{
Log.Debug(e, "Error parsing log {0}", log.Source);
}
}
}
public void TestAddedInMemory()
{
var elo2 = new EventLogObject("Entry")
{
Timestamp = DateTime.Now.AddYears(1)
};
BaseCompare bc = new BaseCompare();
bc.Compare(new List <CollectObject>()
{
}, new List <CollectObject>()
{
elo2
}, "FirstRun", "SecondRun");
var results = bc.Results;
Assert.IsTrue(results[(RESULT_TYPE.LOG, CHANGE_TYPE.CREATED)].Any(x => x.Compare is EventLogObject));
public void TestAddedInDatabase()
{
var elo2 = new EventLogObject("Entry")
{
Timestamp = DateTime.Now.AddYears(1)
};
DatabaseManager.Write(elo2, "SecondRun");
// Let Database Finish Writing
Thread.Sleep(1);
BaseCompare bc = new BaseCompare();
bc.Compare("FirstRun", "SecondRun");
var results = bc.Results;
Assert.IsTrue(results[(RESULT_TYPE.LOG, CHANGE_TYPE.CREATED)].Any(x => x.Compare is EventLogObject));
/// <summary>
/// Collect event logs on Windows using System.Diagnostics.EventLog
/// </summary>
public void ExecuteWindows()
{
EventLog[] logs = EventLog.GetEventLogs();
foreach (var log in logs)
{
try
{
EventLogEntryCollection coll = log.Entries;
foreach (EventLogEntry entry in coll)
{
if (GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error")
{
var sentences = entry.Message.Split('.');
//Let's add the periods back.
for (var i = 0; i < sentences.Length; i++)
{
sentences[i] = string.Concat(sentences[i], ".");
}
EventLogObject obj = new EventLogObject()
{
Level = entry.EntryType.ToString(),
Summary = sentences[0],
Source = string.IsNullOrEmpty(entry.Source) ? null:entry.Source,
Data = new List <string>()
{
entry.Message
},
Timestamp = entry.TimeGenerated.ToString(),
Event = string.Format("{0} {1} {2}", entry.TimeGenerated.ToString(), entry.EntryType.ToString(), entry.Message)
};
DatabaseManager.Write(obj, runId);
}
}
}
catch (Exception e)
{
Log.Debug(e, "Error parsing log {0}", log.Source);
}
}
}
public void TestSerializeAndDeserializeEventLogObject()
{
var elo = new EventLogObject("Disk");
Assert.IsTrue(elo.RowKey.Equals(JsonUtils.Hydrate(JsonUtils.Dehydrate(elo), RESULT_TYPE.LOG)?.RowKey));
}
/// <summary>
/// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity)
/// </summary>
public void ExecuteLinux(CancellationToken cancellationToken)
{
Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)", RegexOptions.Compiled);
void HandleLinuxEvent(string entry, string path)
{
// New log entries start with a timestamp like so: Sep 7 02:16:16 testbed sudo:
// pam_unix(sudo:session):session opened for user root
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject(entry)
{
Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value,
Source = path,
Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
};
if (DateTime.TryParse(LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value, out DateTime Timestamp))
{
obj.Timestamp = Timestamp;
}
HandleChange(obj);
}
}
void ParseLinuxLog(string path)
{
try
{
string[] log = File.ReadAllLines(path);
if (opts.SingleThread)
{
foreach (var entry in log)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
HandleLinuxEvent(entry, path);
}
}
else
{
ParallelOptions po = new ParallelOptions()
{
CancellationToken = cancellationToken
};
Parallel.ForEach(log, po, entry => HandleLinuxEvent(entry, path));
}
}
catch (Exception e) when(
e is ArgumentException ||
e is ArgumentNullException ||
e is DirectoryNotFoundException ||
e is PathTooLongException ||
e is FileNotFoundException ||
e is IOException ||
e is NotSupportedException ||
e is System.Security.SecurityException ||
e is UnauthorizedAccessException)
{
Log.Debug("Failed to parse {0}", path);
}
}
ParseLinuxLog("/var/log/auth.log");
ParseLinuxLog("/var/log/syslog");
}
public void ExecuteWindows(CancellationToken cancellationToken)
{
void ParseWindowsLog(EventLogEntry entry)
{
if (opts.GatherVerboseLogs || entry.EntryType.ToString() == "Warning" || entry.EntryType.ToString() == "Error")
{
var sentences = entry.Message.Split('.');
//Let's add the periods back.
for (var i = 0; i < sentences.Length; i++)
{
sentences[i] = string.Concat(sentences[i], ".");
}
EventLogObject obj = new EventLogObject($"{entry.TimeGenerated.ToString("o", CultureInfo.InvariantCulture)} {entry.EntryType.ToString()} {entry.Message}")
{
Level = entry.EntryType.ToString(),
Summary = sentences[0],
Source = string.IsNullOrEmpty(entry.Source) ? null : entry.Source,
Timestamp = entry.TimeGenerated,
Data = new List <string>()
{
entry.Message
}
};
HandleChange(obj);
}
}
EventLog[] logs = EventLog.GetEventLogs();
foreach (var log in logs)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
try
{
EventLogEntryCollection coll = log.Entries;
if (opts.SingleThread)
{
foreach (EventLogEntry?entry in coll)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (entry != null)
{
ParseWindowsLog(entry);
}
}
}
else
{
List <EventLogEntry> coll2 = new List <EventLogEntry>();
ParallelOptions po = new ParallelOptions();
po.CancellationToken = cancellationToken;
foreach (EventLogEntry?entry in coll)
{
if (entry != null)
{
coll2.Add(entry);
}
}
Parallel.ForEach(coll2, po, entry => ParseWindowsLog(entry));
}
}
catch (Exception e)
{
Log.Debug(e, "Error parsing log {0}", log.Source);
}
}
}
/// <summary>
/// Collect event logs on macOS using the 'log' utility
/// </summary>
public void ExecuteMacOs(CancellationToken cancellationToken)
{
// New log entries start with a timestamp like so: 2019-09-25 20:38:53.784594-0700 0xdbf47 Error
// 0x0 0 0 kernel: (Sandbox) Sandbox: mdworker(15726) deny(1) mach-lookup com.apple.security.syspolicy
Regex MacLogHeader = new Regex("^([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]).*?0x[0-9a-f]*[\\s]*([A-Za-z]*)[\\s]*0x[0-9a-f][\\s]*[0-9]*[\\s]*([0-9]*)[\\s]*(.*?):(.*)", RegexOptions.Compiled);
EventLogObject?curObject = null;
using var process = new Process()
{
StartInfo = new ProcessStartInfo
{
FileName = "log",
Arguments = opts.GatherVerboseLogs ? "show" : "show --predicate \"messageType == 16 || messageType == 17\"",
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden
}
};
var stdError = new StringBuilder();
process.ErrorDataReceived += (sender, args) => stdError.AppendLine(args.Data);
try
{
process.Start();
//Throw away header
process.StandardOutput.ReadLine();
while (!process.StandardOutput.EndOfStream)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var evt = process.StandardOutput.ReadLine();
if (evt != null && MacLogHeader.IsMatch(evt))
{
if (curObject != null)
{
HandleChange(curObject);
}
curObject = new EventLogObject(evt)
{
Level = MacLogHeader.Matches(evt).Single().Groups[2].Value,
Summary = $"{MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value}:{MacLogHeader.Matches(evt).Single().Groups[5].Captures[0].Value}",
Source = MacLogHeader.Matches(evt).Single().Groups[4].Captures[0].Value,
};
if (DateTime.TryParse(MacLogHeader.Matches(evt).Single().Groups[1].Captures[0].Value, out DateTime Timestamp))
{
curObject.Timestamp = Timestamp;
}
}
else
{
if (curObject != null)
{
if (curObject.Data == null)
{
curObject.Data = new List <string>();
}
curObject.Data.Append(evt);
}
}
}
process.WaitForExit();
if (curObject != null)
{
HandleChange(curObject);
}
}
catch (Exception e)
{
Log.Debug(e, "Failed to gather event logs on Mac OS. {0}", stdError);
}
}
/// <summary>
/// Parses /var/log/auth.log and /var/log/syslog (no way to distinguish severity)
/// </summary>
public void ExecuteLinux()
{
Regex LogHeader = new Regex("^([A-Z][a-z][a-z][0-9:\\s]*)?[\\s].*?[\\s](.*?): (.*)");
try
{
string[] authLog = File.ReadAllLines("/var/log/auth.log");
foreach (var entry in authLog)
{
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed sudo: pam_unix(sudo:session):session opened for user root
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject()
{
Event = entry,
Summary = LogHeader.Matches(entry).Single().Groups[3].Captures[0].Value,
Timestamp = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value,
Source = "/var/log/auth.log",
Process = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
};
DatabaseManager.Write(obj, RunId);
}
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed systemd[1]: Reloading
}
}
catch (Exception e) when(
e is ArgumentException ||
e is ArgumentNullException ||
e is DirectoryNotFoundException ||
e is PathTooLongException ||
e is FileNotFoundException ||
e is IOException ||
e is NotSupportedException ||
e is System.Security.SecurityException ||
e is UnauthorizedAccessException)
{
Log.Debug("Failed to parse /var/auth/auth.log");
}
try
{
string[] sysLog = File.ReadAllLines("/var/log/syslog");
foreach (var entry in sysLog)
{
// New log entries start with a timestamp like so:
// Sep 7 02:16:16 testbed systemd[1]: Reloading
if (LogHeader.IsMatch(entry))
{
var obj = new EventLogObject()
{
Event = entry,
Summary = LogHeader.Matches(entry).Single().Groups[2].Captures[0].Value,
Timestamp = LogHeader.Matches(entry).Single().Groups[0].Captures[0].Value,
Source = "/var/log/syslog",
Process = LogHeader.Matches(entry).Single().Groups[1].Captures[0].Value,
};
DatabaseManager.Write(obj, RunId);
}
}
}
catch (Exception e) when(
e is ArgumentException ||
e is ArgumentNullException ||
e is DirectoryNotFoundException ||
e is PathTooLongException ||
e is FileNotFoundException ||
e is IOException ||
e is NotSupportedException ||
e is System.Security.SecurityException ||
e is UnauthorizedAccessException)
{
Log.Debug("Failed to parse /var/log/syslog");
}
}
