private void DumpGetSetAttributes(bool isLocal)
{
Console.WriteLine("\n=== TEST {0} ===", isLocal ? UnitTestConstants.Local : UnitTestConstants.Network);
var tmp = Path.Combine(Path.GetTempPath(), "File.SetAttributes()-" + Path.GetRandomFileName());
var tempPath = isLocal ? tmp : Path.LocalToUnc(tmp);
var sys32 = isLocal ? UnitTestConstants.SysRoot32 : Path.LocalToUnc(UnitTestConstants.SysRoot32);
Console.WriteLine("\nInput Path: [{0}]", sys32);
// Just enumerate and compare attributes in folder: C:\Windows\System32
foreach (var file in Directory.EnumerateFiles(sys32))
{
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
Console.WriteLine("\nInput Path: [{0}]", tempPath);
// Create some folders and files.
UnitTestConstants.CreateDirectoriesAndFiles(tempPath, 1, false, false, true);
var apply = FileAttributes.Hidden | FileAttributes.Archive | FileAttributes.System | FileAttributes.ReadOnly;
Console.WriteLine("\nSetAttributes(): [{0}]", apply);
var allOk = true;
var cnt = 0;
UnitTestConstants.StopWatcher(true);
foreach (var file in Directory.EnumerateFiles(tempPath))
{
try
{
File.SetAttributes(file, apply);
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Console.WriteLine("\n\t#{0:000}\tFile : [{1}]\n\t\tAlphaFS : [{2}]\n\t\tSystem.IO: [{3}]", ++cnt, file, expected, actual);
if (cnt == 0)
{
Assert.Inconclusive("Nothing is enumerated, but it is expected.");
}
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
catch (Exception ex)
{
allOk = false;
Console.WriteLine("\n\tCaught (UNEXPECTED) {0}: [{1}]", ex.GetType().FullName, ex.Message.Replace(Environment.NewLine, " "));
}
}
Console.WriteLine();
Console.WriteLine(UnitTestConstants.Reporter());
Assert.IsTrue(allOk);
apply = FileAttributes.Normal;
Console.WriteLine("\nSetAttributes(): [{0}]", apply);
allOk = true;
cnt = 0;
UnitTestConstants.StopWatcher(true);
foreach (var file in Directory.EnumerateFiles(tempPath))
{
try
{
File.SetAttributes(file, apply);
var actual = File.GetAttributes(file);
var expected = System.IO.File.GetAttributes(file);
Console.WriteLine("\n\t#{0:000}\tFile : [{1}]\n\t\tAlphaFS : [{2}]\n\t\tSystem.IO: [{3}]", ++cnt, file, expected, actual);
if (cnt == 0)
{
Assert.Inconclusive("Nothing is enumerated, but it is expected.");
}
Assert.AreEqual(expected, actual, "AlphaFS != System.IO");
}
catch (Exception ex)
{
allOk = false;
Console.WriteLine("\n\tCaught (UNEXPECTED) {0}: [{1}]", ex.GetType().FullName, ex.Message.Replace(Environment.NewLine, " "));
}
}
Console.WriteLine();
Console.WriteLine(UnitTestConstants.Reporter());
Directory.Delete(tempPath, true, true);
Assert.IsFalse(Directory.Exists(tempPath), "Cleanup failed: Directory should have been removed.");
Assert.IsTrue(allOk);
Console.WriteLine();
}
static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("DyZCm8aZbNXf2iZ6BV00wY2UoR3U2tymh3cftNZs");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains SQLite files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
// _fluentCommandLineParser.Setup(arg => arg.CsvName)
// .As("csvf")
// .WithDescription(
// "File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to.\r\n"); // This, --csv, or --xml required
// _fluentCommandLineParser.Setup(arg => arg.JsonName)
// .As("jsonf")
// .WithDescription(
// "File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.Dedupe)
.As("dedupe")
.WithDescription(
"Deduplicate -f or -d files based on SHA-1. First file found wins. Default is TRUE")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.Hunt)
.As("hunt")
.WithDescription(
"When true, all files are looked at regardless of name and file header is used to identify SQLite files, else filename in map is used to find databases. Default is FALSE\r\n ")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
_fluentCommandLineParser.Setup(arg => arg.Sync)
.As("sync")
.WithDescription(
"If true, the latest maps from https://github.com/EricZimmerman/SQLECmd/tree/master/SQLMap/Maps are downloaded and local maps updated. Default is FALSE\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"SQLECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/SQLECmd";
var footer =
@"Examples: SQLECmd.exe -f ""C:\Temp\someFile.db"" --csv ""c:\temp\out"" " +
"\r\n\t " +
@" SQLECmd.exe -d ""C:\Temp\"" --csv ""c:\temp\out""" + "\r\n\t " +
@" EvtxECmd.exe -d ""C:\Temp\"" --hunt --csv ""c:\temp\out""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.Sync)
{
try
{
_logger.Info(header);
UpdateFromRepo();
}
catch (Exception e)
{
_logger.Error(e, $"There was an error checking for updates: {e.Message}");
}
Environment.Exit(0);
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("--csv is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
DumpSqliteDll();
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Database maps will not be loaded!!");
}
else
{
_logger.Debug($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = SQLMap.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {SQLMap.MapFiles.Count:N0}");
}
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
ProcessFile(Path.GetFullPath(_fluentCommandLineParser.Object.File));
}
else
{
//Directories
_logger.Info($"Looking for files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var f = new DirectoryEnumerationFilters
{
RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
};
var dbNames = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
if (_fluentCommandLineParser.Object.Hunt)
{
f.InclusionFilter = fsei => true;
}
else
{
foreach (var mapFile in SQLMap.MapFiles)
{
dbNames.Add(mapFile.Value.FileName);
}
f.InclusionFilter = fsei => dbNames.Contains(fsei.FileName);
}
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), dirEnumOptions, f);
foreach (var file in files2)
{
ProcessFile(file);
}
}
sw.Stop();
if (_unmatchedDbs.Any())
{
Console.WriteLine();
_logger.Fatal($"At least one database was found with no corresponding map (Use --debug for more details about discovery process)");
foreach (var unmatchedDb in _unmatchedDbs)
{
DumpUnmatched(unmatchedDb);
}
}
var extra = string.Empty;
if (_processedFiles.Count > 1)
{
extra = "s";
}
_logger.Info($"\r\nProcessed {_processedFiles.Count:N0} file{extra} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (File.Exists("SQLite.Interop.dll"))
{
try
{
File.Delete("SQLite.Interop.dll");
}
catch (Exception)
{
_logger.Warn("Unable to delete 'SQLite.Interop.dll'. Delete manually if needed.\r\n");
}
}
}
/// <summary>
/// Write an input stream to an output folder
/// </summary>
/// <param name="inputStream">Input stream to be moved</param>
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">DatItem representing the new information</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise (default)</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
/// <returns>True if the write was a success, false otherwise</returns>
/// <remarks>This works for now, but it can be sped up by using Ionic.Zip or another zlib wrapper that allows for header values built-in. See edc's code.</remarks>
public virtual bool Write(Stream inputStream, string outDir, Rom rom, bool date = false, bool romba = false)
{
bool success = false;
// If either input is null or empty, return
if (inputStream == null || rom == null || rom.Name == null)
{
return(success);
}
// If the stream is not readable, return
if (!inputStream.CanRead)
{
return(success);
}
// Set internal variables
FileStream outputStream = null;
// Get the output folder name from the first rebuild rom
string fileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(rom.MachineName), Utilities.RemovePathUnsafeCharacters(rom.Name));
try
{
// If the full output path doesn't exist, create it
if (!Directory.Exists(Path.GetDirectoryName(fileName)))
{
Directory.CreateDirectory(Path.GetDirectoryName(fileName));
}
// Overwrite output files by default
outputStream = Utilities.TryCreate(fileName);
// If the output stream isn't null
if (outputStream != null)
{
// Copy the input stream to the output
inputStream.Seek(0, SeekOrigin.Begin);
int bufferSize = 4096 * 128;
byte[] ibuffer = new byte[bufferSize];
int ilen;
while ((ilen = inputStream.Read(ibuffer, 0, bufferSize)) > 0)
{
outputStream.Write(ibuffer, 0, ilen);
outputStream.Flush();
}
outputStream.Dispose();
if (rom.ItemType == ItemType.Rom)
{
if (date && !String.IsNullOrWhiteSpace(((Rom)rom).Date))
{
File.SetCreationTime(fileName, DateTime.Parse(((Rom)rom).Date));
}
}
success = true;
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
success = false;
}
finally
{
inputStream.Dispose();
outputStream?.Dispose();
}
return(success);
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains evtx files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription(
"File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to."); // This, --csv, or --xml required
_fluentCommandLineParser.Setup(arg => arg.JsonName)
.As("jsonf")
.WithDescription(
"File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.XmlDirectory)
.As("xml")
.WithDescription(
"Directory to save XML formatted results to."); // This, --csv, or --json required
_fluentCommandLineParser.Setup(arg => arg.XmlName)
.As("xmlf")
.WithDescription(
"File name to save XML formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff")
.SetDefault("yyyy-MM-dd HH:mm:ss.fffffff");
_fluentCommandLineParser.Setup(arg => arg.IncludeIds)
.As("inc")
.WithDescription(
"List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.ExcludeIds)
.As("exc")
.WithDescription(
"List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.FullJson)
.As("fj")
.WithDescription(
"When true, export all available data when using --json. Default is FALSE.")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Metrics)
.As("met")
.WithDescription(
"When true, show metrics about processed event log. Default is TRUE.\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/evtx";
var footer =
@"Examples: EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out"" --csvf MyOutputFile.csv" +
"\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out""" + "\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --json ""c:\temp\jsonout""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (IsAdministrator() == false)
{
_logger.Fatal("Warning: Administrator privileges not found!\r\n");
}
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName);
_logger.Warn($"json output will be saved to '{outFile}'\r\n");
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName);
_logger.Warn($"XML output will be saved to '{outFile}'\r\n");
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_logger.Warn($"CSV output will be saved to '{outFile}'\r\n");
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
_csvWriter = new CsvWriter(_swCsv);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
var foo = _csvWriter.Configuration.AutoMap <EventRecord>();
foo.Map(t => t.PayloadXml).Ignore();
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.TimeCreated).Index(1);
foo.Map(t => t.TimeCreated).ConvertUsing(t =>
$"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}");
foo.Map(t => t.EventId).Index(2);
foo.Map(t => t.Level).Index(3);
foo.Map(t => t.Provider).Index(4);
foo.Map(t => t.Channel).Index(5);
foo.Map(t => t.ProcessId).Index(6);
foo.Map(t => t.ThreadId).Index(7);
foo.Map(t => t.Computer).Index(8);
foo.Map(t => t.UserId).Index(9);
foo.Map(t => t.MapDescription).Index(10);
foo.Map(t => t.UserName).Index(11);
foo.Map(t => t.RemoteHost).Index(12);
foo.Map(t => t.PayloadData1).Index(13);
foo.Map(t => t.PayloadData2).Index(14);
foo.Map(t => t.PayloadData3).Index(15);
foo.Map(t => t.PayloadData4).Index(16);
foo.Map(t => t.PayloadData5).Index(17);
foo.Map(t => t.PayloadData6).Index(18);
foo.Map(t => t.ExecutableInfo).Index(19);
foo.Map(t => t.SourceFile).Index(20);
_csvWriter.Configuration.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!");
}
else
{
_logger.Debug($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}");
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false)
{
var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
ProcessFile(_fluentCommandLineParser.Object.File);
}
else
{
_logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
var f = new DirectoryEnumerationFilters();
f.InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX";
f.RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink;
f.ErrorFilter = (errorCode, errorMessage, pathProcessed) => true;
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(_fluentCommandLineParser.Object.Directory, dirEnumOptions, f);
foreach (var file in files2)
{
ProcessFile(file);
}
}
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
sw.Stop();
_logger.Info("");
var suff = string.Empty;
if (_fileCount != 1)
{
suff = "s";
}
_logger.Error(
$"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (_errorFiles.Count > 0)
{
_logger.Info("");
_logger.Error("Files with errors");
foreach (var errorFile in _errorFiles)
{
_logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}");
}
_logger.Info("");
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains evtx files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription(
"File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to."); // This, --csv, or --xml required
_fluentCommandLineParser.Setup(arg => arg.JsonName)
.As("jsonf")
.WithDescription(
"File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.XmlDirectory)
.As("xml")
.WithDescription(
"Directory to save XML formatted results to."); // This, --csv, or --json required
_fluentCommandLineParser.Setup(arg => arg.XmlName)
.As("xmlf")
.WithDescription(
"File name to save XML formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff")
.SetDefault("yyyy-MM-dd HH:mm:ss.fffffff");
_fluentCommandLineParser.Setup(arg => arg.IncludeIds)
.As("inc")
.WithDescription(
"List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.ExcludeIds)
.As("exc")
.WithDescription(
"List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.StartDate)
.As("sd")
.WithDescription(
"Start date for including events (UTC). Anything OLDER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.EndDate)
.As("ed")
.WithDescription(
"End date for including events (UTC). Anything NEWER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.FullJson)
.As("fj")
.WithDescription(
"When true, export all available data when using --json. Default is FALSE.")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.PayloadAsJson)
.As("pj")
.WithDescription(
"When true, include event *payload* as json. Default is TRUE.")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.TimeDiscrepancyThreshold)
.As("tdt")
.WithDescription(
"The number of seconds to use for time discrepancy detection. Default is 1 second")
.SetDefault(1);
_fluentCommandLineParser.Setup(arg => arg.Metrics)
.As("met")
.WithDescription(
"When true, show metrics about processed event log. Default is TRUE.\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
_fluentCommandLineParser.Setup(arg => arg.Vss)
.As("vss")
.WithDescription(
"Process all Volume Shadow Copies that exist on drive specified by -f or -d . Default is FALSE")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Dedupe)
.As("dedupe")
.WithDescription(
"Deduplicate -f or -d & VSCs based on SHA-1. First file found wins. Default is TRUE\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.Sync)
.As("sync")
.WithDescription(
"If true, the latest maps from https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps are downloaded and local maps updated. Default is FALSE\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/evtx";
var footer =
@"Examples: EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out"" --csvf MyOutputFile.csv" +
"\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out""" + "\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --json ""c:\temp\jsonout""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.Sync)
{
try
{
_logger.Info(header);
UpdateFromRepo();
}
catch (Exception e)
{
_logger.Error(e, $"There was an error checking for updates: {e.Message}");
}
Environment.Exit(0);
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (IsAdministrator() == false)
{
_logger.Fatal("Warning: Administrator privileges not found!\r\n");
}
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
if (_fluentCommandLineParser.Object.Vss & (IsAdministrator() == false))
{
_logger.Error("--vss is present, but administrator rights not found. Exiting\r\n");
return;
}
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName);
_logger.Warn($"json output will be saved to '{outFile}'\r\n");
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName);
_logger.Warn($"XML output will be saved to '{outFile}'\r\n");
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.StartDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.StartDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_startDate = dt;
_logger.Info($"Setting Start date to '{_startDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.StartDate}' to a valud datetime! Events will not be filtered by Start date!");
}
}
if (_fluentCommandLineParser.Object.EndDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.EndDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_endDate = dt;
_logger.Info($"Setting End date to '{_endDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.EndDate}' to a valud datetime! Events will not be filtered by End date!");
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
_logger.Info("");
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_logger.Warn($"CSV output will be saved to '{outFile}'\r\n");
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
_csvWriter = new CsvWriter(_swCsv, CultureInfo.InvariantCulture);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
var foo = _csvWriter.Configuration.AutoMap <EventRecord>();
if (_fluentCommandLineParser.Object.PayloadAsJson == false)
{
foo.Map(t => t.Payload).Ignore();
}
else
{
foo.Map(t => t.Payload).Index(22);
}
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).ConvertUsing(t =>
$"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.HiddenRecord).Index(21);
foo.Map(t => t.SourceFile).Index(22);
foo.Map(t => t.Keywords).Index(23);
_csvWriter.Configuration.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!");
}
else
{
_logger.Debug($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}");
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false)
{
var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.Vss)
{
string driveLetter;
if (_fluentCommandLineParser.Object.File.IsEmpty() == false)
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File))
.Substring(0, 1);
}
else
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory))
.Substring(0, 1);
}
Helper.MountVss(driveLetter, VssDir);
Console.WriteLine();
}
EventLog.TimeDiscrepancyThreshold = _fluentCommandLineParser.Object.TimeDiscrepancyThreshold;
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
_fluentCommandLineParser.Object.Dedupe = false;
ProcessFile(Path.GetFullPath(_fluentCommandLineParser.Object.File));
if (_fluentCommandLineParser.Object.Vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File));
var stem = Path.GetFullPath(_fluentCommandLineParser.Object.File).Replace(root, "");
foreach (var vssDir in vssDirs)
{
var newPath = Path.Combine(vssDir, stem);
if (File.Exists(newPath))
{
ProcessFile(newPath);
}
}
}
}
else
{
_logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
var f = new DirectoryEnumerationFilters
{
InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
};
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), dirEnumOptions, f);
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file);
}
if (_fluentCommandLineParser.Object.Vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
Console.WriteLine();
foreach (var vssDir in vssDirs)
{
var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory));
var stem = Path.GetFullPath(_fluentCommandLineParser.Object.Directory).Replace(root, "");
var target = Path.Combine(vssDir, stem);
_logger.Fatal($"\r\nSearching 'VSS{target.Replace($"{VssDir}\\","")}' for event logs...");
var vssFiles = Helper.GetFilesFromPath(target, true, "*.evtx");
foreach (var file in vssFiles)
{
ProcessFile(file);
}
}
}
}
try
{
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
}
catch (Exception e)
{
_logger.Error($"Error when flushing output files to disk! Error message: {e.Message}");
}
sw.Stop();
_logger.Info("");
var suff = string.Empty;
if (_fileCount != 1)
{
suff = "s";
}
_logger.Error(
$"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (_errorFiles.Count > 0)
{
_logger.Info("");
_logger.Error("Files with errors");
foreach (var errorFile in _errorFiles)
{
_logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}");
}
_logger.Info("");
}
if (_fluentCommandLineParser.Object.Vss)
{
if (Directory.Exists(VssDir))
{
foreach (var directory in Directory.GetDirectories(VssDir))
{
Directory.Delete(directory);
}
Directory.Delete(VssDir, true, true);
}
}
}
private static void Main(string[] args)
{
SetupNLog();
SetupPatterns();
_logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to search. Either this or -d is required");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to recursively process. Either this or -f is required");
_fluentCommandLineParser.Setup(arg => arg.SaveTo)
.As('o')
.WithDescription("File to save results to");
_fluentCommandLineParser.Setup(arg => arg.GetAscii)
.As('a')
.SetDefault(true)
.WithDescription("If set, look for ASCII strings. Default is true. Use -a false to disable");
_fluentCommandLineParser.Setup(arg => arg.GetUnicode)
.As('u')
.SetDefault(true)
.WithDescription("If set, look for Unicode strings. Default is true. Use -u false to disable");
_fluentCommandLineParser.Setup(arg => arg.MinimumLength)
.As('m').SetDefault(3).WithDescription("Minimum string length. Default is 3");
_fluentCommandLineParser.Setup(arg => arg.BlockSizeMb)
.As('b').SetDefault(512).WithDescription("Chunk size in MB. Valid range is 1 to 1024. Default is 512");
_fluentCommandLineParser.Setup(arg => arg.Quiet)
.As('q').SetDefault(false).WithDescription("Quiet mode (Do not show header or total number of hits)");
_fluentCommandLineParser.Setup(arg => arg.QuietQuiet)
.As('s')
.SetDefault(false)
.WithDescription(
"Really Quiet mode (Do not display hits to console. Speeds up processing when using -o)");
_fluentCommandLineParser.Setup(arg => arg.MaximumLength)
.As('x').SetDefault(-1).WithDescription("Maximum string length. Default is unlimited\r\n");
_fluentCommandLineParser.Setup(arg => arg.GetPatterns)
.As('p').SetDefault(false).WithDescription("Display list of built in regular expressions");
_fluentCommandLineParser.Setup(arg => arg.LookForString)
.As("ls")
.SetDefault(string.Empty)
.WithDescription("String to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.LookForRegex)
.As("lr")
.SetDefault(string.Empty)
.WithDescription("Regex to look for. When set, only strings matching the regex are returned");
_fluentCommandLineParser.Setup(arg => arg.StringFile)
.As("fs")
.SetDefault(string.Empty)
.WithDescription("File containing strings to look for. When set, only matching strings are returned");
_fluentCommandLineParser.Setup(arg => arg.RegexFile)
.As("fr")
.SetDefault(string.Empty)
.WithDescription(
"File containing regex patterns to look for. When set, only strings matching regex patterns are returned\r\n");
_fluentCommandLineParser.Setup(arg => arg.AsciiRange)
.As("ar")
.SetDefault("[\x20-\x7E]")
.WithDescription(
@"Range of characters to search for in 'Code page' strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\x20 -\x7E]");
_fluentCommandLineParser.Setup(arg => arg.UnicodeRange)
.As("ur")
.SetDefault("[\u0020-\u007E]")
.WithDescription(
"Range of characters to search for in Unicode strings. Specify as a range of characters in hex format and enclose in quotes. Default is [\\u0020-\\u007E]\r\n");
_fluentCommandLineParser.Setup(arg => arg.CodePage)
.As("cp")
.SetDefault(1252)
.WithDescription(
"Code page to use. Default is 1252. Use the Identifier value for code pages at https://goo.gl/ig6DxW");
_fluentCommandLineParser.Setup(arg => arg.FileMask)
.As("mask")
.SetDefault(string.Empty)
.WithDescription(
"When using -d, file mask to search for. * and ? are supported. This option has no effect when using -f");
_fluentCommandLineParser.Setup(arg => arg.RegexOnly)
.As("ro")
.SetDefault(false)
.WithDescription(
"When true, list the string matched by regex pattern vs string the pattern was found in (This may result in duplicate strings in output. ~ denotes approx. offset)");
_fluentCommandLineParser.Setup(arg => arg.ShowOffset)
.As("off")
.SetDefault(false)
.WithDescription(
$"Show offset to hit after string, followed by the encoding (A={_fluentCommandLineParser.Object.CodePage}, U=Unicode)\r\n");
_fluentCommandLineParser.Setup(arg => arg.SortAlpha)
.As("sa").SetDefault(false).WithDescription("Sort results alphabetically");
_fluentCommandLineParser.Setup(arg => arg.SortLength)
.As("sl").SetDefault(false).WithDescription("Sort results by length");
var header =
$"bstrings version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/bstrings";
var footer = @"Examples: bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls URL" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someFile.txt"" --lr guid" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\aBigFile.bin"" --fs c:\temp\searchStrings.txt --fr c:\temp\searchRegex.txt -s" +
"\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --mask ""*.dll""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ar ""[\x20-\x37]""" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --cp 10007" + "\r\n\t " +
@" bstrings.exe -d ""C:\Temp"" --ls test" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\someOtherFile.txt"" --lr cc -sa -m 15 -x 22" + "\r\n\t " +
@" bstrings.exe -f ""C:\Temp\UsrClass 1.dat"" --ls mui -sl" + "\r\n\t ";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (_fluentCommandLineParser.Object.GetPatterns)
{
_logger.Warn("Name \t\tDescription");
foreach (var regExPattern in RegExPatterns.OrderBy(t => t.Key))
{
var desc = RegExDesc[regExPattern.Key];
_logger.Info($"{regExPattern.Key}\t{desc}");
}
_logger.Info("");
_logger.Info("To use a built in pattern, supply the Name to the --lr switch\r\n");
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("Either -f or -d is required. Exiting");
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false &&
!File.Exists(_fluentCommandLineParser.Object.File) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting");
return;
}
if (_fluentCommandLineParser.Object.Directory.IsNullOrEmpty() == false &&
!Directory.Exists(_fluentCommandLineParser.Object.Directory) &&
_fluentCommandLineParser.Object.FileMask.Length == 0)
{
_logger.Warn($"Directory '{_fluentCommandLineParser.Object.Directory}' not found. Exiting");
return;
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(header);
_logger.Info("");
}
var files = new List <string>();
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
files.Add(_fluentCommandLineParser.Object.File);
}
else
{
try
{
if (_fluentCommandLineParser.Object.FileMask.Length > 0)
{
files.AddRange(Directory.EnumerateFiles(_fluentCommandLineParser.Object.Directory,
_fluentCommandLineParser.Object.FileMask, SearchOption.AllDirectories));
}
else
{
files.AddRange(Directory.EnumerateFiles(_fluentCommandLineParser.Object.Directory, "*",
SearchOption.AllDirectories));
}
}
catch (Exception ex)
{
_logger.Error(
$"Error getting files in '{_fluentCommandLineParser.Object.Directory}'. Error message: {ex.Message}");
return;
}
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Command line: {string.Join(" ", args)}");
_logger.Info("");
}
StreamWriter sw = null;
var globalCounter = 0;
var globalHits = 0;
double globalTimespan = 0;
var withBoundaryHits = false;
if (_fluentCommandLineParser.Object.SaveTo.Length > 0)
{
var dir = Path.GetDirectoryName(_fluentCommandLineParser.Object.SaveTo);
if (dir != null && Directory.Exists(dir) == false)
{
try
{
Directory.CreateDirectory(dir);
}
catch (Exception)
{
_logger.Warn(
$"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}'. Results will not be saved to a file.");
_logger.Info("");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
else
{
if (dir == null)
{
_logger.Warn($"Invalid path: '{_fluentCommandLineParser.Object.SaveTo}");
_fluentCommandLineParser.Object.SaveTo = string.Empty;
}
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Saving hits to '{_fluentCommandLineParser.Object.SaveTo}'");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SaveTo.Length > 0)
{
sw = new StreamWriter(_fluentCommandLineParser.Object.SaveTo, true);
}
}
foreach (var file in files)
{
if (File.Exists(file) == false)
{
_logger.Warn($"'{file}' does not exist! Skipping");
}
_sw = new Stopwatch();
_sw.Start();
var counter = 0;
var hits = new HashSet <string>();
var regPattern = _fluentCommandLineParser.Object.LookForRegex;
if (RegExPatterns.ContainsKey(_fluentCommandLineParser.Object.LookForRegex))
{
regPattern = RegExPatterns[_fluentCommandLineParser.Object.LookForRegex];
}
if (regPattern.Length > 0 && !_fluentCommandLineParser.Object.Quiet)
{
_logger.Info($"Searching via RegEx pattern: {regPattern}");
_logger.Info("");
}
var minLength = 3;
if (_fluentCommandLineParser.Object.MinimumLength > 0)
{
minLength = _fluentCommandLineParser.Object.MinimumLength;
}
var maxLength = -1;
if (_fluentCommandLineParser.Object.MaximumLength > minLength)
{
maxLength = _fluentCommandLineParser.Object.MaximumLength;
}
var chunkSizeMb = _fluentCommandLineParser.Object.BlockSizeMb < 1 ||
_fluentCommandLineParser.Object.BlockSizeMb > 1024
? 512
: _fluentCommandLineParser.Object.BlockSizeMb;
var chunkSizeBytes = chunkSizeMb * 1024 * 1024;
var fileSizeBytes = new FileInfo(file).Length;
var bytesRemaining = fileSizeBytes;
long offset = 0;
var chunkIndex = 1;
var totalChunks = fileSizeBytes / chunkSizeBytes + 1;
var hsuffix = totalChunks == 1 ? "" : "s";
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Searching {totalChunks:N0} chunk{hsuffix} ({chunkSizeMb} MB each) across {GetSizeReadable(fileSizeBytes)} in '{file}'");
_logger.Info("");
}
try
{
using (
var mmf =
MemoryMappedFile.CreateFromFile(
File.Open(File.GetFileSystemEntryInfo(file).LongFullPath, FileMode.Open, FileAccess.Read,
FileShare.Read, PathFormat.LongFullPath), "source", 0, MemoryMappedFileAccess.Read,
HandleInheritability.None, false))
{
while (bytesRemaining > 0)
{
if (bytesRemaining <= chunkSizeBytes)
{
chunkSizeBytes = (int)bytesRemaining;
}
using (
var accessor = mmf.CreateViewStream(offset, chunkSizeBytes, MemoryMappedFileAccess.Read)
)
{
var chunk = new byte[chunkSizeBytes];
accessor.Read(chunk, 0, chunkSizeBytes);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(h);
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(h);
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
$"Chunk {chunkIndex:N0} of {totalChunks:N0} finished. Total strings so far: {hits.Count:N0} Elapsed time: {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
}
}
chunkIndex += 1;
}
//do chunk boundary checks to make sure we get everything and not split things
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info(
"Primary search complete. Looking for strings across chunk boundaries...");
}
bytesRemaining = fileSizeBytes;
chunkSizeBytes = chunkSizeMb * 1024 * 1024;
offset = chunkSizeBytes - _fluentCommandLineParser.Object.MinimumLength * 10 * 2;
//move starting point backwards for our starting point
chunkIndex = 0;
var boundaryChunkSize = _fluentCommandLineParser.Object.MinimumLength * 10 * 2 * 2;
//grab the same # of bytes on both sides of the boundary
while (bytesRemaining > 0)
{
if (offset + boundaryChunkSize > fileSizeBytes)
{
break;
}
using (
var accessor = mmf.CreateViewStream(offset, boundaryChunkSize,
MemoryMappedFileAccess.Read)
)
{
var chunk = new byte[boundaryChunkSize];
accessor.Read(chunk, 0, boundaryChunkSize);
if (_fluentCommandLineParser.Object.GetUnicode)
{
var uh = GetUnicodeHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in uh)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && uh.Count > 0)
{
withBoundaryHits = uh.Count > 0;
}
}
if (_fluentCommandLineParser.Object.GetAscii)
{
var ah = GetAsciiHits(chunk, minLength, maxLength, offset,
_fluentCommandLineParser.Object.ShowOffset);
foreach (var h in ah)
{
hits.Add(" " + h);
}
if (withBoundaryHits == false && ah.Count > 0)
{
withBoundaryHits = true;
}
}
offset += chunkSizeBytes;
bytesRemaining -= chunkSizeBytes;
}
chunkIndex += 1;
}
}
}
catch (Exception ex)
{
_logger.Info("");
_logger.Error($"Error: {ex.Message}");
}
_sw.Stop();
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Search complete.");
_logger.Info("");
}
if (_fluentCommandLineParser.Object.SortAlpha)
{
_logger.Info("Sorting alphabetically...");
_logger.Info("");
var tempList = hits.ToList();
tempList.Sort();
hits = new HashSet <string>(tempList);
}
else if (_fluentCommandLineParser.Object.SortLength)
{
_logger.Info("Sorting by length...");
_logger.Info("");
var tempList = SortByLength(hits.ToList()).ToList();
hits = new HashSet <string>(tempList);
}
var fileStrings = new HashSet <string>();
var regexStrings = new HashSet <string>();
//set up highlighting
if (_fluentCommandLineParser.Object.LookForString.Length > 0)
{
fileStrings.Add(_fluentCommandLineParser.Object.LookForString);
}
if (_fluentCommandLineParser.Object.LookForRegex.Length > 0)
{
regexStrings.Add(regPattern);
}
if (_fluentCommandLineParser.Object.StringFile.Length > 0 || _fluentCommandLineParser.Object.RegexFile.Length > 0)
{
if (_fluentCommandLineParser.Object.StringFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.StringFile))
{
fileStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.StringFile)));
}
else
{
_logger.Error($"Strings file '{_fluentCommandLineParser.Object.StringFile}' not found.");
}
}
if (_fluentCommandLineParser.Object.RegexFile.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.RegexFile))
{
regexStrings.UnionWith(new HashSet <string>(File.ReadAllLines(_fluentCommandLineParser.Object.RegexFile)));
}
else
{
_logger.Error($"Regex file '{_fluentCommandLineParser.Object.RegexFile}' not found.");
}
}
}
AddHighlightingRules(fileStrings.ToList());
if (_fluentCommandLineParser.Object.RegexOnly == false)
{
AddHighlightingRules(regexStrings.ToList(), true);
}
if (!_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("Processing strings...");
_logger.Info("");
}
foreach (var hit in hits)
{
if (hit.Length == 0)
{
continue;
}
if (fileStrings.Count > 0 || regexStrings.Count > 0)
{
foreach (var fileString in fileStrings)
{
if (fileString.Trim().Length == 0)
{
continue;
}
if (hit.IndexOf(fileString, StringComparison.InvariantCultureIgnoreCase) < 0)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
var hitoffset = "";
if (_fluentCommandLineParser.Object.ShowOffset)
{
hitoffset = $"~{hit.Split('\t').Last()}";
}
foreach (var regString in regexStrings)
{
if (regString.Trim().Length == 0)
{
continue;
}
try
{
var reg1 = new Regex(regString,
RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
if (reg1.IsMatch(hit) == false)
{
continue;
}
counter += 1;
if (_fluentCommandLineParser.Object.RegexOnly)
{
foreach (var match in reg1.Matches(hit))
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info($"{match}\t{hitoffset}");
}
sw?.WriteLine($"{match}\t{hitoffset}");
}
}
else
{
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
catch (Exception ex)
{
_logger.Error($"Error setting up regular expression '{regString}': {ex.Message}");
}
}
}
else
{
//dump all strings
counter += 1;
if (_fluentCommandLineParser.Object.QuietQuiet == false)
{
_logger.Info(hit);
}
sw?.WriteLine(hit);
}
}
if (_fluentCommandLineParser.Object.Quiet)
{
continue;
}
var suffix = counter == 1 ? "" : "s";
_logger.Info("");
if (withBoundaryHits)
{
_logger.Info("** Strings prefixed with 2 spaces are hits found across chunk boundaries **");
_logger.Info("");
}
_logger.Info(
$"Found {counter:N0} string{suffix} in {_sw.Elapsed.TotalSeconds:N3} seconds. Average strings/sec: {hits.Count/_sw.Elapsed.TotalSeconds:N0}");
globalCounter += counter;
globalHits += hits.Count;
globalTimespan += _sw.Elapsed.TotalSeconds;
if (files.Count > 1)
{
_logger.Info(
"-------------------------------------------------------------------------------------\r\n");
}
}
if (sw != null)
{
sw.Flush();
sw.Close();
}
if (!_fluentCommandLineParser.Object.Quiet && files.Count > 1)
{
var suffix = globalCounter == 1 ? "" : "s";
_logger.Info("");
_logger.Info(
$"Found {globalCounter:N0} string{suffix} in {globalTimespan:N3} seconds. Average strings/sec: {globalHits/globalTimespan:N0}");
}
}
private static void DoWork(string f, string d, string csv, string csvf, string json, string jsonf, string xml, string xmlf, string dt, string inc, string exc, string sd, string ed, bool fj, int tdt, bool met, string maps, bool vss, bool dedupe, bool sync, bool debug, bool trace)
{
var levelSwitch = new LoggingLevelSwitch();
_activeDateTimeFormat = dt;
var formatter =
new DateTimeOffsetFormatter(CultureInfo.CurrentCulture);
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template, formatProvider: formatter)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
if (sync)
{
try
{
Log.Information("{Header}", Header);
UpdateFromRepo();
}
catch (Exception e)
{
Log.Error(e, "There was an error checking for updates: {Message}", e.Message);
}
Environment.Exit(0);
}
if (f.IsNullOrEmpty() &&
d.IsNullOrEmpty())
{
var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth);
var hc = new HelpContext(helpBld, _rootCommand, Console.Out);
helpBld.Write(hc);
Log.Warning("-f or -d is required. Exiting");
Console.WriteLine();
return;
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", Environment.GetCommandLineArgs().Skip(1)));
Console.WriteLine();
if (IsAdministrator() == false)
{
Log.Warning("Warning: Administrator privileges not found!");
Console.WriteLine();
}
if (vss & !RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
vss = false;
Log.Warning("{Vss} not supported on non-Windows platforms. Disabling...", "--vss");
Console.WriteLine();
}
if (vss & (IsAdministrator() == false))
{
Log.Error("{Vss} is present, but administrator rights not found. Exiting", "--vss");
Console.WriteLine();
return;
}
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (json.IsNullOrEmpty() == false)
{
if (Directory.Exists(json) == false)
{
Log.Information("Path to {Json} doesn't exist. Creating...", json);
try
{
Directory.CreateDirectory(json);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Json}. Does a file with the same name exist? Exiting", json);
Console.WriteLine();
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (jsonf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(jsonf);
}
var outFile = Path.Combine(json, outName);
Log.Information("json output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (xml.IsNullOrEmpty() == false)
{
if (Directory.Exists(xml) == false)
{
Log.Information("Path to {Xml} doesn't exist. Creating...", xml);
try
{
Directory.CreateDirectory(xml);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Xml}. Does a file with the same name exist? Exiting", xml);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (xmlf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(xmlf);
}
var outFile = Path.Combine(xml, outName);
Log.Information("XML output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
}
if (sd.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(sd, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_startDate = dateTimeOffset;
Log.Information("Setting Start date to {StartDate}", _startDate.Value);
}
else
{
Log.Warning("Could not parse {Sd} to a valid datetime! Events will not be filtered by Start date!", sd);
}
}
if (ed.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(ed, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_endDate = dateTimeOffset;
Log.Information("Setting End date to {EndDate}", _endDate.Value);
}
else
{
Log.Warning("Could not parse {Ed} to a valid datetime! Events will not be filtered by End date!", ed);
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
Console.WriteLine();
}
if (csv.IsNullOrEmpty() == false)
{
if (Directory.Exists(csv) == false)
{
Log.Information(
"Path to {Csv} doesn't exist. Creating...", csv);
try
{
Directory.CreateDirectory(csv);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Csv}. Does a file with the same name exist? Exiting", csv);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (csvf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(csvf);
}
var outFile = Path.Combine(csv, outName);
Log.Information("CSV output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
var opt = new CsvConfiguration(CultureInfo.InvariantCulture)
{
ShouldUseConstructorParameters = _ => false
};
_csvWriter = new CsvWriter(_swCsv, opt);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
var foo = _csvWriter.Context.AutoMap <EventRecord>();
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).Convert(t =>
$"{t.Value.TimeCreated.ToString(dt)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.HiddenRecord).Index(21);
foo.Map(t => t.SourceFile).Index(22);
foo.Map(t => t.Keywords).Index(23);
foo.Map(t => t.Payload).Index(24);
_csvWriter.Context.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(maps) == false)
{
Log.Warning("Maps directory {Maps} does not exist! Event ID maps will not be loaded!!", maps);
}
else
{
Log.Debug("Loading maps from {Path}", Path.GetFullPath(maps));
var errors = EventLog.LoadMaps(Path.GetFullPath(maps));
if (errors)
{
return;
}
Log.Information("Maps loaded: {Count:N0}", EventLog.EventLogMaps.Count);
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (exc.IsNullOrEmpty() == false)
{
var excSegs = exc.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (inc.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = inc.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (vss)
{
string driveLetter;
if (f.IsEmpty() == false)
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(f))
.Substring(0, 1);
}
else
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(d))
.Substring(0, 1);
}
Helper.MountVss(driveLetter, VssDir);
Console.WriteLine();
}
EventLog.TimeDiscrepancyThreshold = tdt;
if (f.IsNullOrEmpty() == false)
{
if (File.Exists(f) == false)
{
Log.Warning("\t{F} does not exist! Exiting", f);
Console.WriteLine();
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
dedupe = false;
ProcessFile(Path.GetFullPath(f), dedupe, fj, met);
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
var root = Path.GetPathRoot(Path.GetFullPath(f));
var stem = Path.GetFullPath(f).Replace(root, "");
foreach (var vssDir in vssDirs)
{
var newPath = Path.Combine(vssDir, stem);
if (File.Exists(newPath))
{
ProcessFile(newPath, dedupe, fj, met);
}
}
}
}
else
{
if (Directory.Exists(d) == false)
{
Log.Warning("\t{D} does not exist! Exiting", d);
Console.WriteLine();
return;
}
Log.Information("Looking for event log files in {D}", d);
Console.WriteLine();
#if !NET6_0
var directoryEnumerationFilters = new DirectoryEnumerationFilters
{
InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
};
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(d), dirEnumOptions, directoryEnumerationFilters);
#else
var enumerationOptions = new EnumerationOptions
{
IgnoreInaccessible = true,
MatchCasing = MatchCasing.CaseInsensitive,
RecurseSubdirectories = true,
AttributesToSkip = 0
};
var files2 =
Directory.EnumerateFileSystemEntries(d, "*.evtx", enumerationOptions);
#endif
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file, dedupe, fj, met);
}
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
Console.WriteLine();
foreach (var vssDir in vssDirs)
{
var root = Path.GetPathRoot(Path.GetFullPath(d));
var stem = Path.GetFullPath(d).Replace(root, "");
var target = Path.Combine(vssDir, stem);
Console.WriteLine();
Log.Information("Searching {Vss} for event logs...", $"VSS{target.Replace($"{VssDir}\\", "")}");
var vssFiles = Helper.GetFilesFromPath(target, "*.evtx", true);
foreach (var file in vssFiles)
{
ProcessFile(file, dedupe, fj, met);
}
}
}
}
try
{
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
}
catch (Exception e)
{
Log.Error(e, "Error when flushing output files to disk! Error message: {Message}", e.Message);
}
sw.Stop();
Console.WriteLine();
if (_fileCount == 1)
{
Log.Information("Processed {FileCount:N0} file in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
else
{
Log.Information("Processed {FileCount:N0} files in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
Console.WriteLine();
if (_errorFiles.Count > 0)
{
Console.WriteLine();
Log.Information("Files with errors");
foreach (var errorFile in _errorFiles)
{
Log.Information("{Key} error count: {Value:N0}", errorFile.Key, errorFile.Value);
}
Console.WriteLine();
}
if (vss)
{
if (Directory.Exists(VssDir))
{
foreach (var directory in Directory.GetDirectories(VssDir))
{
Directory.Delete(directory);
}
#if !NET6_0
Directory.Delete(VssDir, true, true);
#else
Directory.Delete(VssDir, true);
#endif
}
}
}
private static void UpdateFromRepo()
{
Console.WriteLine();
Log.Information("Checking for updated maps at {Url}...", "https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps");
Console.WriteLine();
var archivePath = Path.Combine(BaseDirectory, "____master.zip");
if (File.Exists(archivePath))
{
File.Delete(archivePath);
}
// using (var client = new WebClient())
// {
// ServicePointManager.Expect100Continue = true;
// ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// client.DownloadFile("https://github.com/EricZimmerman/evtx/archive/master.zip", archivePath);
// }
"https://github.com/EricZimmerman/evtx/archive/master.zip".DownloadFileTo(archivePath);
var fff = new FastZip();
if (Directory.Exists(Path.Combine(BaseDirectory, "Maps")) == false)
{
Directory.CreateDirectory(Path.Combine(BaseDirectory, "Maps"));
}
var directoryFilter = "Maps";
// Will prompt to overwrite if target file names already exist
fff.ExtractZip(archivePath, BaseDirectory, FastZip.Overwrite.Always, null,
null, directoryFilter, true);
if (File.Exists(archivePath))
{
File.Delete(archivePath);
}
var newMapPath = Path.Combine(BaseDirectory, "evtx-master", "evtx", "Maps");
var orgMapPath = Path.Combine(BaseDirectory, "Maps");
var newMaps = Directory.GetFiles(newMapPath);
var newLocalMaps = new List <string>();
var updatedLocalMaps = new List <string>();
if (File.Exists(Path.Combine(orgMapPath, "Security_4624.map")))
{
Log.Warning("Old map format found. Zipping to {Old} and cleaning up old maps", "!!oldMaps.zip");
//old maps found, so zip em first
var oldZip = Path.Combine(orgMapPath, "!!oldMaps.zip");
fff.CreateZip(oldZip, orgMapPath, false, @"\.map$");
foreach (var m in Directory.GetFiles(orgMapPath, "*.map"))
{
File.Delete(m);
}
}
if (File.Exists(Path.Combine(orgMapPath, "!!!!README.txt")))
{
File.Delete(Path.Combine(orgMapPath, "!!!!README.txt"));
}
foreach (var newMap in newMaps)
{
var mName = Path.GetFileName(newMap);
var dest = Path.Combine(orgMapPath, mName);
if (File.Exists(dest) == false)
{
//new target
newLocalMaps.Add(mName);
}
else
{
//current destination file exists, so compare to new
var s1 = new StreamReader(newMap);
var newSha = Helper.GetSha1FromStream(s1.BaseStream, 0);
var s2 = new StreamReader(dest);
var destSha = Helper.GetSha1FromStream(s2.BaseStream, 0);
s2.Close();
s1.Close();
if (newSha != destSha)
{
//updated file
updatedLocalMaps.Add(mName);
}
}
#if !NET6_0
File.Copy(newMap, dest, CopyOptions.None);
#else
File.Copy(newMap, dest, true);
#endif
}
if (newLocalMaps.Count > 0 || updatedLocalMaps.Count > 0)
{
Log.Information("Updates found!");
Console.WriteLine();
if (newLocalMaps.Count > 0)
{
Log.Information("New maps");
foreach (var newLocalMap in newLocalMaps)
{
Log.Information("{File}", Path.GetFileNameWithoutExtension(newLocalMap));
}
Console.WriteLine();
}
if (updatedLocalMaps.Count > 0)
{
Log.Information("Updated maps");
foreach (var um in updatedLocalMaps)
{
Log.Information("{File}", Path.GetFileNameWithoutExtension(um));
}
Console.WriteLine();
}
Console.WriteLine();
}
else
{
Console.WriteLine();
Log.Information("No new maps available");
Console.WriteLine();
}
Directory.Delete(Path.Combine(BaseDirectory, "evtx-master"), true);
}
private static void UpdateFromRepo()
{
Console.WriteLine();
_logger.Info(
"Checking for updated maps at https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps...");
Console.WriteLine();
var archivePath = Path.Combine(BaseDirectory, "____master.zip");
if (File.Exists(archivePath))
{
File.Delete(archivePath);
}
using (var client = new WebClient())
{
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
client.DownloadFile("https://github.com/EricZimmerman/evtx/archive/master.zip", archivePath);
}
var fff = new FastZip();
if (Directory.Exists(Path.Combine(BaseDirectory, "Maps")) == false)
{
Directory.CreateDirectory(Path.Combine(BaseDirectory, "Maps"));
}
var directoryFilter = "Maps";
// Will prompt to overwrite if target file names already exist
fff.ExtractZip(archivePath, BaseDirectory, FastZip.Overwrite.Always, null,
null, directoryFilter, true);
if (File.Exists(archivePath))
{
File.Delete(archivePath);
}
var newMapPath = Path.Combine(BaseDirectory, "evtx-master", "evtx", "Maps");
var orgMapPath = Path.Combine(BaseDirectory, "Maps");
var newMaps = Directory.GetFiles(newMapPath);
var newlocalMaps = new List <string>();
var updatedlocalMaps = new List <string>();
if (File.Exists(Path.Combine(orgMapPath, "Security_4624.map")))
{
_logger.Warn($"Old map format found. Zipping to '!!oldMaps.zip' and cleaning up old maps");
//old maps found, so zip em first
var oldZip = Path.Combine(orgMapPath, "!!oldMaps.zip");
fff.CreateZip(oldZip, orgMapPath, false, @"\.map$");
foreach (var m in Directory.GetFiles(orgMapPath, "*.map"))
{
File.Delete(m);
}
}
if (File.Exists(Path.Combine(orgMapPath, "!!!!README.txt")))
{
File.Delete(Path.Combine(orgMapPath, "!!!!README.txt"));
}
foreach (var newMap in newMaps)
{
var mName = Path.GetFileName(newMap);
var dest = Path.Combine(orgMapPath, mName);
if (File.Exists(dest) == false)
{
//new target
newlocalMaps.Add(mName);
}
else
{
//current destination file exists, so compare to new
var fiNew = new FileInfo(newMap);
var fi = new FileInfo(dest);
if (fiNew.GetHash(HashType.SHA1) != fi.GetHash(HashType.SHA1))
{
//updated file
updatedlocalMaps.Add(mName);
}
}
File.Copy(newMap, dest, CopyOptions.None);
}
if (newlocalMaps.Count > 0 || updatedlocalMaps.Count > 0)
{
_logger.Fatal("Updates found!");
Console.WriteLine();
if (newlocalMaps.Count > 0)
{
_logger.Error("New maps");
foreach (var newLocalMap in newlocalMaps)
{
_logger.Info(Path.GetFileNameWithoutExtension(newLocalMap));
}
Console.WriteLine();
}
if (updatedlocalMaps.Count > 0)
{
_logger.Error("Updated maps");
foreach (var um in updatedlocalMaps)
{
_logger.Info(Path.GetFileNameWithoutExtension(um));
}
Console.WriteLine();
}
Console.WriteLine();
}
else
{
Console.WriteLine();
_logger.Info("No new maps available");
Console.WriteLine();
}
Directory.Delete(Path.Combine(BaseDirectory, "evtx-master"), true);
}
static void Main(string[] args)
{
Exceptionless.ExceptionlessClient.Default.Startup("fTcEOUkt1CxljTyOZfsr8AcSGQwWE4aYaYqk7cE1");
SetupNLog();
_logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription(
"Directory to look for hives (recursively). -f or -d is required.");
_fluentCommandLineParser.Setup(arg => arg.HiveFile)
.As('f')
.WithDescription("Hive to process. -f or -d is required.\r\n");
_fluentCommandLineParser.Setup(arg => arg.OutDirectory)
.As("out")
.WithDescription(
"Directory to save updated hives to. Only dirty hives with logs applied will end up in --out directory\r\n");
_fluentCommandLineParser.Setup(arg => arg.CopyAlways)
.As("ca")
.WithDescription(
"When true, always copy hives to --out directory, even if they aren't dirty. Default is TRUE").SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.CompressNames)
.As("cn")
.WithDescription(
"When true, compress names for profile based hives. Default is TRUE\r\n").SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing").SetDefault(false);
var header =
$"rla version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/RECmd\r\n\r\nNote: Enclose all strings containing spaces (and all RegEx) with double quotes";
var footer = @"Example: rla.exe --f ""C:\Temp\UsrClass 1.dat"" --out C:\temp" +
"\r\n\t " +
@"rla.exe --d ""D:\temp\"" --out c:\temp" + "\r\n";
_fluentCommandLineParser.SetupHelp("?", "help").WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (_fluentCommandLineParser.Object.HiveFile.IsNullOrEmpty() == false ||
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty() == false)
{
if (_fluentCommandLineParser.Object.OutDirectory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
Console.WriteLine();
_logger.Warn($"--out is required. Exiting");
Console.WriteLine();
return;
}
}
if (_fluentCommandLineParser.Object.Debug)
{
foreach (var r in LogManager.Configuration.LoggingRules)
{
r.EnableLoggingForLevel(LogLevel.Debug);
}
LogManager.ReconfigExistingLoggers();
_logger.Debug("Enabled debug messages...");
}
if (_fluentCommandLineParser.Object.Trace)
{
foreach (var r in LogManager.Configuration.LoggingRules)
{
r.EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
_logger.Trace("Enabled trace messages...");
}
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
var hivesToProcess = new List <string>();
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (_fluentCommandLineParser.Object.HiveFile?.Length > 0)
{
if (File.Exists(_fluentCommandLineParser.Object.HiveFile) == false)
{
_logger.Error($"File '{_fluentCommandLineParser.Object.HiveFile}' does not exist.");
return;
}
hivesToProcess.Add(_fluentCommandLineParser.Object.HiveFile);
}
else if (_fluentCommandLineParser.Object.Directory?.Length > 0)
{
if (Directory.Exists(_fluentCommandLineParser.Object.Directory) == false)
{
_logger.Error($"Directory '{_fluentCommandLineParser.Object.Directory}' does not exist.");
return;
}
var okFileParts = new HashSet <string>();
okFileParts.Add("USRCLASS");
okFileParts.Add("NTUSER");
okFileParts.Add("SYSTEM");
okFileParts.Add("SAM");
okFileParts.Add("SOFTWARE");
okFileParts.Add("AMCACHE");
okFileParts.Add("SYSCACHE");
okFileParts.Add("SECURITY");
okFileParts.Add("DRIVERS");
okFileParts.Add("COMPONENTS");
var f = new DirectoryEnumerationFilters();
f.InclusionFilter = fsei =>
{
if (fsei.Extension.ToUpperInvariant() == ".LOG1" || fsei.Extension.ToUpperInvariant() == ".LOG2" ||
fsei.Extension.ToUpperInvariant() == ".DLL" ||
fsei.Extension.ToUpperInvariant() == ".LOG" ||
fsei.Extension.ToUpperInvariant() == ".CSV" ||
fsei.Extension.ToUpperInvariant() == ".BLF" ||
fsei.Extension.ToUpperInvariant() == ".REGTRANS-MS" ||
fsei.Extension.ToUpperInvariant() == ".EXE" ||
fsei.Extension.ToUpperInvariant() == ".TXT" || fsei.Extension.ToUpperInvariant() == ".INI")
{
return(false);
}
var foundOkFilePart = false;
foreach (var okFilePart in okFileParts)
{
if (fsei.FileName.ToUpperInvariant().Contains(okFilePart))
{
foundOkFilePart = true;
// return true;
}
}
if (foundOkFilePart == false)
{
return(false);
}
var fi = new FileInfo(fsei.FullPath);
if (fi.Length < 4)
{
return(false);
}
try
{
using (var fs = new FileStream(fsei.FullPath, FileMode.Open, FileAccess.Read))
{
using (var br = new BinaryReader(fs, new ASCIIEncoding()))
{
try
{
var chunk = br.ReadBytes(4);
var sig = BitConverter.ToInt32(chunk, 0);
if (sig == 0x66676572)
{
return(true);
}
}
catch (Exception)
{
}
return(false);
}
}
}
catch (IOException)
{
if (Helper.IsAdministrator() == false)
{
throw new UnauthorizedAccessException("Administrator privileges not found!");
}
var files = new List <string>();
files.Add(fsei.FullPath);
var rawf = Helper.GetFiles(files);
if (rawf.First().FileStream.Length == 0)
{
return(false);
}
try
{
var b = new byte[4];
rawf.First().FileStream.ReadExactly(b, 4);
var sig = BitConverter.ToInt32(b, 0);
if (sig == 0x66676572)
{
return(true);
}
}
catch (Exception)
{
}
return(false);
}
};
f.RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink;
f.ErrorFilter = (errorCode, errorMessage, pathProcessed) => true;
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
if (Directory.Exists(_fluentCommandLineParser.Object.OutDirectory) == false)
{
_logger.Info($"Creating --out directory '{_fluentCommandLineParser.Object.OutDirectory}'...");
Directory.CreateDirectory(_fluentCommandLineParser.Object.OutDirectory);
}
else
{
if (Directory.GetFiles(_fluentCommandLineParser.Object.OutDirectory).Length > 0 && _fluentCommandLineParser.Object.CompressNames)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.OutDirectory}' contains files! This may cause --cn to revert back to uncompressed names. Ideally, '{_fluentCommandLineParser.Object.OutDirectory}' should be empty.");
Console.WriteLine();
}
}
_logger.Fatal($"Searching '{_fluentCommandLineParser.Object.Directory}' for hives...");
var files2 =
Directory.EnumerateFileSystemEntries(_fluentCommandLineParser.Object.Directory, dirEnumOptions, f);
var count = 0;
try
{
hivesToProcess.AddRange(files2);
count = hivesToProcess.Count;
_logger.Info($"\tHives found: {count:N0}");
}
catch (Exception ex)
{
_logger.Fatal($"Could not access all files in '{_fluentCommandLineParser.Object.Directory}'! Error: {ex.Message}");
_logger.Error("");
_logger.Fatal("Rerun the program with Administrator privileges to try again\r\n");
//Environment.Exit(-1);
}
}
else
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (hivesToProcess.Count == 0)
{
_logger.Warn("No hives were found. Exiting...");
return;
}
_sw = new Stopwatch();
_sw.Start();
foreach (var hiveToProcess in hivesToProcess)
{
_logger.Info("");
byte[] updatedBytes = null;
_logger.Info($"Processing hive '{hiveToProcess}'");
if (File.Exists(hiveToProcess) == false)
{
_logger.Warn($"'{hiveToProcess}' does not exist. Skipping");
continue;
}
try
{
RegistryHive reg;
var dirname = Path.GetDirectoryName(hiveToProcess);
var hiveBase = Path.GetFileName(hiveToProcess);
List <RawCopyReturn> rawFiles = null;
try
{
using (var fs = new FileStream(hiveToProcess, FileMode.Open, FileAccess.Read))
{
reg = new RegistryHive(fs.ReadFully(), hiveToProcess)
{
};
}
}
catch (IOException)
{
//file is in use
if (Helper.IsAdministrator() == false)
{
throw new UnauthorizedAccessException("Administrator privileges not found!");
}
_logger.Warn($"\t'{hiveToProcess}' is in use. Rerouting...\r\n");
var files = new List <string>();
files.Add(hiveToProcess);
var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?");
foreach (var logFile in logFiles)
{
files.Add(logFile);
}
rawFiles = Helper.GetFiles(files);
if (rawFiles.First().FileStream.Length == 0)
{
continue;
}
var bb = rawFiles.First().FileStream.ReadFully();
reg = new RegistryHive(bb, rawFiles.First().InputFilename);
}
if (reg.Header.PrimarySequenceNumber != reg.Header.SecondarySequenceNumber)
{
if (string.IsNullOrEmpty(dirname))
{
dirname = ".";
}
var logFiles = Directory.GetFiles(dirname, $"{hiveBase}.LOG?");
if (logFiles.Length == 0)
{
if (_fluentCommandLineParser.Object.CopyAlways)
{
_logger.Info($"\tHive '{hiveToProcess}' is dirty, but no logs were found in the same directory. --ca is true. Copying...");
updatedBytes = File.ReadAllBytes(hiveToProcess);
}
else
{
_logger.Info($"\tHive '{hiveToProcess}' is dirty and no transaction logs were found in the same directory. --ca is false. Skipping...");
continue;
}
}
if (updatedBytes == null)
{
if (rawFiles != null)
{
var lt = new List <TransactionLogFileInfo>();
foreach (var rawCopyReturn in rawFiles.Skip(1).ToList())
{
var bb1 = rawCopyReturn.FileStream.ReadFully();
var tt = new TransactionLogFileInfo(rawCopyReturn.InputFilename, bb1);
lt.Add(tt);
}
updatedBytes = reg.ProcessTransactionLogs(lt);
}
else
{
updatedBytes = reg.ProcessTransactionLogs(logFiles.ToList());
}
}
}
if (updatedBytes == null)
{
if (_fluentCommandLineParser.Object.CopyAlways)
{
_logger.Info($"\tHive '{hiveToProcess}' is not dirty, but --ca is true. Copying...");
updatedBytes = File.ReadAllBytes(hiveToProcess);
}
else
{
_logger.Info($"\tHive '{hiveToProcess}' is not dirty and --ca is false. Skipping...");
continue;
}
}
var outFile = hiveToProcess.Replace(":", "").Replace("\\", "_");
var outFileAll = Path.Combine(_fluentCommandLineParser.Object.OutDirectory, outFile);
if (_fluentCommandLineParser.Object.CompressNames &&
(outFileAll.ToUpperInvariant().Contains("NTUSER") || outFileAll.ToUpperInvariant().Contains("USRCLASS")))
{
var dl = hiveToProcess[0].ToString();
var segs = hiveToProcess.SplitAndTrim('\\');
var profile = segs[2];
var filename = Path.GetFileName(hiveToProcess);
var outFile2 = $"{dl}_{profile}_{filename}";
outFileAll = Path.Combine(_fluentCommandLineParser.Object.OutDirectory, outFile2);
}
if (File.Exists(outFileAll))
{
var oldOut = outFileAll;
outFileAll = Path.Combine(_fluentCommandLineParser.Object.OutDirectory, outFile);
_logger.Warn($"\tFile '{oldOut}' exists! Saving as non-compressed name: '{outFileAll}'");
}
_logger.Fatal($"\tSaving updated hive to '{outFileAll}'");
using (var fs = new FileStream(outFileAll, FileMode.Create))
{
fs.Write(updatedBytes, 0, updatedBytes.Length);
fs.Flush();
fs.Close();
}
}
catch (Exception ex)
{
if (ex.Message.Contains("Sequence numbers do not match and transaction") == false)
{
if (ex.Message.Contains("Administrator privileges not found"))
{
_logger.Fatal($"Could not access '{hiveToProcess}' because it is in use");
_logger.Error("");
_logger.Fatal("Rerun the program with Administrator privileges to try again\r\n");
}
else
{
_logger.Error($"There was an error: {ex.Message}");
}
}
}
}
_sw.Stop();
_logger.Info("");
_logger.Info($"Total processing time: {_sw.Elapsed.TotalSeconds:N3} seconds");
_logger.Info("");
}
protected override async Task <bool> _Begin(CancellationToken cancel)
{
if (cancel.IsCancellationRequested)
{
return(false);
}
var metric = Metrics.Send(Metrics.BeginInstall, ModList.Name);
ConfigureProcessor(20, ConstructDynamicNumThreads(await RecommendQueueSize()));
var game = ModList.GameType.MetaData();
if (GameFolder == null)
{
GameFolder = game.GameLocation();
}
if (GameFolder == null)
{
await Utils.Log(new CriticalFailureIntervention(
$"In order to do a proper install Wabbajack needs to know where your {game.MO2Name} folder resides. We tried looking the " +
"game location up in the Windows Registry but were unable to find it, please make sure you launch the game once before running this installer. ",
"Could not find game location")).Task;
Utils.Log("Exiting because we couldn't find the game folder.");
return(false);
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Game ESMs");
ValidateGameESMs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Validating Modlist");
await ValidateModlist.RunValidation(Queue, ModList);
Directory.CreateDirectory(OutputFolder);
Directory.CreateDirectory(DownloadFolder);
if (Directory.Exists(Path.Combine(OutputFolder, "mods")) && WarnOnOverwrite)
{
if ((await Utils.Log(new ConfirmUpdateOfExistingInstall {
ModListName = ModList.Name, OutputFolder = OutputFolder
}).Task) == ConfirmUpdateOfExistingInstall.Choice.Abort)
{
Utils.Log("Exiting installation at the request of the user, existing mods folder found.");
return(false);
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Optimizing ModList");
await OptimizeModlist();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Archives");
await HashArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Downloading Missing Archives");
await DownloadArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Hashing Remaining Archives");
await HashArchives();
var missing = ModList.Archives.Where(a => !HashedArchives.ContainsKey(a.Hash)).ToList();
if (missing.Count > 0)
{
foreach (var a in missing)
{
Info($"Unable to download {a.Name}");
}
if (IgnoreMissingFiles)
{
Info("Missing some archives, but continuing anyways at the request of the user");
}
else
{
Error("Cannot continue, was unable to download one or more archives");
}
}
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Priming VFS");
await PrimeVFS();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building Folder Structure");
BuildFolderStructure();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archives");
await InstallArchives();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Included files");
await InstallIncludedFiles();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Installing Archive Metas");
await InstallIncludedDownloadMetas();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Building BSAs");
await BuildBSAs();
if (cancel.IsCancellationRequested)
{
return(false);
}
UpdateTracker.NextStep("Generating Merges");
await zEditIntegration.GenerateMerges(this);
UpdateTracker.NextStep("Set MO2 into portable");
ForcePortable();
UpdateTracker.NextStep("Create Empty Output Mods");
CreateOutputMods();
UpdateTracker.NextStep("Updating System-specific ini settings");
SetScreenSizeInPrefs();
UpdateTracker.NextStep("Installation complete! You may exit the program.");
var metric2 = Metrics.Send(Metrics.FinishInstall, ModList.Name);
return(true);
}
public static IErrorResponse CheckValidInstallPath(string path, string downloadFolder)
{
var ret = Utils.IsDirectoryPathValid(path);
if (!ret.Succeeded)
{
return(ret);
}
if (!Directory.Exists(path))
{
return(ErrorResponse.Success);
}
// Check folder does not have a Wabbajack ModList
foreach (var file in Directory.EnumerateFiles(path))
{
if (!File.Exists(file))
{
continue;
}
if (System.IO.Path.GetExtension(file).Equals(ExtensionManager.Extension))
{
return(ErrorResponse.Fail($"Cannot install into a folder with a Wabbajack ModList inside of it"));
}
}
// Check folder is either empty, or a likely valid previous install
if (!Directory.IsEmpty(path))
{
// If we have a MO2 install, assume good to go
if (Directory.EnumerateFiles(path).Any(file =>
{
var fileName = Path.GetFileName(file);
if (fileName.Equals("ModOrganizer.exe", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
if (fileName.Equals("ModOrganizer.ini", StringComparison.OrdinalIgnoreCase))
{
return(true);
}
return(false);
}))
{
return(ErrorResponse.Success);
}
// If we don't have a MO2 install, and there's any file that's not in the downloads folder, mark failure
if (Directory.EnumerateFiles(path).Any(file =>
{
var fileName = Path.GetFileName(file);
if (string.IsNullOrWhiteSpace(downloadFolder))
{
return(true);
}
return(!Utils.IsUnderneathDirectory(file, downloadFolder));
}))
{
return(ErrorResponse.Fail($"Cannot install into a non-empty folder that does not look like a previous WJ installation.\n" +
$"To override, delete all installed files from your target installation folder. Any files in your download folder are okay to keep."));
}
}
return(ErrorResponse.Success);
}
private static void DumpPathFileExists(string path, bool doesExist)
{
Console.WriteLine("\n\tPath: [{0}]\n", path);
bool fileExists = Shell32.PathFileExists(path);
Console.WriteLine("\t\tShell32.PathFileExists() == [{0}]: {1}\t\t[{2}]", doesExist ? TextTrue : TextFalse, doesExist == fileExists, path);
Console.WriteLine("\t\tFile.Exists() == [{0}]: {1}\t\t[{2}]", doesExist ? TextTrue : TextFalse, doesExist == File.Exists(path), path);
Console.WriteLine("\t\tDirectory.Exists() == [{0}]: {1}\t\t[{2}]", doesExist ? TextTrue : TextFalse, doesExist == Directory.Exists(path), path);
if (doesExist)
{
Assert.IsTrue(fileExists);
}
if (!doesExist)
{
Assert.IsTrue(!fileExists);
}
}
