|
public static EnumerateFileSystemEntries ( string path ) : System.Collections.Generic.IEnumerable |
||
| path | string | |
| return | System.Collections.Generic.IEnumerable |
|
public IEnumerable <string> GetKeys()
{
EnsureDirectoryExists(Directory, Create);
return(IODirectory
.EnumerateFileSystemEntries(Directory, "*.*", SearchOption.AllDirectories)
.Select(ComputeKey));
}
public static void CreateTestFilesAndDirectories(TestContext context)
{
var rootDirectory = new Directory
{
Name = TestRootDirectoryName,
Files = Array.Empty <string>(),
Directories = new Directory[]
{
new Directory
{
Name = "first-dir",
Files = new string[]
{
"file1.txt",
"file2.cs",
"file3.exe",
},
Directories = new Directory[]
{
new Directory
{
Name = "sub-dir1.cs",
Files = new string[]
{
"sub-file1.pdb",
"sub-file2.pdb",
"sub-file3.pdb",
},
Directories = Array.Empty <Directory>(),
},
new Directory
{
Name = "sub-dir2",
Files = new string[]
{
"sub2-file1.pdb",
"sub2-file2.xlsx",
"sub2-file3.xlsx",
},
Directories = Array.Empty <Directory>(),
},
new Directory
{
Name = "sub-dir3.pdb",
Files = Array.Empty <string>(),
Directories = Array.Empty <Directory>(),
},
}
},
}
};
CreateDirectory(rootDirectory);
_allTestFileSystemEntries = DirectoryHelper
.EnumerateFileSystemEntries(TestRootDirectoryName, "*", SearchOption.AllDirectories)
.ToArray();
}
public bool IsDirectoryEmpty(string path)
{
try
{
return(!Directory.EnumerateFileSystemEntries(path).Any());
}
catch
{
return(false);
}
}
public Task LoadChildrenAsync()
{
if (_children != null)
{
return(Task.CompletedTask);
}
return(Task.Run(() => {
_children = IODirectory
.EnumerateFileSystemEntries(FullPath)
.Select(name => (name, GetChild(name)))
.ToDictionary(x => x.Item1, x => x.Item2);
}));
}
public DirectoryBackup(string source, string destination, string tempPath, bool allowOverwrite)
{
this.source = source;
this.destination = destination;
this.tempPath = tempPath;
this.allowOverwrite = allowOverwrite;
EnsureDirectoryExists(tempPath);
if (!allowOverwrite && Directory.Exists(destination) && Directory.EnumerateFileSystemEntries(destination).Any())
{
throw new InvalidOperationException("Directory exists and it is not empty; overwrite was not explicitly requested by user: " + destination);
}
EnsureDirectoryExists(destination);
}
private static IEnumerable <string> EnumerateFileSystemEntriesInternal(
string path,
string searchPattern,
bool recursively)
{
var localPath = Requires.NotNullOrWhiteSpace(
path,
nameof(path));
var localSearchPattern = Requires.NotNullOrWhiteSpace(
searchPattern,
nameof(searchPattern));
return(FSDir.EnumerateFileSystemEntries(
localPath,
localSearchPattern,
recursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
}
private static Task <IEnumerable <string> > EnumerateFileSystemEntriesInternalAsync(
string path,
string searchPattern,
bool recursively,
CancellationToken cancellationToken)
{
var localPath = Requires.NotNullOrWhiteSpace(
path,
nameof(path));
var localSearchPattern = Requires.NotNullOrWhiteSpace(
searchPattern,
nameof(searchPattern));
return(Work.OnThreadPoolAsync(
state => FSDir.EnumerateFileSystemEntries(
state.Path,
state.Pattern,
state.Recursively),
(Path: localPath, Pattern: localSearchPattern, Recursively: recursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly),
cancellationToken));
}
//private static readonly string VssDir = @"C:\____vssMount";
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains evtx files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription(
"File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to."); // This, --csv, or --xml required
_fluentCommandLineParser.Setup(arg => arg.JsonName)
.As("jsonf")
.WithDescription(
"File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.XmlDirectory)
.As("xml")
.WithDescription(
"Directory to save XML formatted results to."); // This, --csv, or --json required
_fluentCommandLineParser.Setup(arg => arg.XmlName)
.As("xmlf")
.WithDescription(
"File name to save XML formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff")
.SetDefault("yyyy-MM-dd HH:mm:ss.fffffff");
_fluentCommandLineParser.Setup(arg => arg.IncludeIds)
.As("inc")
.WithDescription(
"List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.ExcludeIds)
.As("exc")
.WithDescription(
"List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.StartDate)
.As("sd")
.WithDescription(
"Start date for including events (UTC). Anything OLDER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.EndDate)
.As("ed")
.WithDescription(
"End date for including events (UTC). Anything NEWER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.FullJson)
.As("fj")
.WithDescription(
"When true, export all available data when using --json. Default is FALSE.")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.PayloadAsJson)
.As("pj")
.WithDescription(
"When true, include event *payload* as json. Default is TRUE.")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.TimeDiscrepancyThreshold)
.As("tdt")
.WithDescription(
"The number of seconds to use for time discrepancy detection. Default is 1 second")
.SetDefault(1);
_fluentCommandLineParser.Setup(arg => arg.Metrics)
.As("met")
.WithDescription(
"When true, show metrics about processed event log. Default is TRUE.\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
/*_fluentCommandLineParser.Setup(arg => arg.Vss)
* .As("vss")
* .WithDescription(
* "Process all Volume Shadow Copies that exist on drive specified by -f or -d . Default is FALSE")
* .SetDefault(false);*/
_fluentCommandLineParser.Setup(arg => arg.Dedupe)
.As("dedupe")
.WithDescription(
"Deduplicate -f or -d & VSCs based on SHA-1. First file found wins. Default is TRUE\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.Sync)
.As("sync")
.WithDescription(
"If true, the latest maps from https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps are downloaded and local maps updated. Default is FALSE\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\n\nAuthor: Eric Zimmerman ([email protected])" +
"\nhttps://github.com/EricZimmerman/evtx" +
"\nLinux Port : Scott Dermott ([email protected])" +
"\nhttps://github.com/ScottDermott/evtx";
var footer =
@"Examples: EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out"" --csvf MyOutputFile.csv" +
"\r\n\t " +
@" EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out""" + "\r\n\t " +
@" EvtxECmd -f ""/Temp/Application.evtx"" --json ""/temp/jsonout""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.Sync)
{
try
{
_logger.Info(header);
UpdateFromRepo();
}
catch (Exception e)
{
_logger.Error(e, $"There was an error checking for updates: {e.Message}");
}
Environment.Exit(0);
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
/* if (IsAdministrator() == false)
* {
* _logger.Fatal("Warning: Administrator privileges not found!\r\n");
* }*/
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
/* if (_fluentCommandLineParser.Object.Vss & (IsAdministrator() == false))
* {
* _logger.Error("--vss is present, but administrator rights not found. Exiting\r\n");
* return;
* }*/
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName);
_logger.Warn($"json output will be saved to '{outFile}'\r\n");
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName);
_logger.Warn($"XML output will be saved to '{outFile}'\r\n");
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.StartDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.StartDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_startDate = dt;
_logger.Info($"Setting Start date to '{_startDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.StartDate}' to a valud datetime! Events will not be filtered by Start date!");
}
}
if (_fluentCommandLineParser.Object.EndDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.EndDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_endDate = dt;
_logger.Info($"Setting End date to '{_endDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.EndDate}' to a valud datetime! Events will not be filtered by End date!");
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
_logger.Info("");
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_logger.Warn($"CSV output will be saved to '{outFile}'\r\n");
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
_csvWriter = new CsvWriter(_swCsv, CultureInfo.InvariantCulture);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
var foo = _csvWriter.Configuration.AutoMap <EventRecord>();
if (_fluentCommandLineParser.Object.PayloadAsJson == false)
{
foo.Map(t => t.Payload).Ignore();
}
else
{
foo.Map(t => t.Payload).Index(22);
}
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).ConvertUsing(t =>
$"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.SourceFile).Index(21);
_csvWriter.Configuration.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!");
}
else
{
_logger.Info($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}");
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false)
{
var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* string driveLetter;
* if (_fluentCommandLineParser.Object.File.IsEmpty() == false)
* {
* driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File))
* .Substring(0, 1);
* }
* else
* {
* driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory))
* .Substring(0, 1);
* }
*
*
* Helper.MountVss(driveLetter,VssDir);
* Console.WriteLine();
* }*/
EventLog.TimeDiscrepancyThreshold = _fluentCommandLineParser.Object.TimeDiscrepancyThreshold;
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
_fluentCommandLineParser.Object.Dedupe = false;
ProcessFile(Path.GetFullPath(_fluentCommandLineParser.Object.File));
/*if (_fluentCommandLineParser.Object.Vss)
* {
* var vssDirs = Directory.GetDirectories(VssDir);
*
* var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File));
* var stem = Path.GetFullPath(_fluentCommandLineParser.Object.File).Replace(root, "");
*
* foreach (var vssDir in vssDirs)
* {
* var newPath = Path.Combine(vssDir, stem);
* if (File.Exists(newPath))
* {
* ProcessFile(newPath);
* }
* }
* }*/
}
else
{
_logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
/* var f = new DirectoryEnumerationFilters
* {
* InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
* RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
* ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
* };*/
/* var dirEnumOptions =
* DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
* DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
* DirectoryEnumerationOptions.BasicSearch;*/
var files2 = Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), "*.evtx");
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file);
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* var vssDirs = Directory.GetDirectories(VssDir);
*
* Console.WriteLine();
*
* foreach (var vssDir in vssDirs)
* {
* var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory));
* var stem = Path.GetFullPath(_fluentCommandLineParser.Object.Directory).Replace(root, "");
*
* var target = Path.Combine(vssDir, stem);
*
* _logger.Fatal($"\r\nSearching 'VSS{target.Replace($"{VssDir}\\","")}' for event logs...");
*
* var vssFiles = Helper.GetFilesFromPath(target, true, "*.evtx");
*
* foreach (var file in vssFiles)
* {
* ProcessFile(file);
* }
* }
*
* }*/
}
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
sw.Stop();
_logger.Info("");
var suff = string.Empty;
if (_fileCount != 1)
{
suff = "s";
}
_logger.Error(
$"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (_errorFiles.Count > 0)
{
_logger.Info("");
_logger.Error("Files with errors");
foreach (var errorFile in _errorFiles)
{
_logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}");
}
_logger.Info("");
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* if (Directory.Exists(VssDir))
* {
* foreach (var directory in Directory.GetDirectories(VssDir))
* {
* Directory.Delete(directory);
* }
* Directory.Delete(VssDir,true);
* }
* }*/
}
private static bool IsDirectoryEmpty(string path)
{
return(!Directory.EnumerateFileSystemEntries(path).Any());
}
public static bool LoadMaps(string mapPath)
{
EventLogMaps = new Dictionary <string, EventLogMap>();
/* var f = new DirectoryEnumerationFilters();
* f.InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".MAP";
*
* f.RecursionFilter = null; //entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink;
*
* f.ErrorFilter = (errorCode, errorMessage, pathProcessed) => true;*/
/* var dirEnumOptions =
* DirectoryEnumerationOptions.Files |
* DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
* DirectoryEnumerationOptions.BasicSearch;*/
var mapFiles = Directory.EnumerateFileSystemEntries(mapPath, "*.map").ToList();
var l = LogManager.GetLogger("LoadMaps");
var deserializer = new DeserializerBuilder()
.Build();
var validator = new EventLogMapValidator();
var errorMaps = new List <string>();
foreach (var mapFile in mapFiles.OrderBy(t => t))
{
try
{
var eventMapFile = deserializer.Deserialize <EventLogMap>(File.ReadAllText(mapFile));
l.Trace(eventMapFile.Dump);
var validate = validator.Validate(eventMapFile);
if (DisplayValidationResults(validate, mapFile))
{
if (EventLogMaps.ContainsKey(
$"{eventMapFile.EventId}-{eventMapFile.Channel.ToUpperInvariant()}") == false)
{
l.Debug($"'{Path.GetFileName(mapFile)}' is valid. Adding to maps...");
EventLogMaps.Add($"{eventMapFile.EventId}-{eventMapFile.Channel.ToUpperInvariant()}",
eventMapFile);
}
else
{
l.Warn(
$"A map for event id '{eventMapFile.EventId}' with Channel '{eventMapFile.Channel}' already exists. Map '{Path.GetFileName(mapFile)}' will be skipped");
}
}
else
{
errorMaps.Add(Path.GetFileName(mapFile));
}
}
catch (SyntaxErrorException se)
{
errorMaps.Add(Path.GetFileName(mapFile));
Console.WriteLine();
l.Warn($"Syntax error in '{mapFile}':");
l.Fatal(se.Message);
var lines = File.ReadLines(mapFile).ToList();
var fileContents = mapFile.ReadAllText();
var badLine = lines[se.Start.Line - 1];
Console.WriteLine();
l.Fatal(
$"Bad line (or close to it) '{badLine}' has invalid data at column '{se.Start.Column}'");
if (fileContents.Contains('\t'))
{
Console.WriteLine();
l.Error(
"Bad line contains one or more tab characters. Replace them with spaces");
Console.WriteLine();
l.Info(fileContents.Replace("\t", "<TAB>"));
}
}
catch (YamlException ye)
{
errorMaps.Add(Path.GetFileName(mapFile));
Console.WriteLine();
l.Warn($"Syntax error in '{mapFile}':");
var fileContents = mapFile.ReadAllText();
l.Info(fileContents);
if (ye.InnerException != null)
{
l.Fatal(ye.InnerException.Message);
}
Console.WriteLine();
l.Fatal("Verify all properties against example files or manual and try again.");
}
catch (Exception e)
{
l.Error($"Error loading map file '{mapFile}': {e.Message}");
}
}
if (errorMaps.Count > 0)
{
l.Error("\r\nThe following maps had errors. Scroll up to review errors, correct them, and try again.");
foreach (var errorMap in errorMaps)
{
l.Info(errorMap);
}
l.Info("");
}
return(errorMaps.Count > 0);
}
/// <summary>
/// This method will construct a three folder structure inside <paramref name="targetDirectory"/> containing: Html, Index, and Source
/// </summary>
/// <param name="sourceDirectory">
/// Directory containing ldoc files
/// </param>
/// <param name="targetDirectory">
/// Output directory
/// </param>
public void Build(string sourceDirectory, string targetDirectory)
{
if (Directory.Exists(targetDirectory) && Directory.EnumerateFileSystemEntries(targetDirectory).Any())
{
throw new InvalidOperationException("Target path is not empty.");
}
this.OnStateChanged(State.Preparing);
string htmlRoot = Path.Combine(targetDirectory, "Html");
string indexRoot = Path.Combine(targetDirectory, "Index");
string sourceRoot = Path.Combine(targetDirectory, "Source");
string logRoot = Path.Combine(targetDirectory, "Logs");
DirectoryInfo htmlDir = Directory.CreateDirectory(htmlRoot);
DirectoryInfo indexDir = Directory.CreateDirectory(indexRoot);
DirectoryInfo sourceDir = Directory.CreateDirectory(sourceRoot);
DirectoryInfo logDir = Directory.CreateDirectory(logRoot);
var sourceFiles = Directory.EnumerateFiles(sourceDirectory, "*.ldoc", SearchOption.TopDirectoryOnly);
// copy all source files to output directory and add to bundle
Bundle bundle = new Bundle(this.IgnoreVersionComponent);
foreach (var sourceFile in sourceFiles)
{
string targetFile = Path.Combine(sourceDir.FullName, Path.GetFileName(sourceFile));
File.Copy(sourceFile, targetFile);
bundle.Add(XDocument.Load(targetFile));
}
TemplateOutput templateOutput;
// wire up logging
string templateLogFile = Path.Combine(logDir.FullName,
string.Format("template_{0:yyyy'_'MM'_'dd'__'HH'_'mm'_'ss}.log", DateTime.Now));
using (TextWriterTraceListener traceListener = new TextWriterTraceListener(templateLogFile))
{
// log everything
traceListener.Filter = new EventTypeFilter(SourceLevels.All);
LostDoc.Diagnostics.TraceSources.TemplateSource.Switch.Level = SourceLevels.All;
LostDoc.Diagnostics.TraceSources.BundleSource.Switch.Level = SourceLevels.All;
LostDoc.Diagnostics.TraceSources.AssetResolverSource.Switch.Level = SourceLevels.All;
LostDoc.Diagnostics.TraceSources.TemplateSource.Listeners.Add(traceListener);
LostDoc.Diagnostics.TraceSources.BundleSource.Listeners.Add(traceListener);
LostDoc.Diagnostics.TraceSources.AssetResolverSource.Listeners.Add(traceListener);
// merge ldoc files
this.OnStateChanged(State.Merging);
AssetRedirectCollection assetRedirects;
var mergedDoc = bundle.Merge(out assetRedirects);
// generate output
var templateData = new TemplateData(mergedDoc)
{
AssetRedirects = assetRedirects,
IgnoredVersionComponent = this.IgnoreVersionComponent,
OutputFileProvider =
new ScopedFileProvider(new DirectoryFileProvider(), htmlDir.FullName),
//TargetDirectory = htmlDir.FullName,
Arguments = new Dictionary <string, object> {
{ "SearchUri", "/search/" }
},
KeepTemporaryFiles = true,
TemporaryFilesPath = Path.Combine(logDir.FullName, "temp")
};
this.OnStateChanged(State.Templating);
templateOutput = this.Template.Generate(templateData);
LostDoc.Diagnostics.TraceSources.TemplateSource.Listeners.Remove(traceListener);
LostDoc.Diagnostics.TraceSources.BundleSource.Listeners.Remove(traceListener);
LostDoc.Diagnostics.TraceSources.AssetResolverSource.Listeners.Remove(traceListener);
}
this.OnStateChanged(State.Indexing);
string indexLogFile = Path.Combine(logDir.FullName,
string.Format("index_{0:yyyy'_'MM'_'dd'__'HH'_'mm'_'ss}.log", DateTime.Now));
using (TextWriterTraceListener traceListener = new TextWriterTraceListener(indexLogFile))
{
// log everything
traceListener.Filter = new EventTypeFilter(SourceLevels.All);
TraceSources.ContentBuilderSource.Switch.Level = SourceLevels.All;
TraceSources.ContentBuilderSource.Listeners.Add(traceListener);
// one stop-word per line
StringReader stopWordsReader = new StringReader(@"missing");
// index output
using (var directory = FSDirectory.Open(indexDir))
using (stopWordsReader)
{
Analyzer analyzer = new StandardAnalyzer(global::Lucene.Net.Util.Version.LUCENE_30, stopWordsReader);
Analyzer titleAnalyzer = new TitleAnalyzer();
IDictionary <string, Analyzer> fieldAnalyzers = new Dictionary <string, Analyzer>
{
{ "title", titleAnalyzer }
};
PerFieldAnalyzerWrapper analyzerWrapper = new PerFieldAnalyzerWrapper(analyzer, fieldAnalyzers);
using (
var writer = new IndexWriter(directory, analyzerWrapper, IndexWriter.MaxFieldLength.UNLIMITED))
{
var saResults =
templateOutput.Results.Select(wur => wur.WorkUnit).OfType <StylesheetApplication>();
var saDict = saResults.ToDictionary(sa => sa.Asset);
var indexResults = saDict.Values.Where(sa => sa.SaveAs.EndsWith(".xml"));
foreach (var sa in indexResults)
{
string absPath = Path.Combine(htmlDir.FullName, sa.SaveAs);
XDocument indexDoc = XDocument.Load(absPath);
string assetId = indexDoc.Root.Attribute("assetId").Value;
string title = indexDoc.Root.Element("title").Value.Trim();
string summary = indexDoc.Root.Element("summary").Value.Trim();
string text = indexDoc.Root.Element("text").Value.Trim();
var ssApplication = saDict[AssetIdentifier.Parse(assetId)];
var doc = new Document();
doc.Add(new Field("uri",
new Uri(ssApplication.SaveAs, UriKind.Relative).ToString(),
Field.Store.YES,
Field.Index.NO));
doc.Add(new Field("aid", ssApplication.Asset, Field.Store.YES, Field.Index.NOT_ANALYZED));
foreach (AssetIdentifier aid in ssApplication.Aliases)
{
doc.Add(new Field("alias", aid, Field.Store.NO, Field.Index.NOT_ANALYZED));
}
foreach (var section in ssApplication.Sections)
{
doc.Add(new Field("section",
section.AssetIdentifier,
Field.Store.NO,
Field.Index.NOT_ANALYZED));
}
doc.Add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("summary", summary, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("content", text, Field.Store.NO, Field.Index.ANALYZED));
TraceSources.ContentBuilderSource.TraceVerbose("Indexing document: {0}", doc.ToString());
writer.AddDocument(doc);
}
writer.Optimize();
writer.Commit();
}
analyzerWrapper.Close();
analyzer.Close();
}
TraceSources.ContentBuilderSource.Listeners.Remove(traceListener);
}
this.OnStateChanged(State.Finalizing);
var infoDoc = new XDocument(
new XElement("content",
new XAttribute("created",
XmlConvert.ToString(DateTime.UtcNow, XmlDateTimeSerializationMode.Utc)),
templateOutput.Results.Select(this.ConvertToXml)));
infoDoc.Save(Path.Combine(targetDirectory, "info.xml"));
this.OnStateChanged(State.Idle);
}
private static void RemoveDirectoryInternal(DirectoryInfo directory, bool recursive, bool throwOnTopLevelDirectoryNotFound)
{
Exception firstException = null;
if ((directory.Attributes & FileAttributes.ReparsePoint) != 0)
{
DeleteFile(directory.FullName);
return;
}
if (recursive)
{
try
{
foreach (string item in Directory.EnumerateFileSystemEntries(directory.FullName))
{
if (!ShouldIgnoreDirectory(Path.GetFileName(item)))
{
try
{
var childDirectory = new DirectoryInfo(item);
if (childDirectory.Exists)
{
RemoveDirectoryInternal(childDirectory, recursive, throwOnTopLevelDirectoryNotFound: false);
}
else
{
DeleteFile(item);
}
}
catch (Exception exc)
{
if (firstException != null)
{
firstException = exc;
}
}
}
}
}
catch (Exception exc)
{
if (firstException != null)
{
firstException = exc;
}
}
if (firstException != null)
{
throw firstException;
}
}
if (Interop.Sys.RmDir(directory.FullName) < 0)
{
Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo();
switch (errorInfo.Error)
{
case Interop.Error.EACCES:
case Interop.Error.EPERM:
case Interop.Error.EROFS:
case Interop.Error.EISDIR:
throw new IOException(SR.Format(SR.UnauthorizedAccess_IODenied_Path, directory.FullName)); // match Win32 exception
case Interop.Error.ENOENT:
if (!throwOnTopLevelDirectoryNotFound)
{
return;
}
goto default;
default:
throw Interop.GetExceptionForIoErrno(errorInfo, directory.FullName, isDirectory: true);
}
}
}
public IEnumerable <FileSystemPath> EnumerateFileSystemEntries(string searchPattern, SearchOption searchOption) => Directory.EnumerateFileSystemEntries(Internal, searchPattern, searchOption).Select(_ => _.ToFileSystemPath());
public IEnumerable <FileSystemPath> EnumerateFileSystemEntries() => Directory.EnumerateFileSystemEntries(Internal).Select(_ => _.ToFileSystemPath());