/// <summary>
/// Exports all customers for the reseller to the specified file.
/// </summary>
/// <param name="defaultDomain">default domain of the reseller</param>
/// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param>
/// <param name="key">Key for this application in Azure Active Directory</param>
/// <param name="customerMicrosoftId">Microsoft Id of the customer</param>
/// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param>
public static void ExportCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId)
{
GetTokens(defaultDomain, appId, key, resellerMicrosoftId);
var adCustomers = Customer.GetAllCustomers(resellerMicrosoftId, adAuthorizationToken.AccessToken);
using (StreamWriter outputFile = new StreamWriter(filename))
using (var writer = new CsvHelper.CsvWriter(outputFile))
{
writer.WriteField("displayName");
writer.WriteField("defaultDomainName");
writer.WriteField("cspCustomerCid");
writer.WriteField("customerContextId");
writer.NextRecord();
foreach (AzureADContract adCustomer in adCustomers)
{
adCustomer.cspCustomerCid = Customer.GetCustomerCid(adCustomer.customerContextId.ToString(), resellerMicrosoftId,
saAuthorizationToken.AccessToken);
writer.WriteField(adCustomer.displayName);
writer.WriteField(adCustomer.defaultDomainName);
writer.WriteField(adCustomer.cspCustomerCid);
writer.WriteField(adCustomer.customerContextId);
writer.NextRecord();
}
}
}
public void Write(IEnumerable<CsvModel> models, TextWriter writer, string delimiter)
{
/* get all headers row */
List<string> headers = new List<string>();
headers.Add(CsvHeaders.ObjectType);
headers.Add(CsvHeaders.Class);
headers.Add(CsvHeaders.Source);
foreach (var model in models)
{
foreach (string key in model.Values.Keys)
if (!headers.Contains(key))
headers.Add(key);
}
using (var csvWriter = new CsvHelper.CsvWriter(writer,
new CsvHelper.Configuration.CsvConfiguration() {
QuoteAllFields = true,
Delimiter = delimiter }
))
{
/* write the header row */
foreach (string header in headers)
csvWriter.WriteField<string>(header);
csvWriter.NextRecord();
/* write the content of each csv model */
foreach (var model in models)
{
for (int i = 0; i < headers.Count; i++)
{
string value = string.Empty;
switch (i)
{
case 0: value = model.ObjectType; break;
case 1: value = model.Class; break;
case 2: value = string.Join(new string (new char[] {Constants.SourceFileDelimiter }), model.SourceFiles); break;
default:
if (!model.Values.TryGetValue(headers[i], out value))
value = string.Empty;
break;
}
csvWriter.WriteField<string>(value);
}
csvWriter.NextRecord();
}
}
}
internal string createCsvWithStructure(IEnumerable<dynamic> data, CsvConfiguration configuration)
{
var guid = Guid.NewGuid().ToString();
var builder = new StringBuilder();
var writer = new CsvHelper.CsvWriter(new StringWriter(builder), configuration);
foreach (var column in Structure)
writer.WriteField(column.Name);
writer.NextRecord();
foreach (var datum in data)
{
var dict = ToDictionary(guid, datum);
foreach (var column in Structure)
{
var value = dict[column.Name];
var str = string.IsNullOrEmpty(column.Format) ? value.ToString()
: value.ToString(column.Format);
writer.WriteField(str);
}
writer.NextRecord();
}
return builder.ToString();
}
protected virtual void WriteHeader(ICsvReportConfiguration config, CsvHelper.CsvWriter writer)
{
if (config.Options.IsHideHeader)
{
return;
}
var memberOptions = config.MemberConfigurations.GetActiveOrderlyOptions();
foreach (var options in memberOptions)
{
var header = options.GetHeader(config.Options.UseHeaderHumanizer);
writer.WriteField(header);
}
writer.NextRecord();
}
public void WriteComment(params string[] comments)
{
if (comments == null || comments.Length == 0)
{
return;
}
var configuration = GetConfiguration();
using (var writer = new CsvHelper.CsvWriter(TextWriter, configuration))
{
foreach (var comment in comments)
{
writer.WriteComment(comment);
writer.NextRecord();
}
}
}
public JsonResult _SubmiteBayEndProductListing(MarketplaceFeed model)
{
try
{
var productItems = _productService.GeteBayItemFeeds(model).ToList();
var fileFullPath = string.Format("{0}\\eBayProductEndListing_{1:yyyyMMdd_HHmmss}.csv",
_systemJobsRoot,
DateTime.Now);
using (var streamWriter = new StreamWriter(fileFullPath))
{
var writer = new CsvHelper.CsvWriter(streamWriter);
foreach (var item in productItems)
{
writer.WriteField(item.EisSKU);
writer.WriteField(item.ItemId);
writer.NextRecord();
}
}
// create new job for eBay suggested categories
var systemJob = new SystemJobDto
{
JobType = JobType.eBayProductsEndItem,
Parameters = fileFullPath,
SupportiveParameters = model.Mode,
SubmittedBy = User.Identity.Name
};
var jobId = _systemJobService.CreateSystemJob(systemJob);
return(Json(new
{
Success = "eBay EndItem has been passed to EIS System Jobs for execution.",
JobId = jobId
},
JsonRequestBehavior.AllowGet));
}
catch (Exception ex)
{
return(Json(new { Error = "Error in submitting eBay EndItem! - Message: " + EisHelper.GetExceptionMessage(ex) },
JsonRequestBehavior.AllowGet));
}
}
private void WriteHeader(CsvHelper.CsvWriter csv)
{
if (!string.IsNullOrEmpty(CustomIdColumn))
{
csv.WriteField("Custom Id");
}
csv.WriteField("Id");
csv.WriteField("Source");
csv.WriteField("Target");
csv.WriteField("State");
if (IsLangColumnRequired)
{
csv.WriteField("Language");
}
csv.NextRecord();
}
public HttpResponseMessage DownloadCsv(GenerateCsvModel model)
{
// Variables.
var start = model.StartDate.HasValue
? model.StartDate.Value
: SqlDateTime.MinValue.Value;
var stop = model.EndDate.HasValue
? model.EndDate.Value.AddDays(1)
: SqlDateTime.MaxValue.Value;
// Get history items.
var items = HistoryHelper.GetHistoryItems(start, stop);
var config = new CsvHelper.Configuration.CsvConfiguration()
{
HasHeaderRecord = true,
QuoteAllFields = true
};
// write CSV data to memory stream
using (var stream = new MemoryStream()) {
using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true)) {
using (var csvWriter = new CsvHelper.CsvWriter(writer, config)) {
csvWriter.WriteHeader <HistoryCsvHeader>();
foreach (var item in items)
{
csvWriter.WriteField(item.Timestamp.ToString());
csvWriter.WriteField(item.Message);
csvWriter.NextRecord();
}
}
}
stream.Seek(0, SeekOrigin.Begin);
return(CreateFileResponse(
stream.ToArray(),
$"site-activity_{start.ToString("yyyy-MM-dd")}_{stop.ToString("yyyy-MM-dd")}.csv",
"text/csv"
));
}
}
public JsonResult _GeteBayBulkSuggestedCategories(MarketplaceFeed model)
{
try
{
// get the product feed for bul eBay suggested categories
var productKeywordFeeds = _productService.GeteBaySuggestedCategoryFeed(model);
var fileFullPath = string.Format("{0}\\BulkeBaySuggestedCategories_{1:yyyyMMdd_HHmmss}.csv",
_systemJobsRoot,
DateTime.Now);
using (var streamWriter = new StreamWriter(fileFullPath))
{
var writer = new CsvHelper.CsvWriter(streamWriter);
foreach (var item in productKeywordFeeds)
{
writer.WriteField(item.EisSKU);
writer.WriteField(item.Keyword);
writer.NextRecord();
}
}
// create new job for eBay suggested categories
var systemJob = new SystemJobDto
{
JobType = JobType.BulkeBaySuggestedCategories,
Parameters = fileFullPath,
SubmittedBy = User.Identity.Name
};
var jobId = _systemJobService.CreateSystemJob(systemJob);
return(Json(new { Success = "Bulk eBay get suggested categories has been passed to EIS System Jobs for execution.",
JobId = jobId },
JsonRequestBehavior.AllowGet));
}
catch (Exception ex)
{
return(Json(new { Error = "Error in getting bulk eBay suggested categories! - Message: " + EisHelper.GetExceptionMessage(ex) },
JsonRequestBehavior.AllowGet));
}
}
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer)
{
var szTrCode = resModel.Name;
var block = resModel.Blocks[szTrCode + "OutBlock"];
writer.WriteHeader <_t8425OutBlock>();
for (var i = 0; i < query.GetBlockCount(block.Name); i++)
{
var result = new _t8425OutBlock()
{
tmcode = query.GetFieldData(block.Name, "tmcode", i),
tmname = query.GetFieldData(block.Name, "tmname", i),
};
writer.NextRecord();
writer.WriteRecord(result);
Constants.CodeThemes.Add(result.tmcode, new CodeTheme()
{
Code = result.tmcode,
Name = result.tmname,
});
}
}
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer)
{
var szTrCode = resModel.Name;
var block = resModel.Blocks[szTrCode + "OutBlock"];
writer.WriteHeader <_t8436OutBlock>();
for (var i = 0; i < query.GetBlockCount(block.Name); i++)
{
var result = new _t8436OutBlock()
{
hname = query.GetFieldData(block.Name, "hname", i),
shcode = query.GetFieldData(block.Name, "shcode", i),
expcode = query.GetFieldData(block.Name, "expcode", i),
etfgubun = query.GetFieldData(block.Name, "etfgubun", i),
uplmtprice = long.Parse(query.GetFieldData(block.Name, "uplmtprice", i)),
dnlmtprice = long.Parse(query.GetFieldData(block.Name, "dnlmtprice", i)),
jnilclose = long.Parse(query.GetFieldData(block.Name, "jnilclose", i)),
memedan = query.GetFieldData(block.Name, "memedan", i),
recprice = long.Parse(query.GetFieldData(block.Name, "recprice", i)),
gubun = query.GetFieldData(block.Name, "gubun", i),
bu12gubun = query.GetFieldData(block.Name, "bu12gubun", i),
spac_gubun = query.GetFieldData(block.Name, "spac_gubun", i),
filler = query.GetFieldData(block.Name, "filler", i),
};
writer.NextRecord();
writer.WriteRecord(result);
Constants.CodeStocks.Add(result.shcode, new CodeStock()
{
Name = result.hname,
Code = result.shcode,
ExpandedCode = result.expcode,
EtfGubun = result.etfgubun,
SpacGubun = result.spac_gubun,
MarketGubun = result.gubun,
Bu12Gubun = result.bu12gubun,
});
}
}
/// <summary>
/// execute the <see cref="ExportCommand"/>
/// </summary>
private void ExecuteExportCommand()
{
var openSaveFileDialogService = ServiceLocator.Current.GetInstance <IOpenSaveFileDialogService>();
var result = openSaveFileDialogService.GetSaveFileDialog("Untitled", ".csv", "CSV File (.csv)|*.csv", "", 0);
if (string.IsNullOrEmpty(result))
{
return;
}
using (TextWriter writer = File.CreateText(result))
{
var csv = new CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture);
foreach (var logInfoRowViewModel in this.LogEventInfo)
{
csv.WriteField(logInfoRowViewModel.TimeStamp);
csv.WriteField(logInfoRowViewModel.Logger);
csv.WriteField(logInfoRowViewModel.LogLevel.Name);
csv.WriteField(logInfoRowViewModel.Message);
csv.NextRecord();
}
}
}
protected virtual void WriteRecords(IEnumerable source, ICsvReportConfiguration config, TypeAccessor.TypeAccessor accessor, CsvHelper.CsvWriter writer)
{
foreach (var row in source)
{
var memberOptions = config.MemberConfigurations.GetActiveOrderlyOptions();
foreach (var column in memberOptions)
{
var value = accessor.GetValue(row, column.MemberName);
string formattedValue = value.ToString();
if (column.Formatter != null)
{
formattedValue = column.Formatter.Format(value);
}
else if (column.FormatterExpression != null)
{
formattedValue = ReportHelper.ReportHelper.CreateFormatFunc(value, column.FormatterExpression)(value);
}
writer.WriteField(formattedValue);
}
writer.NextRecord();
}
}
public JsonResult _DeleteBulkVendorProducts(VendorProductFilterDto model)
{
try
{
// get product's EIS SKU
var eisSupplierSKUs = _vendorProductService.GetVendorProductsEisSupplierSKUs(model);
var fileFullPath = string.Format("{0}\\BulkDeleteVendorProducts_{1:yyyyMMdd_HHmmss}.txt", _systemJobsRoot, DateTime.Now);
using (var streamWriter = new StreamWriter(fileFullPath))
{
var writer = new CsvHelper.CsvWriter(streamWriter);
foreach (var sku in eisSupplierSKUs)
{
// just write the vendor product's Id which we need to delete
writer.WriteField(sku);
writer.NextRecord();
}
}
// create new job for the Amazon Get Info for the asins
var systemJob = new SystemJobDto
{
JobType = JobType.BulkDeleteVendorProduct,
Parameters = fileFullPath,
SubmittedBy = User.Identity.Name
};
_systemJobService.CreateSystemJob(systemJob);
return(Json("Bulk deletion of vendor products has been passed to EIS System Jobs for execution.", JsonRequestBehavior.AllowGet));
}
catch (Exception ex)
{
return(Json(new { Error = "Error in deleting vendor products! - Message: " + EisHelper.GetExceptionMessage(ex) },
JsonRequestBehavior.AllowGet));
}
}
public static void ExportToCsv(List <List <string> > data)
{
var pathToFile = GetPathToFile();
using (var stream = File.OpenWrite(pathToFile))
{
using (var streamWriter = new StreamWriter(stream, Encoding.UTF8))
{
using (var csvWriter = new CsvHelper.CsvWriter(streamWriter, CultureInfo.InvariantCulture))
{
foreach (var row in data)
{
foreach (var i in row)
{
csvWriter.WriteField(i);
}
csvWriter.NextRecord();
}
}
}
}
Process.Start(pathToFile);
}
private static void ProcessFile(string file)
{
if (File.Exists(file) == false)
{
_logger.Warn($"'{file}' does not exist! Skipping");
return;
}
if (file.StartsWith(VssDir))
{
_logger.Warn($"\r\nProcessing 'VSS{file.Replace($"{VssDir}\\", "")}'");
}
else
{
_logger.Warn($"\r\nProcessing '{file}'...");
}
Stream fileS;
try
{
fileS = new FileStream(file, FileMode.Open, FileAccess.Read);
}
catch (Exception)
{
//file is in use
if (Helper.IsAdministrator() == false)
{
_logger.Fatal("\r\nAdministrator privileges not found! Exiting!!\r\n");
Environment.Exit(0);
}
_logger.Warn($"\r\n'{file}' is in use. Rerouting...");
var files = new List <string>();
files.Add(file);
var rawFiles = Helper.GetFiles(files);
fileS = rawFiles.First().FileStream;
}
try
{
if (_fluentCommandLineParser.Object.Dedupe)
{
var sha = Helper.GetSha1FromStream(fileS, 0);
if (_seenHashes.Contains(sha))
{
_logger.Debug($"Skipping '{file}' as a file with SHA-1 '{sha}' has already been processed");
return;
}
_seenHashes.Add(sha);
}
var evt = new EventLog(fileS);
var seenRecords = 0;
foreach (var eventRecord in evt.GetEventRecords())
{
if (_includeIds.Count > 0)
{
if (_includeIds.Contains(eventRecord.EventId) == false)
{
//it is NOT in the list, so skip
_droppedEvents += 1;
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains(eventRecord.EventId))
{
//it IS in the list, so skip
_droppedEvents += 1;
continue;
}
}
if (_startDate.HasValue)
{
if (eventRecord.TimeCreated < _startDate.Value)
{
//too old
_logger.Debug($"Dropping record Id '{eventRecord.EventRecordId}' with timestamp '{eventRecord.TimeCreated.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}' as its too old.");
_droppedEvents += 1;
continue;
}
}
if (_endDate.HasValue)
{
if (eventRecord.TimeCreated > _endDate.Value)
{
//too new
_logger.Debug($"Dropping record Id '{eventRecord.EventRecordId}' with timestamp '{eventRecord.TimeCreated.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}' as its too new.");
_droppedEvents += 1;
continue;
}
}
if (file.StartsWith(VssDir))
{
eventRecord.SourceFile = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
else
{
eventRecord.SourceFile = file;
}
try
{
if (_fluentCommandLineParser.Object.PayloadAsJson)
{
var xdo = new XmlDocument();
xdo.LoadXml(eventRecord.Payload);
var payOut = JsonConvert.SerializeXmlNode(xdo);
eventRecord.Payload = payOut;
}
_csvWriter?.WriteRecord(eventRecord);
_csvWriter?.NextRecord();
var xml = string.Empty;
if (_swXml != null)
{
xml = eventRecord.ConvertPayloadToXml();
_swXml.WriteLine(xml);
}
if (_swJson != null)
{
var jsOut = eventRecord.ToJson();
if (_fluentCommandLineParser.Object.FullJson)
{
if (xml.IsNullOrEmpty())
{
xml = eventRecord.ConvertPayloadToXml();
}
jsOut = GetPayloadAsJson(xml);
}
_swJson.WriteLine(jsOut);
}
seenRecords += 1;
}
catch (Exception e)
{
_logger.Error($"Error processing record #{eventRecord.RecordNumber}: {e.Message}");
evt.ErrorRecords.Add(21, e.Message);
}
}
if (evt.ErrorRecords.Count > 0)
{
var fn = file;
if (file.StartsWith(VssDir))
{
fn = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
_errorFiles.Add(fn, evt.ErrorRecords.Count);
}
_fileCount += 1;
_logger.Info("");
_logger.Fatal("Event log details");
_logger.Info(evt);
_logger.Info($"Records included: {seenRecords:N0} Errors: {evt.ErrorRecords.Count:N0} Events dropped: {_droppedEvents:N0}");
if (evt.ErrorRecords.Count > 0)
{
_logger.Warn("\r\nErrors");
foreach (var evtErrorRecord in evt.ErrorRecords)
{
_logger.Info($"Record #{evtErrorRecord.Key}: Error: {evtErrorRecord.Value}");
}
}
if (_fluentCommandLineParser.Object.Metrics && evt.EventIdMetrics.Count > 0)
{
_logger.Fatal("\r\nMetrics (including dropped events)");
_logger.Warn("Event Id\tCount");
foreach (var esEventIdMetric in evt.EventIdMetrics.OrderBy(t => t.Key))
{
if (_includeIds.Count > 0)
{
if (_includeIds.Contains((int)esEventIdMetric.Key) == false)
{
//it is NOT in the list, so skip
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains((int)esEventIdMetric.Key))
{
//it IS in the list, so skip
continue;
}
}
_logger.Info($"{esEventIdMetric.Key}\t\t{esEventIdMetric.Value:N0}");
}
}
}
catch (Exception e)
{
var fn = file;
if (file.StartsWith(VssDir))
{
fn = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
if (e.Message.Contains("Invalid signature! Expected 'ElfFile"))
{
_logger.Info($"'{fn}' is not an evtx file! Message: {e.Message} Skipping...");
}
else
{
_logger.Error($"Error processing '{fn}'! Message: {e.Message}");
}
}
fileS?.Close();
}
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer)
{
var szTrCode = resModel.Name;
if (query.GetBlockCount(szTrCode + "OutBlock1") == 0)
{
_codeIdx++;
var code = codes.ElementAt(_codeIdx);
InBlock(code);
return;
}
var block = resModel.Blocks[szTrCode + "OutBlock"];
var meta = new _t4203OutBlock()
{
shcode = query.GetFieldData(block.Name, "shcode", 0),
jisiga = decimal.Parse(query.GetFieldData(block.Name, "jisiga", 0)),
jihigh = decimal.Parse(query.GetFieldData(block.Name, "jihigh", 0)),
jilow = decimal.Parse(query.GetFieldData(block.Name, "jilow", 0)),
jiclose = decimal.Parse(query.GetFieldData(block.Name, "jiclose", 0)),
jivolume = long.Parse(query.GetFieldData(block.Name, "jivolume", 0)),
disiga = decimal.Parse(query.GetFieldData(block.Name, "disiga", 0)),
dihigh = decimal.Parse(query.GetFieldData(block.Name, "dihigh", 0)),
dilow = decimal.Parse(query.GetFieldData(block.Name, "dilow", 0)),
diclose = decimal.Parse(query.GetFieldData(block.Name, "diclose", 0)),
disvalue = long.Parse(query.GetFieldData(block.Name, "disvalue", 0)),
cts_date = query.GetFieldData(block.Name, "cts_date", 0),
cts_time = query.GetFieldData(block.Name, "cts_time", 0),
cts_daygb = query.GetFieldData(block.Name, "cts_daygb", 0),
};
var block1 = resModel.Blocks[szTrCode + "OutBlock1"];
writer.WriteHeader <_t4203OutBlock1>();
for (var i = 0; i < query.GetBlockCount(block1.Name); i++)
{
var result = new _t4203OutBlock1()
{
date = query.GetFieldData(block1.Name, "date", i),
time = query.GetFieldData(block1.Name, "time", i),
open = decimal.Parse(query.GetFieldData(block1.Name, "open", i)),
high = decimal.Parse(query.GetFieldData(block1.Name, "high", i)),
low = decimal.Parse(query.GetFieldData(block1.Name, "low", i)),
close = decimal.Parse(query.GetFieldData(block1.Name, "close", i)),
jdiff_vol = long.Parse(query.GetFieldData(block1.Name, "jdiff_vol", i)),
value = long.Parse(query.GetFieldData(block1.Name, "value", i)),
};
writer.NextRecord();
writer.WriteRecord(result);
}
Thread.Sleep(1000);
if (_query.IsNext)
{
_inBlock.cts_date = meta.cts_date;
_inBlock.cts_time = meta.cts_time;
_inBlock.cts_daygb = meta.cts_daygb;
InBlock(meta.shcode, _query.IsNext);
}
else
{
_codeIdx++;
if (codes.Count > _codeIdx)
{
var code = codes.ElementAt(_codeIdx);
InBlock(code);
}
else
{
//TODO next callback hell
}
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("Wdlq68AwLteBtuqOwNv5rgphcMxzKuHKJQAVK5JN");
SetupNLog();
_logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <AppArgs>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. Required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV (tab separated) formatted results to. Be sure to include the full path in double quotes");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save json representation to. Use --pretty for a more human readable layout");
_fluentCommandLineParser.Setup(arg => arg.JsonPretty)
.As("pretty")
.WithDescription(
"When exporting to json, use a more human readable layout\r\n").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Quiet)
.As('q')
.WithDescription(
"Only show the filename being processed vs all output. Useful to speed up exporting to json and/or csv\r\n")
.SetDefault(false);
var header =
$"RecentFileCacheParser version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/RecentFileCacheParser";
var footer = @"Examples: RecentFileCacheParser.exe -f ""C:\Temp\RecentFileCache.bcf"" --csv ""c:\temp""" +
"\r\n\t " +
@" RecentFileCacheParser.exe -f ""C:\Temp\RecentFileCache.bcf"" --json ""D:\jsonOutput"" --jsonpretty" +
"\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f is required. Exiting");
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false &&
!File.Exists(_fluentCommandLineParser.Object.File))
{
_logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
try
{
if (_fluentCommandLineParser.Object.Quiet == false)
{
_logger.Warn($"Processing '{_fluentCommandLineParser.Object.File}'");
_logger.Info("");
}
var sw = new Stopwatch();
sw.Start();
var rfc = RecentFileCache.RecentFileCache.LoadFile(_fluentCommandLineParser.Object.File);
if (_fluentCommandLineParser.Object.Quiet == false)
{
_logger.Error($"Source file: {rfc.SourceFile}");
_logger.Info($" Source created: {rfc.SourceCreated.ToString(_dateTimeFormat)} ");
_logger.Info($" Source modified: {rfc.SourceModified.ToString(_dateTimeFormat)}");
_logger.Info($" Source accessed: {rfc.SourceAccessed.ToString(_dateTimeFormat)}");
_logger.Info("");
_logger.Warn("File names");
foreach (var rfcFileName in rfc.FileNames)
{
_logger.Info($"{rfcFileName}");
}
_logger.Info("");
}
sw.Stop();
if (_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("");
}
_logger.Info(
$"---------- Processed '{rfc.SourceFile}' in {sw.Elapsed.TotalSeconds:N8} seconds ----------");
if (_fluentCommandLineParser.Object.Quiet == false)
{
_logger.Info("\r\n");
}
try
{
StreamWriter sw1 = null;
if (_fluentCommandLineParser.Object.CsvDirectory?.Length > 0)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"'{_fluentCommandLineParser.Object.CsvDirectory} does not exist. Creating...'");
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
var outName =
$"{DateTimeOffset.Now:yyyyMMddHHmmss}_RecentFileCacheParser_Output.tsv";
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_fluentCommandLineParser.Object.CsvDirectory =
Path.GetFullPath(outFile);
_logger.Warn(
$"CSV (tab separated) output will be saved to '{Path.GetFullPath(outFile)}'");
try
{
sw1 = new StreamWriter(outFile);
var csv = new CsvWriter(sw1);
csv.Configuration.Delimiter = $"{'\t'}";
csv.Configuration.HasHeaderRecord = true;
var foo = csv.Configuration.AutoMap <CsvOut>();
foo.Map(t => t.SourceAccessed)
.ConvertUsing(t => t.SourceAccessed.ToString(_dateTimeFormat));
foo.Map(t => t.SourceCreated).ConvertUsing(t => t.SourceCreated.ToString(_dateTimeFormat));
foo.Map(t => t.SourceModified)
.ConvertUsing(t => t.SourceModified.ToString(_dateTimeFormat));
csv.WriteHeader(typeof(CsvOut));
csv.NextRecord();
csv.WriteRecords(GetCsvFormat(rfc));
}
catch (Exception ex)
{
_logger.Error(
$"Unable to open '{outFile}' for writing. CSV export canceled. Error: {ex.Message}");
}
}
if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"'{_fluentCommandLineParser.Object.JsonDirectory} does not exist. Creating...'");
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
_logger.Warn($"Saving json output to '{_fluentCommandLineParser.Object.JsonDirectory}'");
SaveJson(rfc, _fluentCommandLineParser.Object.JsonPretty,
_fluentCommandLineParser.Object.JsonDirectory);
}
//Close CSV stuff
sw1?.Flush();
sw1?.Close();
}
catch (Exception e)
{
_logger.Error(
$"Error exporting data! Error: {e.Message}");
}
}
catch (UnauthorizedAccessException ua)
{
_logger.Error(
$"Unable to access '{_fluentCommandLineParser.Object.File}'. Are you running as an administrator? Error: {ua.Message}");
}
catch (Exception ex)
{
_logger.Error(
$"Error processing file '{_fluentCommandLineParser.Object.File}' Please send it to [email protected]. Error: {ex.Message}");
}
}
//private static readonly string VssDir = @"C:\____vssMount";
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains evtx files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription(
"File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to."); // This, --csv, or --xml required
_fluentCommandLineParser.Setup(arg => arg.JsonName)
.As("jsonf")
.WithDescription(
"File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.XmlDirectory)
.As("xml")
.WithDescription(
"Directory to save XML formatted results to."); // This, --csv, or --json required
_fluentCommandLineParser.Setup(arg => arg.XmlName)
.As("xmlf")
.WithDescription(
"File name to save XML formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff")
.SetDefault("yyyy-MM-dd HH:mm:ss.fffffff");
_fluentCommandLineParser.Setup(arg => arg.IncludeIds)
.As("inc")
.WithDescription(
"List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.ExcludeIds)
.As("exc")
.WithDescription(
"List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.StartDate)
.As("sd")
.WithDescription(
"Start date for including events (UTC). Anything OLDER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.EndDate)
.As("ed")
.WithDescription(
"End date for including events (UTC). Anything NEWER than this is dropped. Format should match --dt")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.FullJson)
.As("fj")
.WithDescription(
"When true, export all available data when using --json. Default is FALSE.")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.PayloadAsJson)
.As("pj")
.WithDescription(
"When true, include event *payload* as json. Default is TRUE.")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.TimeDiscrepancyThreshold)
.As("tdt")
.WithDescription(
"The number of seconds to use for time discrepancy detection. Default is 1 second")
.SetDefault(1);
_fluentCommandLineParser.Setup(arg => arg.Metrics)
.As("met")
.WithDescription(
"When true, show metrics about processed event log. Default is TRUE.\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
/*_fluentCommandLineParser.Setup(arg => arg.Vss)
* .As("vss")
* .WithDescription(
* "Process all Volume Shadow Copies that exist on drive specified by -f or -d . Default is FALSE")
* .SetDefault(false);*/
_fluentCommandLineParser.Setup(arg => arg.Dedupe)
.As("dedupe")
.WithDescription(
"Deduplicate -f or -d & VSCs based on SHA-1. First file found wins. Default is TRUE\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.Sync)
.As("sync")
.WithDescription(
"If true, the latest maps from https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps are downloaded and local maps updated. Default is FALSE\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\n\nAuthor: Eric Zimmerman ([email protected])" +
"\nhttps://github.com/EricZimmerman/evtx" +
"\nLinux Port : Scott Dermott ([email protected])" +
"\nhttps://github.com/ScottDermott/evtx";
var footer =
@"Examples: EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out"" --csvf MyOutputFile.csv" +
"\r\n\t " +
@" EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out""" + "\r\n\t " +
@" EvtxECmd -f ""/Temp/Application.evtx"" --json ""/temp/jsonout""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.Sync)
{
try
{
_logger.Info(header);
UpdateFromRepo();
}
catch (Exception e)
{
_logger.Error(e, $"There was an error checking for updates: {e.Message}");
}
Environment.Exit(0);
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
/* if (IsAdministrator() == false)
* {
* _logger.Fatal("Warning: Administrator privileges not found!\r\n");
* }*/
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
/* if (_fluentCommandLineParser.Object.Vss & (IsAdministrator() == false))
* {
* _logger.Error("--vss is present, but administrator rights not found. Exiting\r\n");
* return;
* }*/
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName);
_logger.Warn($"json output will be saved to '{outFile}'\r\n");
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName);
_logger.Warn($"XML output will be saved to '{outFile}'\r\n");
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.StartDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.StartDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_startDate = dt;
_logger.Info($"Setting Start date to '{_startDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.StartDate}' to a valud datetime! Events will not be filtered by Start date!");
}
}
if (_fluentCommandLineParser.Object.EndDate.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.EndDate, null, DateTimeStyles.AssumeUniversal, out var dt))
{
_endDate = dt;
_logger.Info($"Setting End date to '{_endDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'");
}
else
{
_logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.EndDate}' to a valud datetime! Events will not be filtered by End date!");
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
_logger.Info("");
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_logger.Warn($"CSV output will be saved to '{outFile}'\r\n");
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
_csvWriter = new CsvWriter(_swCsv, CultureInfo.InvariantCulture);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
var foo = _csvWriter.Configuration.AutoMap <EventRecord>();
if (_fluentCommandLineParser.Object.PayloadAsJson == false)
{
foo.Map(t => t.Payload).Ignore();
}
else
{
foo.Map(t => t.Payload).Index(22);
}
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).ConvertUsing(t =>
$"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.SourceFile).Index(21);
_csvWriter.Configuration.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!");
}
else
{
_logger.Info($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}");
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false)
{
var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* string driveLetter;
* if (_fluentCommandLineParser.Object.File.IsEmpty() == false)
* {
* driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File))
* .Substring(0, 1);
* }
* else
* {
* driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory))
* .Substring(0, 1);
* }
*
*
* Helper.MountVss(driveLetter,VssDir);
* Console.WriteLine();
* }*/
EventLog.TimeDiscrepancyThreshold = _fluentCommandLineParser.Object.TimeDiscrepancyThreshold;
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
_fluentCommandLineParser.Object.Dedupe = false;
ProcessFile(Path.GetFullPath(_fluentCommandLineParser.Object.File));
/*if (_fluentCommandLineParser.Object.Vss)
* {
* var vssDirs = Directory.GetDirectories(VssDir);
*
* var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File));
* var stem = Path.GetFullPath(_fluentCommandLineParser.Object.File).Replace(root, "");
*
* foreach (var vssDir in vssDirs)
* {
* var newPath = Path.Combine(vssDir, stem);
* if (File.Exists(newPath))
* {
* ProcessFile(newPath);
* }
* }
* }*/
}
else
{
_logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
/* var f = new DirectoryEnumerationFilters
* {
* InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
* RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
* ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
* };*/
/* var dirEnumOptions =
* DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
* DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
* DirectoryEnumerationOptions.BasicSearch;*/
var files2 = Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), "*.evtx");
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
_logger.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file);
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* var vssDirs = Directory.GetDirectories(VssDir);
*
* Console.WriteLine();
*
* foreach (var vssDir in vssDirs)
* {
* var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory));
* var stem = Path.GetFullPath(_fluentCommandLineParser.Object.Directory).Replace(root, "");
*
* var target = Path.Combine(vssDir, stem);
*
* _logger.Fatal($"\r\nSearching 'VSS{target.Replace($"{VssDir}\\","")}' for event logs...");
*
* var vssFiles = Helper.GetFilesFromPath(target, true, "*.evtx");
*
* foreach (var file in vssFiles)
* {
* ProcessFile(file);
* }
* }
*
* }*/
}
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
sw.Stop();
_logger.Info("");
var suff = string.Empty;
if (_fileCount != 1)
{
suff = "s";
}
_logger.Error(
$"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (_errorFiles.Count > 0)
{
_logger.Info("");
_logger.Error("Files with errors");
foreach (var errorFile in _errorFiles)
{
_logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}");
}
_logger.Info("");
}
/*if (_fluentCommandLineParser.Object.Vss)
* {
* if (Directory.Exists(VssDir))
* {
* foreach (var directory in Directory.GetDirectories(VssDir))
* {
* Directory.Delete(directory);
* }
* Directory.Delete(VssDir,true);
* }
* }*/
}
private void ExportDataToFolder(string outputPath)
{
var metadataPane = this.ActiveDocument.MetadataPane;
var exceptionFound = false;
// TODO: Use async but to be well done need to apply async on the DBCommand & DBConnection
// TODO: Show warning message?
if (metadataPane.SelectedModel == null)
{
return;
}
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
ActiveDocument.QueryStopWatch.Start();
foreach (var table in metadataPane.SelectedModel.Tables)
{
var rows = 0;
var tableCnt = 0;
try
{
var csvFilePath = System.IO.Path.Combine(outputPath, $"{table.Name}.csv");
var daxQuery = $"EVALUATE {table.DaxName}";
var totalTables = metadataPane.SelectedModel.Tables.Count;
using (var textWriter = new StreamWriter(csvFilePath, false, Encoding.UTF8))
using (var csvWriter = new CsvHelper.CsvWriter(textWriter))
using (var statusMsg = new StatusBarMessage(this.ActiveDocument, $"Exporting {table.Name}"))
using (var reader = ActiveDocument.Connection.ExecuteReader(daxQuery))
{
rows = 0;
tableCnt++;
// Write Header
foreach (var colName in reader.CleanColumnNames())
{
csvWriter.WriteField(colName);
}
csvWriter.NextRecord();
// Write data
while (reader.Read())
{
for (var fieldOrdinal = 0; fieldOrdinal < reader.FieldCount; fieldOrdinal++)
{
var fieldValue = reader[fieldOrdinal];
csvWriter.WriteField(fieldValue);
}
rows++;
if (rows % 5000 == 0)
{
new StatusBarMessage(ActiveDocument, $"Exporting Table {tableCnt} of {totalTables} : {table.Name} ({rows:N0} rows)");
ActiveDocument.RefreshElapsedTime();
// if cancel has been requested do not write any more records
if (CancelRequested)
{
break;
}
}
csvWriter.NextRecord();
}
ActiveDocument.RefreshElapsedTime();
_eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, $"Exported {rows:N0} rows to {table.Name}.csv"));
// if cancel has been requested do not write any more files
if (CancelRequested)
{
break;
}
}
}
catch (Exception ex)
{
exceptionFound = true;
Log.Error(ex, "{class} {method} {message}", "ExportDataDialogViewModel", "ExportDataToFolder", "Error while exporting model to CSV");
_eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Error, $"Error Exporting '{table.Name}': {ex.Message}"));
break; // exit from the loop if we have caught an exception
}
}
ActiveDocument.QueryStopWatch.Stop();
// export complete
if (!exceptionFound)
{
_eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, $"Model Export Complete: {metadataPane.SelectedModel.Tables.Count} tables exported", ActiveDocument.QueryStopWatch.ElapsedMilliseconds));
}
ActiveDocument.QueryStopWatch.Reset();
}
private void ExportDataToCSV(string outputPath)
{
var exceptionFound = false;
// TODO: Use async but to be well done need to apply async on the DBCommand & DBConnection
// TODO: Show warning message?
if (string.IsNullOrEmpty(Document.Connection.SelectedModelName))
{
return;
}
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
Document.QueryStopWatch.Start();
var selectedTables = Tables.Where(t => t.IsSelected);
var totalTables = selectedTables.Count();
var tableCnt = 0;
string decimalSep = System.Globalization.CultureInfo.CurrentUICulture.NumberFormat.CurrencyDecimalSeparator;
string isoDateFormat = string.Format(Constants.IsoDateMask, decimalSep);
var encoding = new UTF8Encoding(false);
foreach (var table in selectedTables)
{
EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(table));
var rows = 0;
tableCnt++;
try
{
table.Status = ExportStatus.Exporting;
var fileName = CleanNameOfIllegalChars(table.Caption);
var csvFilePath = System.IO.Path.Combine(outputPath, $"{fileName}.csv");
var daxRowCount = $"EVALUATE ROW(\"RowCount\", COUNTROWS( {table.DaxName} ) )";
// get a count of the total rows in the table
var connRead = Document.Connection;
DataTable dtRows = connRead.ExecuteDaxQueryDataTable(daxRowCount);
var totalRows = dtRows.Rows[0].Field <long?>(0) ?? 0;
table.TotalRows = totalRows;
StreamWriter textWriter = null;
try {
textWriter = new StreamWriter(csvFilePath, false, encoding);
using (var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture))
using (var statusMsg = new StatusBarMessage(Document, $"Exporting {table.Caption}"))
{
for (long batchRows = 0; batchRows < totalRows; batchRows += maxBatchSize)
{
var daxQuery = $"EVALUATE {table.DaxName}";
// if the connection supports TOPNSKIP then use that to query batches of rows
if (connRead.AllFunctions.Contains("TOPNSKIP"))
{
daxQuery = $"EVALUATE TOPNSKIP({maxBatchSize}, {batchRows}, {table.DaxName} )";
}
using (var reader = connRead.ExecuteReader(daxQuery))
{
rows = 0;
// configure delimiter
csvWriter.Configuration.Delimiter = CsvDelimiter;
// output dates using ISO 8601 format
csvWriter.Configuration.TypeConverterOptionsCache.AddOptions(
typeof(DateTime),
new CsvHelper.TypeConversion.TypeConverterOptions()
{
Formats = new string[] { isoDateFormat }
});
// if this is the first batch of rows
if (batchRows == 0)
{
// Write Header
foreach (var colName in reader.CleanColumnNames())
{
csvWriter.WriteField(colName);
}
csvWriter.NextRecord();
}
// Write data
while (reader.Read())
{
for (var fieldOrdinal = 0; fieldOrdinal < reader.FieldCount; fieldOrdinal++)
{
var fieldValue = reader[fieldOrdinal];
// quote all string fields
if (reader.GetFieldType(fieldOrdinal) == typeof(string))
{
if (reader.IsDBNull(fieldOrdinal))
{
csvWriter.WriteField("", this.CsvQuoteStrings);
}
else
{
csvWriter.WriteField(fieldValue.ToString(), this.CsvQuoteStrings);
}
}
else
{
csvWriter.WriteField(fieldValue);
}
}
rows++;
if (rows % 5000 == 0)
{
table.RowCount = rows + batchRows;
statusMsg.Update($"Exporting Table {tableCnt} of {totalTables} : {table.DaxName} ({rows + batchRows:N0} rows)");
Document.RefreshElapsedTime();
// if cancel has been requested do not write any more records
if (CancelRequested)
{
table.Status = ExportStatus.Cancelled;
// break out of datareader.Read() loop
break;
}
}
csvWriter.NextRecord();
}
Document.RefreshElapsedTime();
table.RowCount = rows + batchRows;
// if cancel has been requested do not write any more files
if (CancelRequested)
{
EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Warning, "Data Export Cancelled"));
MarkWaitingTablesAsSkipped();
// break out of foreach table loop
break;
}
}
// do not loop around if the current connection does not support TOPNSKIP
if (!connRead.AllFunctions.Contains("TOPNSKIP"))
{
break;
}
} // end of batch
EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, exportTableMsg.Format(rows, rows == 1 ? "":"s", table.DaxName + ".csv")));;
}
}
finally
{
textWriter?.Dispose();
}
table.Status = ExportStatus.Done;
}
catch (Exception ex)
{
table.Status = ExportStatus.Error;
exceptionFound = true;
Log.Error(ex, "{class} {method} {message}", nameof(ExportDataWizardViewModel), nameof(ExportDataToCSV), "Error while exporting model to CSV");
EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Error, $"Error Exporting '{table.DaxName}': {ex.Message}"));
EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(currentTable, true));
continue; // skip to the next table if we have caught an exception
}
}
Document.QueryStopWatch.Stop();
// export complete
if (!exceptionFound)
{
EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, exportCompleteMsg.Format(tableCnt), Document.QueryStopWatch.ElapsedMilliseconds));
}
EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(currentTable, true));
Document.QueryStopWatch.Reset();
}
/// <summary>
///
/// </summary>
/// <param name="dataDirectory"></param>
/// <param name="outputDirectory"></param>
public void PostProcess(string dataDirectory,
string outputDirectory)
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.QuoteAllFields = true;
using (FileStream fileStream = new FileStream(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv"), FileMode.Append, FileAccess.Write, FileShare.Read))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
// Now MD5 the files
foreach (string file in System.IO.Directory.EnumerateFiles(outputDirectory,
"*.xml",
SearchOption.AllDirectories))
{
string fileName = System.IO.Path.GetFileName(file);
if (fileName.StartsWith("Message.Details.") == false)
{
continue;
}
MessageDetails messageDetails = new MessageDetails();
string ret = messageDetails.Load(file);
if (ret.Length == 0)
{
foreach (AttachmentDetails attachment in messageDetails.Attachments)
{
csvWriter.WriteField(attachment.Md5);
csvWriter.WriteField(attachment.File);
csvWriter.WriteField(messageDetails.SrcIp);
csvWriter.WriteField(messageDetails.SrcPort);
csvWriter.WriteField(messageDetails.DstIp);
csvWriter.WriteField(messageDetails.DstPort);
csvWriter.WriteField(messageDetails.To);
csvWriter.WriteField(messageDetails.From);
csvWriter.WriteField(messageDetails.MailFrom);
csvWriter.WriteField(messageDetails.Sender);
csvWriter.WriteField(messageDetails.Subject);
csvWriter.WriteField(messageDetails.Date);
csvWriter.NextRecord();
}
}
}
}
ProcessAttachmentHashes(outputDirectory);
}
/// <summary>
///
/// </summary>
/// <param name="filePath">The output file name</param>
/// <param name="dateFrom"></param>
/// <param name="dateTo"></param>
/// <param name="sid"></param>
public void ExportEventsAll(string filePath,
string dateFrom,
string dateTo,
string sid)
{
if (IsRunning == true)
{
OnExclamation("Already performing an export");
return;
}
IsRunning = true;
new Thread(() =>
{
try
{
using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection()))
{
string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_RULES_EVENTS_EXPORT);
query = query.Replace("#WHERE#", @"WHERE exclude.id IS NULL
AND event.timestamp > @0
AND event.timestamp < @1
AND signature.sig_sid = @2");
List<Event> events = db.Fetch<Event>(query, new object[] { dateFrom, dateTo, sid });
events = Helper.ProcessEventDataSet(events);
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = '\t';
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
// Write out the file headers
csvWriter.WriteField("CID");
csvWriter.WriteField("Src IP");
csvWriter.WriteField("Src Port");
csvWriter.WriteField("Dst IP");
csvWriter.WriteField("Dst Port");
csvWriter.WriteField("Protocol");
csvWriter.WriteField("Timestamp");
csvWriter.WriteField("TCP Flags");
csvWriter.WriteField("Payload (ASCII)");
csvWriter.WriteField("Payload (HEX)");
csvWriter.NextRecord();
foreach (var item in events)
{
csvWriter.WriteField(item.Cid);
csvWriter.WriteField(item.IpSrcTxt);
csvWriter.WriteField(item.SrcPort);
csvWriter.WriteField(item.IpDst);
csvWriter.WriteField(item.DstPort);
csvWriter.WriteField(item.Protocol);
csvWriter.WriteField(item.Timestamp);
csvWriter.WriteField(item.TcpFlagsString);
csvWriter.WriteField(item.PayloadAscii);
csvWriter.WriteField( Helper.ConvertByteArrayToHexString(item.PayloadHex));
csvWriter.NextRecord();
}
}
OnComplete();
}
}
catch (Exception ex)
{
OnError("An error occurred whilst performing the export: " + ex.Message);
}
finally
{
IsRunning = false;
}
}).Start();
}
/// <summary>
///
/// </summary>
/// <param name="filePath"></param>
/// <param name="dateFrom"></param>
/// <param name="initials"></param>
/// <param name="text"></param>
public void ExportAcknowledgmentsFrom(string filePath,
string dateFrom,
string initials,
bool text)
{
if (IsRunning == true)
{
OnExclamation("Already performing an export");
return;
}
IsRunning = true;
new Thread(() =>
{
try
{
if (text == true)
{
using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection()))
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
{
string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_ACKNOWLEDGEMENT);
query = query.Replace("#WHERE#", @"WHERE acknowledgment.initials = @0
AND acknowledgment.timestamp > @1");
var data = db.Fetch<Dictionary<string, object>>(query, new object[] { initials, dateFrom });
foreach (Dictionary<string, object> temp in data)
{
streamWriter.WriteLine("Signature: " + temp["sig_name"].ToString());
streamWriter.WriteLine("SID/GID: " + temp["sig_sid"].ToString() + "/" + temp["sig_gid"].ToString());
if (temp["successful"] == null)
{
streamWriter.WriteLine("Acknowledgement: " + temp["description"].ToString());
}
else
{
streamWriter.WriteLine("Acknowledgement: " + temp["description"].ToString() + " (" + temp["successful"].ToString() + ")");
}
streamWriter.WriteLine("Notes: " + temp["notes"].ToString());
streamWriter.WriteLine(string.Empty);
}
OnComplete();
}
}
else
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = '\t';
using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection()))
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_ACKNOWLEDGEMENT);
query = query.Replace("#WHERE#", @"WHERE acknowledgment.initials = @0
AND acknowledgment.timestamp > @1");
var data = db.Fetch<Dictionary<string, object>>(query, new object[] { initials, dateFrom });
// Write out the file headers
csvWriter.WriteField("Sig Name");
csvWriter.WriteField("Sig SID");
csvWriter.WriteField("Sig GID");
csvWriter.WriteField("Acknowledgement");
csvWriter.WriteField("Notes");
csvWriter.NextRecord();
foreach (Dictionary<string, object> temp in data)
{
csvWriter.WriteField(temp["sig_name"].ToString());
csvWriter.WriteField(temp["sig_sid"].ToString());
csvWriter.WriteField(temp["sig_gid"].ToString());
if (temp["successful"] == null)
{
csvWriter.WriteField(temp["description"].ToString());
}
else
{
csvWriter.WriteField(temp["description"].ToString() + " (" + temp["successful"].ToString() + ")");
}
csvWriter.WriteField(temp["notes"].ToString());
csvWriter.NextRecord();
}
OnComplete();
}
}
}
catch (Exception ex)
{
OnError("An error occurred whilst performing the export: " + ex.Message);
}
finally
{
IsRunning = false;
}
}).Start();
}
/// <summary>
///
/// </summary>
/// <param name="signatures"></param>
/// <param name="filePath">The output file name</param>
public void ExportRules(List<Signature> signatures,
string filePath)
{
if (IsRunning == true)
{
OnExclamation("Already performing an export");
return;
}
IsRunning = true;
new Thread(() =>
{
try
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = '\t';
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
// Write out the file headers
csvWriter.WriteField("SID");
csvWriter.WriteField("Name");
csvWriter.WriteField("Priority");
csvWriter.WriteField("Count");
csvWriter.NextRecord();
foreach (Signature temp in signatures)
{
csvWriter.WriteField(temp.Sid);
csvWriter.WriteField(temp.Name);
csvWriter.WriteField(temp.Priority);
csvWriter.WriteField(temp.Count);
csvWriter.NextRecord();
}
}
OnComplete();
}
catch (Exception ex)
{
OnError("An error occurred whilst performing the export: " + ex.Message);
}
finally
{
IsRunning = false;
}
}).Start();
}
/// <summary>
///
/// </summary>
/// <param name="filePath">The output file name</param>
public void ExportExcludes(string filePath)
{
if (IsRunning == true)
{
OnExclamation("Already performing an export");
return;
}
IsRunning = true;
new Thread(() =>
{
try
{
List<Exclude> excludes = new List<Exclude>();
using (NPoco.Database dbMySql = new NPoco.Database(Db.GetOpenMySqlConnection()))
{
var data = dbMySql.Fetch<Dictionary<string, object>>(_sql.GetQuery(snorbert.Configs.Sql.Query.SQL_EXCLUDES));
foreach (Dictionary<string, object> temp in data)
{
Exclude exclude = new Exclude();
exclude.Id = long.Parse(temp["id"].ToString());
exclude.SigId = long.Parse(temp["sig_id"].ToString());
exclude.SigSid = long.Parse(temp["sig_sid"].ToString());
exclude.Rule = temp["sig_name"].ToString();
exclude.SourceIpText = temp["ip_src"].ToString();
exclude.DestinationIpText = temp["ip_dst"].ToString();
if (((byte[])temp["fp"])[0] == 48)
{
exclude.FalsePositive = false;
}
else
{
exclude.FalsePositive = true;
}
exclude.Timestamp = DateTime.Parse(temp["timestamp"].ToString());
excludes.Add(exclude);
}
}
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = '\t';
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
// Write out the file headers
csvWriter.WriteField("Sig. ID");
csvWriter.WriteField("Source IP");
csvWriter.WriteField("Destination IP");
csvWriter.WriteField("FP");
csvWriter.WriteField("Comment");
csvWriter.WriteField("Sig. Name");
csvWriter.WriteField("Timestamp");
csvWriter.WriteField("Sig.");
csvWriter.NextRecord();
foreach (var temp in excludes)
{
csvWriter.WriteField(temp.SigId);
csvWriter.WriteField(temp.SourceIpText);
csvWriter.WriteField(temp.DestinationIpText);
csvWriter.WriteField(temp.FalsePositive);
csvWriter.WriteField(temp.Comment);
csvWriter.WriteField(temp.Rule);
csvWriter.WriteField(temp.Timestamp);
csvWriter.WriteField(temp.Rule);
csvWriter.NextRecord();
}
}
OnComplete();
}
catch (Exception ex)
{
OnError("An error occurred whilst performing the export: " + ex.Message);
}
finally
{
IsRunning = false;
}
}).Start();
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq");
SetupNLog();
_logger = LogManager.GetLogger("EvtxECmd");
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. This or -d is required\r\n");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to process that contains evtx files. This or -f is required");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV formatted results to."); // This, --json, or --xml required
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription(
"File name to save CSV formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save JSON formatted results to."); // This, --csv, or --xml required
_fluentCommandLineParser.Setup(arg => arg.JsonName)
.As("jsonf")
.WithDescription(
"File name to save JSON formatted results to. When present, overrides default name");
_fluentCommandLineParser.Setup(arg => arg.XmlDirectory)
.As("xml")
.WithDescription(
"Directory to save XML formatted results to."); // This, --csv, or --json required
_fluentCommandLineParser.Setup(arg => arg.XmlName)
.As("xmlf")
.WithDescription(
"File name to save XML formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff")
.SetDefault("yyyy-MM-dd HH:mm:ss.fffffff");
_fluentCommandLineParser.Setup(arg => arg.IncludeIds)
.As("inc")
.WithDescription(
"List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.ExcludeIds)
.As("exc")
.WithDescription(
"List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.FullJson)
.As("fj")
.WithDescription(
"When true, export all available data when using --json. Default is FALSE.")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Metrics)
.As("met")
.WithDescription(
"When true, show metrics about processed event log. Default is TRUE.\r\n")
.SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.MapsDirectory)
.As("maps")
.WithDescription(
"The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ")
.SetDefault(Path.Combine(BaseDirectory, "Maps"));
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Show debug information during processing").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.Trace)
.As("trace")
.WithDescription("Show trace information during processing\r\n").SetDefault(false);
var header =
$"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/evtx";
var footer =
@"Examples: EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out"" --csvf MyOutputFile.csv" +
"\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out""" + "\r\n\t " +
@" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --json ""c:\temp\jsonout""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() &&
_fluentCommandLineParser.Object.Directory.IsNullOrEmpty())
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("-f or -d is required. Exiting");
return;
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (IsAdministrator() == false)
{
_logger.Fatal("Warning: Administrator privileges not found!\r\n");
}
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
if (_fluentCommandLineParser.Object.Trace)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace);
}
LogManager.ReconfigExistingLoggers();
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName);
_logger.Warn($"json output will be saved to '{outFile}'\r\n");
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName);
_logger.Warn($"XML output will be saved to '{outFile}'\r\n");
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false)
{
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating...");
try
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
catch (Exception)
{
_logger.Fatal(
$"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting");
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
_logger.Warn($"CSV output will be saved to '{outFile}'\r\n");
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
_csvWriter = new CsvWriter(_swCsv);
}
catch (Exception)
{
_logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n");
Environment.Exit(0);
}
var foo = _csvWriter.Configuration.AutoMap <EventRecord>();
foo.Map(t => t.PayloadXml).Ignore();
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.TimeCreated).Index(1);
foo.Map(t => t.TimeCreated).ConvertUsing(t =>
$"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}");
foo.Map(t => t.EventId).Index(2);
foo.Map(t => t.Level).Index(3);
foo.Map(t => t.Provider).Index(4);
foo.Map(t => t.Channel).Index(5);
foo.Map(t => t.ProcessId).Index(6);
foo.Map(t => t.ThreadId).Index(7);
foo.Map(t => t.Computer).Index(8);
foo.Map(t => t.UserId).Index(9);
foo.Map(t => t.MapDescription).Index(10);
foo.Map(t => t.UserName).Index(11);
foo.Map(t => t.RemoteHost).Index(12);
foo.Map(t => t.PayloadData1).Index(13);
foo.Map(t => t.PayloadData2).Index(14);
foo.Map(t => t.PayloadData3).Index(15);
foo.Map(t => t.PayloadData4).Index(16);
foo.Map(t => t.PayloadData5).Index(17);
foo.Map(t => t.PayloadData6).Index(18);
foo.Map(t => t.ExecutableInfo).Index(19);
foo.Map(t => t.SourceFile).Index(20);
_csvWriter.Configuration.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false)
{
_logger.Warn(
$"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!");
}
else
{
_logger.Debug($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'");
var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory));
if (errors)
{
return;
}
_logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}");
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false)
{
var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false)
{
if (File.Exists(_fluentCommandLineParser.Object.File) == false)
{
_logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting");
return;
}
ProcessFile(_fluentCommandLineParser.Object.File);
}
else
{
_logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
var f = new DirectoryEnumerationFilters();
f.InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX";
f.RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink;
f.ErrorFilter = (errorCode, errorMessage, pathProcessed) => true;
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), dirEnumOptions, f);
foreach (var file in files2)
{
ProcessFile(file);
}
}
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
sw.Stop();
_logger.Info("");
var suff = string.Empty;
if (_fileCount != 1)
{
suff = "s";
}
_logger.Error(
$"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n");
if (_errorFiles.Count > 0)
{
_logger.Info("");
_logger.Error("Files with errors");
foreach (var errorFile in _errorFiles)
{
_logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}");
}
_logger.Info("");
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("x3MPpeQSBUUsXl3DjekRQ9kYjyN3cr5JuwdoOBpZ");
SetupNLog();
_keywords = new HashSet <string> {
"temp", "tmp"
};
_logger = LogManager.GetCurrentClassLogger();
if (!CheckForDotnet46())
{
_logger.Warn(".net 4.6 not detected. Please install .net 4.6 and try again.");
return;
}
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>
{
IsCaseSensitive = false
};
_fluentCommandLineParser.Setup(arg => arg.File)
.As('f')
.WithDescription("File to process. Either this or -d is required");
_fluentCommandLineParser.Setup(arg => arg.Directory)
.As('d')
.WithDescription("Directory to recursively process. Either this or -f is required");
_fluentCommandLineParser.Setup(arg => arg.Keywords)
.As('k')
.WithDescription(
"Comma separated list of keywords to highlight in output. By default, 'temp' and 'tmp' are highlighted. Any additional keywords will be added to these.");
_fluentCommandLineParser.Setup(arg => arg.OutFile)
.As('o')
.WithDescription(
"When specified, save prefetch file bytes to the given path. Useful to look at decompressed Win10 files");
_fluentCommandLineParser.Setup(arg => arg.Quiet)
.As('q')
.WithDescription(
"Do not dump full details about each file processed. Speeds up processing when using --json or --csv\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.JsonDirectory)
.As("json")
.WithDescription(
"Directory to save json representation to. Use --pretty for a more human readable layout");
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription(
"Directory to save CSV results to. Be sure to include the full path in double quotes");
_fluentCommandLineParser.Setup(arg => arg.xHtmlDirectory)
.As("html")
.WithDescription(
"Directory to save xhtml formatted results to. Be sure to include the full path in double quotes");
_fluentCommandLineParser.Setup(arg => arg.JsonPretty)
.As("pretty")
.WithDescription(
"When exporting to json, use a more human readable layout\r\n").SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.CsvSeparator)
.As("cs")
.WithDescription(
"When true, use comma instead of tab for field separator. Default is true").SetDefault(true);
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss")
.SetDefault("yyyy-MM-dd HH:mm:ss");
_fluentCommandLineParser.Setup(arg => arg.PreciseTimestamps)
.As("mp")
.WithDescription(
"When true, display higher precision for timestamps. Default is false").SetDefault(false);
var header =
$"PECmd version {Assembly.GetExecutingAssembly().GetName().Version}" +
"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
"\r\nhttps://github.com/EricZimmerman/PECmd";
var footer = @"Examples: PECmd.exe -f ""C:\Temp\CALC.EXE-3FBEF7FD.pf""" + "\r\n\t " +
@" PECmd.exe -f ""C:\Temp\CALC.EXE-3FBEF7FD.pf"" --json ""D:\jsonOutput"" --jsonpretty" +
"\r\n\t " +
@" PECmd.exe -d ""C:\Temp"" -k ""system32, fonts""" + "\r\n\t " +
@" PECmd.exe -d ""C:\Temp"" --csv ""c:\temp"" --json c:\temp\json" +
"\r\n\t " +
@" PECmd.exe -d ""C:\Windows\Prefetch""" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help")
.WithHeader(header)
.Callback(text => _logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_logger.Error("");
_logger.Error(result.ErrorText);
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.File) &&
UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.Directory))
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
_logger.Warn("Either -f or -d is required. Exiting");
return;
}
if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.File) == false &&
!File.Exists(_fluentCommandLineParser.Object.File))
{
_logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting");
return;
}
if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.Directory) == false &&
!Directory.Exists(_fluentCommandLineParser.Object.Directory))
{
_logger.Warn($"Directory '{_fluentCommandLineParser.Object.Directory}' not found. Exiting");
return;
}
if (_fluentCommandLineParser.Object.Keywords?.Length > 0)
{
var kws = _fluentCommandLineParser.Object.Keywords.ToLowerInvariant().Split(new[] { ',' },
StringSplitOptions.RemoveEmptyEntries);
foreach (var kw in kws)
{
_keywords.Add(kw.Trim());
}
}
if (_fluentCommandLineParser.Object.CsvSeparator)
{
_exportExt = "csv";
}
_logger.Info(header);
_logger.Info("");
_logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}");
if (IsAdministrator() == false)
{
_logger.Fatal("\r\nWarning: Administrator privileges not found!");
}
_logger.Info("");
_logger.Info($"Keywords: {string.Join(", ", _keywords)}");
_logger.Info("");
if (_fluentCommandLineParser.Object.PreciseTimestamps)
{
_fluentCommandLineParser.Object.DateTimeFormat = _preciseTimeFormat;
}
_processedFiles = new List <IPrefetch>();
_failedFiles = new List <string>();
if (_fluentCommandLineParser.Object.File?.Length > 0)
{
IPrefetch pf = null;
try
{
pf = LoadFile(_fluentCommandLineParser.Object.File);
if (pf != null)
{
if (_fluentCommandLineParser.Object.OutFile.IsNullOrEmpty() == false)
{
try
{
if (Directory.Exists(Path.GetDirectoryName(_fluentCommandLineParser.Object.OutFile)) ==
false)
{
Directory.CreateDirectory(
Path.GetDirectoryName(_fluentCommandLineParser.Object.OutFile));
}
PrefetchFile.SavePrefetch(_fluentCommandLineParser.Object.OutFile, pf);
_logger.Info($"Saved prefetch bytes to '{_fluentCommandLineParser.Object.OutFile}'");
}
catch (Exception e)
{
_logger.Error($"Unable to save prefetch file. Error: {e.Message}");
}
}
_processedFiles.Add(pf);
}
}
catch (UnauthorizedAccessException ex)
{
_logger.Error(
$"Unable to access '{_fluentCommandLineParser.Object.File}'. Are you running as an administrator? Error: {ex.Message}");
}
catch (Exception ex)
{
_logger.Error(
$"Error getting prefetch files in '{_fluentCommandLineParser.Object.Directory}'. Error: {ex.Message}");
}
}
else
{
_logger.Info($"Looking for prefetch files in '{_fluentCommandLineParser.Object.Directory}'");
_logger.Info("");
string[] pfFiles = null;
try
{
pfFiles = Directory.GetFiles(_fluentCommandLineParser.Object.Directory, "*.pf",
SearchOption.AllDirectories);
}
catch (UnauthorizedAccessException ua)
{
_logger.Error(
$"Unable to access '{_fluentCommandLineParser.Object.Directory}'. Are you running as an administrator? Error: {ua.Message}");
return;
}
catch (Exception ex)
{
_logger.Error(
$"Error getting prefetch files in '{_fluentCommandLineParser.Object.Directory}'. Error: {ex.Message}");
return;
}
_logger.Info($"Found {pfFiles.Length:N0} Prefetch files");
_logger.Info("");
var sw = new Stopwatch();
sw.Start();
foreach (var file in pfFiles)
{
var pf = LoadFile(file);
if (pf != null)
{
_processedFiles.Add(pf);
}
}
sw.Stop();
if (_fluentCommandLineParser.Object.Quiet)
{
_logger.Info("");
}
_logger.Info(
$"Processed {pfFiles.Length - _failedFiles.Count:N0} out of {pfFiles.Length:N0} files in {sw.Elapsed.TotalSeconds:N4} seconds");
if (_failedFiles.Count > 0)
{
_logger.Info("");
_logger.Warn("Failed files");
foreach (var failedFile in _failedFiles)
{
_logger.Info($" {failedFile}");
}
}
}
if (_processedFiles.Count > 0)
{
_logger.Info("");
try
{
CsvWriter csv = null;
StreamWriter streamWriter = null;
CsvWriter csvTl = null;
StreamWriter streamWriterTl = null;
if (_fluentCommandLineParser.Object.CsvDirectory?.Length > 0)
{
var outName = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_PECmd_Output.{_exportExt}";
var outNameTl = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_PECmd_Output_Timeline.{_exportExt}";
var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName);
var outFileTl = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outNameTl);
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
_logger.Warn(
$"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' does not exist. Creating...");
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
_logger.Warn($"CSV output will be saved to '{outFile}'");
_logger.Warn($"CSV time line output will be saved to '{outFileTl}'");
try
{
streamWriter = new StreamWriter(outFile);
csv = new CsvWriter(streamWriter);
if (_fluentCommandLineParser.Object.CsvSeparator == false)
{
csv.Configuration.Delimiter = "\t";
}
csv.WriteHeader(typeof(CsvOut));
csv.NextRecord();
streamWriterTl = new StreamWriter(outFileTl);
csvTl = new CsvWriter(streamWriterTl);
if (_fluentCommandLineParser.Object.CsvSeparator == false)
{
csvTl.Configuration.Delimiter = "\t";
}
csvTl.WriteHeader(typeof(CsvOutTl));
csvTl.NextRecord();
}
catch (Exception ex)
{
_logger.Error(
$"Unable to open '{outFile}' for writing. CSV export canceled. Error: {ex.Message}");
}
}
if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0)
{
if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false)
{
_logger.Warn(
$"'{_fluentCommandLineParser.Object.JsonDirectory} does not exist. Creating...'");
Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory);
}
_logger.Warn($"Saving json output to '{_fluentCommandLineParser.Object.JsonDirectory}'");
}
XmlTextWriter xml = null;
if (_fluentCommandLineParser.Object.xHtmlDirectory?.Length > 0)
{
if (Directory.Exists(_fluentCommandLineParser.Object.xHtmlDirectory) == false)
{
_logger.Warn(
$"'{_fluentCommandLineParser.Object.xHtmlDirectory} does not exist. Creating...'");
Directory.CreateDirectory(_fluentCommandLineParser.Object.xHtmlDirectory);
}
var outDir = Path.Combine(_fluentCommandLineParser.Object.xHtmlDirectory,
$"{DateTimeOffset.UtcNow:yyyyMMddHHmmss}_PECmd_Output_for_{_fluentCommandLineParser.Object.xHtmlDirectory.Replace(@":\", "_").Replace(@"\", "_")}");
if (Directory.Exists(outDir) == false)
{
Directory.CreateDirectory(outDir);
}
var styleDir = Path.Combine(outDir, "styles");
if (Directory.Exists(styleDir) == false)
{
Directory.CreateDirectory(styleDir);
}
File.WriteAllText(Path.Combine(styleDir, "normalize.css"), Resources.normalize);
File.WriteAllText(Path.Combine(styleDir, "style.css"), Resources.style);
Resources.directories.Save(Path.Combine(styleDir, "directories.png"));
Resources.filesloaded.Save(Path.Combine(styleDir, "filesloaded.png"));
var outFile = Path.Combine(_fluentCommandLineParser.Object.xHtmlDirectory, outDir,
"index.xhtml");
_logger.Warn($"Saving HTML output to '{outFile}'");
xml = new XmlTextWriter(outFile, Encoding.UTF8)
{
Formatting = Formatting.Indented,
Indentation = 4
};
xml.WriteStartDocument();
xml.WriteProcessingInstruction("xml-stylesheet", "href=\"styles/normalize.css\"");
xml.WriteProcessingInstruction("xml-stylesheet", "href=\"styles/style.css\"");
xml.WriteStartElement("document");
}
if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false ||
_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false ||
_fluentCommandLineParser.Object.xHtmlDirectory.IsNullOrEmpty() == false)
{
foreach (var processedFile in _processedFiles)
{
var o = GetCsvFormat(processedFile);
try
{
foreach (var dateTimeOffset in processedFile.LastRunTimes)
{
var t = new CsvOutTl();
var exePath =
processedFile.Filenames.FirstOrDefault(
y => y.EndsWith(processedFile.Header.ExecutableFilename));
if (exePath == null)
{
exePath = processedFile.Header.ExecutableFilename;
}
t.ExecutableName = exePath;
t.RunTime = dateTimeOffset.ToString(_fluentCommandLineParser.Object.DateTimeFormat);
csvTl?.WriteRecord(t);
csvTl?.NextRecord();
}
}
catch (Exception ex)
{
_logger.Error(
$"Error getting time line record for '{processedFile.SourceFilename}' to '{_fluentCommandLineParser.Object.CsvDirectory}'. Error: {ex.Message}");
}
try
{
csv?.WriteRecord(o);
csv?.NextRecord();
}
catch (Exception ex)
{
_logger.Error(
$"Error writing CSV record for '{processedFile.SourceFilename}' to '{_fluentCommandLineParser.Object.CsvDirectory}'. Error: {ex.Message}");
}
if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0)
{
SaveJson(processedFile, _fluentCommandLineParser.Object.JsonPretty,
_fluentCommandLineParser.Object.JsonDirectory);
}
//XHTML
xml?.WriteStartElement("Container");
xml?.WriteElementString("SourceFile", o.SourceFilename);
xml?.WriteElementString("SourceCreated", o.SourceCreated);
xml?.WriteElementString("SourceModified", o.SourceModified);
xml?.WriteElementString("SourceAccessed", o.SourceAccessed);
xml?.WriteElementString("LastRun", o.LastRun);
xml?.WriteElementString("PreviousRun0", $"{o.PreviousRun0}");
xml?.WriteElementString("PreviousRun1", $"{o.PreviousRun1}");
xml?.WriteElementString("PreviousRun2", $"{o.PreviousRun2}");
xml?.WriteElementString("PreviousRun3", $"{o.PreviousRun3}");
xml?.WriteElementString("PreviousRun4", $"{o.PreviousRun4}");
xml?.WriteElementString("PreviousRun5", $"{o.PreviousRun5}");
xml?.WriteElementString("PreviousRun6", $"{o.PreviousRun6}");
xml?.WriteStartElement("ExecutableName");
xml?.WriteAttributeString("title",
"Note: The name of the executable tracked by the pf file");
xml?.WriteString(o.ExecutableName);
xml?.WriteEndElement();
xml?.WriteElementString("RunCount", $"{o.RunCount}");
xml?.WriteStartElement("Size");
xml?.WriteAttributeString("title", "Note: The size of the executable in bytes");
xml?.WriteString(o.Size);
xml?.WriteEndElement();
xml?.WriteStartElement("Hash");
xml?.WriteAttributeString("title",
"Note: The calculated hash for the pf file that should match the hash in the source file name");
xml?.WriteString(o.Hash);
xml?.WriteEndElement();
xml?.WriteStartElement("Version");
xml?.WriteAttributeString("title",
"Note: The operating system that generated the prefetch file");
xml?.WriteString(o.Version);
xml?.WriteEndElement();
xml?.WriteElementString("Note", o.Note);
xml?.WriteElementString("Volume0Name", o.Volume0Name);
xml?.WriteElementString("Volume0Serial", o.Volume0Serial);
xml?.WriteElementString("Volume0Created", o.Volume0Created);
xml?.WriteElementString("Volume1Name", o.Volume1Name);
xml?.WriteElementString("Volume1Serial", o.Volume1Serial);
xml?.WriteElementString("Volume1Created", o.Volume1Created);
xml?.WriteStartElement("Directories");
xml?.WriteAttributeString("title",
"A comma separated list of all directories accessed by the executable");
xml?.WriteString(o.Directories);
xml?.WriteEndElement();
xml?.WriteStartElement("FilesLoaded");
xml?.WriteAttributeString("title",
"A comma separated list of all files that were loaded by the executable");
xml?.WriteString(o.FilesLoaded);
xml?.WriteEndElement();
xml?.WriteEndElement();
}
//Close CSV stuff
streamWriter?.Flush();
streamWriter?.Close();
streamWriterTl?.Flush();
streamWriterTl?.Close();
//Close XML
xml?.WriteEndElement();
xml?.WriteEndDocument();
xml?.Flush();
}
}
catch (Exception ex)
{
_logger.Error($"Error exporting data! Error: {ex.Message}");
}
}
}
public Task OutputResultsAsync(IQueryRunner runner)
{
var dlg = new Microsoft.Win32.SaveFileDialog
{
DefaultExt = ".txt",
Filter = "Tab separated text file|*.txt|Comma separated text file - UTF8|*.csv|Comma separated text file - Unicode|*.csv|Custom Export Format (Configure in Options)|*.csv"
};
string fileName = "";
long durationMs = 0;
// Show save file dialog box
var result = dlg.ShowDialog();
// Process save file dialog box results
if (result == true)
{
// Save document
fileName = dlg.FileName;
return(Task.Run(() =>
{
try
{
runner.OutputMessage("Query Started");
var sw = Stopwatch.StartNew();
string sep = "\t";
bool shouldQuoteStrings = true; //default to quoting all string fields
string decimalSep = System.Globalization.CultureInfo.CurrentUICulture.NumberFormat.CurrencyDecimalSeparator;
string isoDateFormat = string.Format(Constants.IsoDateMask, decimalSep);
var enc = Encoding.UTF8;
switch (dlg.FilterIndex)
{
case 1: // tab separated
sep = "\t";
break;
case 2: // utf-8 csv
sep = System.Globalization.CultureInfo.CurrentUICulture.TextInfo.ListSeparator;
break;
case 3: //unicode csv
enc = Encoding.Unicode;
sep = System.Globalization.CultureInfo.CurrentUICulture.TextInfo.ListSeparator;
break;
case 4:
// TODO - custom export format
sep = runner.Options.GetCustomCsvDelimiter();
shouldQuoteStrings = runner.Options.CustomCsvQuoteStringFields;
break;
}
var daxQuery = runner.QueryText;
var reader = runner.ExecuteDataReaderQuery(daxQuery);
try
{
if (reader != null)
{
int iFileCnt = 1;
var outputFilename = fileName;
runner.OutputMessage("Command Complete, writing output file");
bool moreResults = true;
while (moreResults)
{
int iMaxCol = reader.FieldCount - 1;
int iRowCnt = 0;
if (iFileCnt > 1)
{
outputFilename = AddFileCntSuffix(fileName, iFileCnt);
}
using (var textWriter = new System.IO.StreamWriter(outputFilename, false, enc))
{
using (var csvWriter = new CsvHelper.CsvWriter(textWriter))
{
// CSV Writer config
csvWriter.Configuration.Delimiter = sep;
// Datetime as ISOFormat
csvWriter.Configuration.TypeConverterOptionsCache.AddOptions(
typeof(DateTime),
new CsvHelper.TypeConversion.TypeConverterOptions()
{
Formats = new string[] { isoDateFormat }
});
// write out clean column names
foreach (var colName in reader.CleanColumnNames())
{
csvWriter.WriteField(colName);
}
csvWriter.NextRecord();
while (reader.Read())
{
iRowCnt++;
for (int iCol = 0; iCol < reader.FieldCount; iCol++)
{
var fieldValue = reader[iCol];
// quote all string fields
if (reader.GetFieldType(iCol) == typeof(string))
{
if (reader.IsDBNull(iCol))
{
csvWriter.WriteField("", shouldQuoteStrings);
}
else
{
csvWriter.WriteField(fieldValue.ToString(), shouldQuoteStrings);
}
}
else
{
csvWriter.WriteField(fieldValue);
}
}
csvWriter.NextRecord();
if (iRowCnt % 1000 == 0)
{
runner.NewStatusBarMessage(string.Format("Written {0:n0} rows to the file output", iRowCnt));
}
}
}
}
runner.OutputMessage(
string.Format("Query {2} Completed ({0:N0} row{1} returned)"
, iRowCnt
, iRowCnt == 1 ? "" : "s", iFileCnt)
);
runner.RowCount = iRowCnt;
moreResults = reader.NextResult();
iFileCnt++;
}
sw.Stop();
durationMs = sw.ElapsedMilliseconds;
runner.SetResultsMessage("Query results written to file", OutputTargets.File);
runner.ActivateOutput();
}
else
{
runner.OutputError("Query Batch Completed with errors", durationMs);
}
}
finally
{
if (reader != null)
{
reader.Dispose();
}
}
}
catch (Exception ex)
{
runner.ActivateOutput();
runner.OutputError(ex.Message);
#if DEBUG
runner.OutputError(ex.StackTrace);
#endif
}
finally
{
runner.OutputMessage("Query Batch Completed", durationMs);
runner.QueryCompleted();
}
}));
}
// else dialog was cancelled so return an empty task.
return(Task.Run(() => { }));
}
/// <summary>
///
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void contextExportUniqueSourceIp_Click(object sender, EventArgs e)
{
SaveFileDialog saveFileDialog = new SaveFileDialog();
saveFileDialog.Title = "Select the export CSV";
saveFileDialog.Filter = "TSV Files|*.tsv";
if (saveFileDialog.ShowDialog(this) == DialogResult.Cancel)
{
return;
}
List<Session> sessions = listSession.Objects.Cast<Session>().ToList();
var unique = sessions.Select(s => s.SrcIpText).Distinct();
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = "\t";
using (FileStream fileStream = new FileStream(saveFileDialog.FileName, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
csvWriter.WriteField("Source IP");
csvWriter.NextRecord();
foreach (var ip in unique)
{
csvWriter.WriteField(ip);
csvWriter.NextRecord();
}
}
}
/// <summary>
///
/// </summary>
/// <param name="events"></param>
/// <param name="filePath">The output file name</param>
public void ExportEventCurrent(List<Event> events,
string filePath)
{
if (IsRunning == true)
{
OnExclamation("Already performing an export");
return;
}
IsRunning = true;
new Thread(() =>
{
try
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = '\t';
using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
// Write out the file headers
csvWriter.WriteField("CID");
csvWriter.WriteField("Src IP");
csvWriter.WriteField("Src Port");
csvWriter.WriteField("Dst IP");
csvWriter.WriteField("Dst Port");
csvWriter.WriteField("Protocol");
csvWriter.WriteField("Timestamp");
csvWriter.WriteField("TCP Flags");
csvWriter.WriteField("Payload (ASCII)");
csvWriter.NextRecord();
foreach (Event temp in events)
{
csvWriter.WriteField(temp.Cid);
csvWriter.WriteField(temp.IpSrcTxt);
csvWriter.WriteField(temp.SrcPort);
csvWriter.WriteField(temp.IpDstTxt);
csvWriter.WriteField(temp.DstPort);
csvWriter.WriteField(temp.Protocol);
csvWriter.WriteField(temp.Timestamp);
csvWriter.WriteField(temp.TcpFlagsString);
csvWriter.WriteField(temp.PayloadAscii);
csvWriter.NextRecord();
}
}
OnComplete();
}
catch (Exception ex)
{
OnError("An error occurred whilst performing the export: " + ex.Message);
}
finally
{
IsRunning = false;
}
}).Start();
}
/// <summary>
///
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void menuFileExportUrls_Click(object sender, EventArgs e)
{
FolderBrowserDialog fbd = new FolderBrowserDialog();
fbd.Description = "Select the export directory";
if (fbd.ShowDialog(this) == DialogResult.Cancel)
{
return;
}
string directory = fbd.SelectedPath;
string fileName = System.IO.Path.Combine(directory, "urls.tsv");
new Thread(() =>
{
MethodInvoker methodInvoker = delegate
{
using (new HourGlass(this))
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.Delimiter = "\t";
using (FileStream fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
csvWriter.WriteField("Host");
csvWriter.WriteField("Destination IP");
csvWriter.WriteField("URL");
csvWriter.NextRecord();
List<string> temp = new List<string>();
foreach (Session session in listSession.Objects)
{
if (File.Exists(System.IO.Path.Combine(this.dataDirectory, session.Guid + ".urls")) == false)
{
continue;
}
string[] urls = File.ReadAllLines(System.IO.Path.Combine(this.dataDirectory, session.Guid + ".urls"));
foreach (string url in urls)
{
string tempUrl = url.Trim();
if (tempUrl.Length == 0)
{
continue;
}
if (temp.Contains(session.HttpHost + tempUrl) == false)
{
temp.Add(session.HttpHost + tempUrl);
}
csvWriter.WriteField(session.HttpHost);
csvWriter.WriteField(session.DstIpText);
csvWriter.WriteField(tempUrl);
csvWriter.NextRecord();
}
}
temp.Sort();
foreach (string url in temp)
{
IO.WriteUnicodeTextToFile(url, System.IO.Path.Combine(directory, "urls.txt"), true);
}
}
}
};
if (this.InvokeRequired == true)
{
this.BeginInvoke(methodInvoker);
}
else
{
methodInvoker.Invoke();
}
}).Start();
}
internal string createCsvWithoutStructure(IEnumerable<dynamic> data, CsvConfiguration configuration)
{
var guid = Guid.NewGuid().ToString();
var builder = new StringBuilder();
var writer = new CsvHelper.CsvWriter(new StringWriter(builder), configuration);
if (data.Count() == 0)
return null;
var first = ToDictionary(guid, data.First());
foreach (var name in first.Keys)
writer.WriteField(name);
writer.NextRecord();
foreach (var datum in data)
{
var dict = ToDictionary(guid, datum);
foreach (var key in first.Keys)
writer.WriteField(dict[key].ToString());
writer.NextRecord();
}
return builder.ToString();
}
static async Task Main()
{
var connectionString = "Data Source=/home/daniel/Downloads/unterdb_DE.db;Cache=Shared";
var query = "select id, tags from locations";
var hashSeparator = "|";
var nodesFilePath = "/home/daniel/Desktop/tag_nodes.csv";
var edgesFilePath = "/home/daniel/Desktop/tag_edges.csv";
var minCoOccurrence = 50;
var index = new Index();
var posts = new SqlitePostReader(
connectionString,
query,
hashSeparator);
await Read(posts, to : index);
var foundEdges = await CountCoOccurrences(index);
var usedTags = new HashSet <string>();
var utf8WithoutBom = new UTF8Encoding(false);
await using var edgesStream = File.Create(edgesFilePath);
await using var edgesWriter = new StreamWriter(edgesStream, utf8WithoutBom);
await using var edgesCsvWriter = new CsvHelper.CsvWriter(
edgesWriter,
new CsvConfiguration(CultureInfo.InvariantCulture));
edgesCsvWriter.WriteField("Source", true);
edgesCsvWriter.WriteField("Target", true);
edgesCsvWriter.WriteField("Weight", true);
edgesCsvWriter.NextRecord();
foreach (var edge in foundEdges)
{
if (edge.Value.Item2 < minCoOccurrence)
{
continue;
}
usedTags.Add(edge.Key);
usedTags.Add(edge.Value.Item1);
edgesCsvWriter.WriteField(edge.Key, true);
edgesCsvWriter.WriteField(edge.Value.Item1, true);
edgesCsvWriter.WriteField(edge.Value.Item2.ToString(), true);
edgesCsvWriter.NextRecord();
}
await using var nodesStream = File.Create(nodesFilePath);
await using var nodesWriter = new StreamWriter(nodesStream, utf8WithoutBom);
await using var nodesCsvWriter = new CsvHelper.CsvWriter(
nodesWriter,
new CsvConfiguration(CultureInfo.InvariantCulture));
nodesCsvWriter.WriteField("Id", true);
nodesCsvWriter.WriteField("Label", true);
nodesCsvWriter.WriteField("Weight", true);
nodesCsvWriter.NextRecord();
foreach (var tag in index.TagValuesByTagHash)
{
if (!usedTags.Contains(tag.Key)) // Skip all tags which have no links
{
continue;
}
var totalOccurrences = index.TotalOccurrencesByTagHash[tag.Key];
nodesCsvWriter.WriteField(tag.Key, true);
nodesCsvWriter.WriteField(tag.Value, true);
nodesCsvWriter.WriteField(totalOccurrences.ToString(), true);
nodesCsvWriter.NextRecord();
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("7iL4b0Me7W8PbFflftqWgfQCIdf55flrT2O11zIP");
SetupNLog();
var logger = LogManager.GetCurrentClassLogger();
if (!CheckForDotnet46())
{
logger.Warn(".net 4.6 not detected. Please install .net 4.6 and try again.");
return;
}
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>();
_fluentCommandLineParser.Setup(arg => arg.SaveTo)
.As("csv")
.WithDescription("Directory to save results. Required")
.Required();
_fluentCommandLineParser.Setup(arg => arg.HiveFile)
.As('f')
.WithDescription(
"Full path to SYSTEM hive to process. If this option is not specified, the live Registry will be used")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.SortTimestamps)
.As('t')
.WithDescription("Sorts last modified timestamps in descending order\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.ControlSet)
.As('c')
.WithDescription("The ControlSet to parse. Default is to extract all control sets.")
.SetDefault(-1);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As('d')
.WithDescription("Debug mode")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss")
.SetDefault("yyyy-MM-dd HH:mm:ss");
var header =
$"AppCompatCache Parser version {Assembly.GetExecutingAssembly().GetName().Version}" +
$"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
$"\r\nhttps://github.com/EricZimmerman/AppCompatCacheParser";
var footer = @"Examples: AppCompatCacheParser.exe --csv c:\temp -t -c 2" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help").WithHeader(header).Callback(text => logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
var hiveToProcess = "Live Registry";
if (_fluentCommandLineParser.Object.HiveFile?.Length > 0)
{
hiveToProcess = _fluentCommandLineParser.Object.HiveFile;
}
logger.Info(header);
logger.Info("");
logger.Info($"Processing hive '{hiveToProcess}'");
logger.Info("");
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
try
{
var appCompat = new AppCompatCache.AppCompatCache(_fluentCommandLineParser.Object.HiveFile,
_fluentCommandLineParser.Object.ControlSet);
var outFileBase = string.Empty;
if (_fluentCommandLineParser.Object.HiveFile?.Length > 0)
{
if (_fluentCommandLineParser.Object.ControlSet >= 0)
{
outFileBase =
$"{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_ControlSet00{_fluentCommandLineParser.Object.ControlSet}_AppCompatCache.tsv";
}
else
{
outFileBase =
$"{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_AppCompatCache.tsv";
}
}
else
{
outFileBase = $"{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.tsv";
}
if (Directory.Exists(_fluentCommandLineParser.Object.SaveTo) == false)
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.SaveTo);
}
var outFilename = Path.Combine(_fluentCommandLineParser.Object.SaveTo, outFileBase);
var sw = new StreamWriter(outFilename);
var csv = new CsvWriter(sw);
csv.Configuration.HasHeaderRecord = true;
csv.Configuration.Delimiter = "\t";
var foo = csv.Configuration.AutoMap <CacheEntry>();
var o = new TypeConverterOptions
{
DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal
};
csv.Configuration.TypeConverterOptionsCache.AddOptions <CacheEntry>(o);
foo.Map(t => t.LastModifiedTimeUTC).ConvertUsing(t => t.LastModifiedTimeUTC.ToString(_fluentCommandLineParser.Object.DateTimeFormat));
foo.Map(t => t.CacheEntrySize).Ignore();
foo.Map(t => t.Data).Ignore();
foo.Map(t => t.InsertFlags).Ignore();
foo.Map(t => t.DataSize).Ignore();
foo.Map(t => t.LastModifiedFILETIMEUTC).Ignore();
foo.Map(t => t.PathSize).Ignore();
foo.Map(t => t.Signature).Ignore();
foo.Map(t => t.ControlSet).Index(0);
foo.Map(t => t.CacheEntryPosition).Index(1);
foo.Map(t => t.Path).Index(2);
foo.Map(t => t.LastModifiedTimeUTC).Index(3);
foo.Map(t => t.Executed).Index(4);
csv.WriteHeader <CacheEntry>();
csv.NextRecord();
logger.Debug($"**** Found {appCompat.Caches.Count} caches");
if (appCompat.Caches.Any())
{
foreach (var appCompatCach in appCompat.Caches)
{
if (_fluentCommandLineParser.Object.Debug)
{
appCompatCach.PrintDump();
}
try
{
logger.Info(
$"Found {appCompatCach.Entries.Count:N0} cache entries for {appCompat.OperatingSystem} in ControlSet00{appCompatCach.ControlSet}");
if (_fluentCommandLineParser.Object.SortTimestamps)
{
csv.WriteRecords(appCompatCach.Entries.OrderByDescending(t => t.LastModifiedTimeUTC));
}
else
{
csv.WriteRecords(appCompatCach.Entries);
}
}
catch (Exception ex)
{
logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}");
try
{
appCompatCach.PrintDump();
}
catch (Exception ex1)
{
logger.Error($"Couldn't PrintDump {ex1.Message} Stack: {ex1.StackTrace}");
}
}
}
sw.Flush();
sw.Close();
logger.Warn($"\r\nResults saved to '{outFilename}'\r\n");
}
else
{
logger.Warn($"\r\nNo caches were found!\r\n");
}
}
catch (Exception ex)
{
logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}");
}
}
private static void Main(string[] args)
{
ExceptionlessClient.Default.Startup("7iL4b0Me7W8PbFflftqWgfQCIdf55flrT2O11zIP");
SetupNLog();
var logger = LogManager.GetCurrentClassLogger();
_fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>();
_fluentCommandLineParser.Setup(arg => arg.CsvDirectory)
.As("csv")
.WithDescription("Directory to save CSV formatted results to. Required")
.Required();
_fluentCommandLineParser.Setup(arg => arg.CsvName)
.As("csvf")
.WithDescription("File name to save CSV formatted results to. When present, overrides default name\r\n");
_fluentCommandLineParser.Setup(arg => arg.HiveFile)
.As('f')
.WithDescription(
"Full path to SYSTEM hive to process. If this option is not specified, the live Registry will be used")
.SetDefault(string.Empty);
_fluentCommandLineParser.Setup(arg => arg.SortTimestamps)
.As('t')
.WithDescription("Sorts last modified timestamps in descending order\r\n")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.ControlSet)
.As('c')
.WithDescription("The ControlSet to parse. Default is to extract all control sets.")
.SetDefault(-1);
_fluentCommandLineParser.Setup(arg => arg.Debug)
.As("debug")
.WithDescription("Debug mode")
.SetDefault(false);
_fluentCommandLineParser.Setup(arg => arg.DateTimeFormat)
.As("dt")
.WithDescription(
"The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss")
.SetDefault("yyyy-MM-dd HH:mm:ss");
_fluentCommandLineParser.Setup(arg => arg.NoTransLogs)
.As("nl")
.WithDescription(
"When true, ignore transaction log files for dirty hives. Default is FALSE").SetDefault(false);
var header =
$"AppCompatCache Parser version {Assembly.GetExecutingAssembly().GetName().Version}" +
$"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" +
$"\r\nhttps://github.com/EricZimmerman/AppCompatCacheParser";
var footer = @"Examples: AppCompatCacheParser.exe --csv c:\temp -t -c 2" + "\r\n\t " +
@" AppCompatCacheParser.exe --csv c:\temp --csvf results.csv" + "\r\n\t " +
"\r\n\t" +
" Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n";
_fluentCommandLineParser.SetupHelp("?", "help").WithHeader(header).Callback(text => logger.Info(text + "\r\n" + footer));
var result = _fluentCommandLineParser.Parse(args);
if (result.HelpCalled)
{
return;
}
if (result.HasErrors)
{
_fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options);
return;
}
var hiveToProcess = "Live Registry";
if (_fluentCommandLineParser.Object.HiveFile?.Length > 0)
{
hiveToProcess = _fluentCommandLineParser.Object.HiveFile;
}
logger.Info(header);
logger.Info("");
logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n");
if (IsAdministrator() == false)
{
logger.Fatal($"Warning: Administrator privileges not found!\r\n");
}
logger.Info($"Processing hive '{hiveToProcess}'");
logger.Info("");
if (_fluentCommandLineParser.Object.Debug)
{
LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug);
}
try
{
var appCompat = new AppCompatCache.AppCompatCache(_fluentCommandLineParser.Object.HiveFile,
_fluentCommandLineParser.Object.ControlSet, _fluentCommandLineParser.Object.NoTransLogs);
var outFileBase = string.Empty;
var ts1 = DateTime.Now.ToString("yyyyMMddHHmmss");
if (_fluentCommandLineParser.Object.HiveFile?.Length > 0)
{
if (_fluentCommandLineParser.Object.ControlSet >= 0)
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_ControlSet00{_fluentCommandLineParser.Object.ControlSet}_AppCompatCache.csv";
}
else
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_AppCompatCache.csv";
}
}
else
{
outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.csv";
}
if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false)
{
outFileBase = Path.GetFileName(_fluentCommandLineParser.Object.CsvName);
}
if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false)
{
Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory);
}
var outFilename = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outFileBase);
var sw = new StreamWriter(outFilename);
var csv = new CsvWriter(sw, CultureInfo.InvariantCulture);
var foo = csv.Context.AutoMap <CacheEntry>();
var o = new TypeConverterOptions
{
DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal
};
csv.Context.TypeConverterOptionsCache.AddOptions <CacheEntry>(o);
foo.Map(t => t.LastModifiedTimeUTC).Convert(t => t.Value.LastModifiedTimeUTC.HasValue ? t.Value.LastModifiedTimeUTC.Value.ToString(_fluentCommandLineParser.Object.DateTimeFormat): "");
foo.Map(t => t.CacheEntrySize).Ignore();
foo.Map(t => t.Data).Ignore();
foo.Map(t => t.InsertFlags).Ignore();
foo.Map(t => t.DataSize).Ignore();
foo.Map(t => t.LastModifiedFILETIMEUTC).Ignore();
foo.Map(t => t.PathSize).Ignore();
foo.Map(t => t.Signature).Ignore();
foo.Map(t => t.ControlSet).Index(0);
foo.Map(t => t.CacheEntryPosition).Index(1);
foo.Map(t => t.Path).Index(2);
foo.Map(t => t.LastModifiedTimeUTC).Index(3);
foo.Map(t => t.Executed).Index(4);
foo.Map(t => t.Duplicate).Index(5);
foo.Map(t => t.SourceFile).Index(6);
csv.WriteHeader <CacheEntry>();
csv.NextRecord();
logger.Debug($"**** Found {appCompat.Caches.Count} caches");
var cacheKeys = new HashSet <string>();
if (appCompat.Caches.Any())
{
foreach (var appCompatCach in appCompat.Caches)
{
if (_fluentCommandLineParser.Object.Debug)
{
appCompatCach.PrintDump();
}
try
{
logger.Info(
$"Found {appCompatCach.Entries.Count:N0} cache entries for {appCompat.OperatingSystem} in ControlSet00{appCompatCach.ControlSet}");
if (_fluentCommandLineParser.Object.SortTimestamps)
{
// csv.WriteRecords(appCompatCach.Entries.OrderByDescending(t => t.LastModifiedTimeUTC));
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csv.WriteRecord(cacheEntry);
csv.NextRecord();
}
}
else
{
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csv.WriteRecord(cacheEntry);
csv.NextRecord();
}
//csv.WriteRecords(appCompatCach.Entries);
}
}
catch (Exception ex)
{
logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}");
try
{
appCompatCach.PrintDump();
}
catch (Exception ex1)
{
logger.Error($"Couldn't PrintDump {ex1.Message} Stack: {ex1.StackTrace}");
}
}
}
sw.Flush();
sw.Close();
logger.Warn($"\r\nResults saved to '{outFilename}'\r\n");
}
else
{
logger.Warn($"\r\nNo caches were found!\r\n");
}
}
catch (Exception ex)
{
if (ex.Message.Contains("Sequence numbers do not match and transaction logs were not found in the same direct") == false)
{
if (ex.Message.Contains("Administrator privileges not found"))
{
logger.Fatal($"Could not access '{_fluentCommandLineParser.Object.HiveFile}'. Does it exist?");
logger.Error("");
logger.Fatal("Rerun the program with Administrator privileges to try again\r\n");
}
else if (ex.Message.Contains("Invalid diskName:"))
{
logger.Fatal($"Could not access '{_fluentCommandLineParser.Object.HiveFile}'. Invalid disk!");
logger.Error("");
}
else
{
logger.Error($"There was an error: {ex.Message}");
logger.Error($"Stacktrace: {ex.StackTrace}");
logger.Info("");
}
}
}
}
public async Task GenerateAsync(IEnumerable <PackageInfo> packageList, Dictionary <string, LicenseRow> cachedLicenses, SpdxLicenseData spdxLicenseData)
{
var licenses = (await ToLicenseRowsAsync(packageList, cachedLicenses, spdxLicenseData)).OrderBy(p => p.Value.Id);
switch (_options.FileType)
{
case FileType.Csv:
using (var write = new StreamWriter(_options.Path))
{
using (var helper = new CsvHelper.CsvWriter(write))
{
helper.Configuration.SanitizeForInjection = true;
helper.Configuration.QuoteAllFields = true;
//Write headers
foreach (var property in _options.Columns)
{
helper.WriteField(property, true);
}
helper.NextRecord();
var properties = typeof(LicenseRow).GetProperties().ToList();
//Write values
foreach (var license in licenses)
{
//Get rid of new lines as they are bad in CSV and other report layouts.
license.Value.LicenseText = license.Value?.LicenseText.Replace("\r", "").Replace("\n", "").Replace(",", " ");
//
if (license.Value?.LicenseText?.Length >= 35000)
{
license.Value.LicenseText = license.Value.LicenseText.Substring(0, 35000);
}
foreach (var propertyName in _options.Columns)
{
var property = properties.FirstOrDefault(p => p.Name == propertyName);
var value = property?.GetValue(license.Value, null);
helper.WriteField(value?.ToString());
}
helper.NextRecord();
}
}
}
break;
case FileType.Html:
throw new NotImplementedException();
case FileType.Pdf:
throw new NotImplementedException();
case FileType.Word:
throw new NotImplementedException();
default:
throw new ArgumentOutOfRangeException();
}
var proc = new Process();
var finfo = new FileInfo(_options.Path);
if (finfo.Exists)
{
proc.StartInfo.FileName = finfo.FullName;
proc.Start();
}
}
/// <summary>
///
/// </summary>
/// <param name="dataDirectory"></param>
/// <param name="outputDirectory"></param>
public void PostProcess(string dataDirectory, string outputDirectory)
{
CsvConfiguration csvConfiguration = new CsvConfiguration();
csvConfiguration.QuoteAllFields = true;
using (FileStream fileStream = new FileStream(System.IO.Path.Combine(outputDirectory, "Urls.csv"), FileMode.Append, FileAccess.Write, FileShare.Read))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration))
{
foreach (string file in System.IO.Directory.EnumerateFiles(outputDirectory,
"*.xml",
SearchOption.AllDirectories))
{
string fileName = System.IO.Path.GetFileName(file);
if (fileName.StartsWith("Url.Details.") == false)
{
continue;
}
UrlDetails urlDetails = new UrlDetails();
string ret = urlDetails.Load(file);
if (ret.Length == 0)
{
foreach (string url in urlDetails.Urls)
{
csvWriter.WriteField(url);
csvWriter.WriteField(urlDetails.SrcIp);
csvWriter.WriteField(urlDetails.SrcPort);
csvWriter.WriteField(urlDetails.DstIp);
csvWriter.WriteField(urlDetails.DstPort);
csvWriter.NextRecord();
}
}
}
}
}
public bool CreateCSvFile(Ijpk jpk)
{
string jpkPath = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "JPK", "jpk.csv");
if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(jpkPath)))
{
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(jpkPath));
}
try
{
using (System.IO.TextWriter writeFile = new System.IO.StreamWriter(jpkPath))
{
CsvHelper.CsvWriter csv = new CsvHelper.CsvWriter(writeFile);
csv.Configuration.Delimiter = ";";
var fields = header.Split(";".ToCharArray());
foreach (string field in fields)
{
csv.WriteField(field);
}
csv.NextRecord();
csv.WriteField(jpk.KodFormularza);
csv.WriteField(jpk.kodSystemowy);
csv.WriteField(jpk.wersjaSchemy);
csv.WriteField(jpk.WariantFormularza);
csv.WriteField(jpk.CelZlozenia);
csv.WriteField(jpk.DataWytworzeniaJPK.ToJPKDataUtworzenia());
csv.WriteField(jpk.DataOd.ToJPK());
csv.WriteField(jpk.DataDo.ToJPK());
csv.WriteField("My JPK");
//csv.WriteField(jpk.KodUrzedu);
for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("NazwaSystemu"); i++)
{
csv.WriteField("");
}
csv.NextRecord();
for (int i = 0; i < fields.ToList().IndexOf("NIP"); i++)
{
csv.WriteField("");
}
csv.WriteField(jpk.NIP);
csv.WriteField(jpk.PelnaNazwa);
csv.WriteField(jpk.REGON);
for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("Email"); i++)
{
csv.WriteField("");
}
csv.NextRecord();
//for (int i = 0; i < fields.ToList().IndexOf("KodKraju"); i++) csv.WriteField("");
//csv.WriteField(jpk.KodKraju);
//csv.WriteField(jpk.Wojewodztwo);
//csv.WriteField(jpk.Powiat);
//csv.WriteField(jpk.Gmina);
//csv.WriteField(jpk.Ulica);
//csv.WriteField(jpk.NrDomu);
//csv.WriteField(jpk.NrLokalu);
//csv.WriteField(jpk.Miejscowosc);
//csv.WriteField(jpk.KodPocztowy);
//csv.WriteField(jpk.Poczta);
//for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("Poczta"); i++) csv.WriteField("");
//csv.NextRecord();
FkRejSprzedazy(jpk, csv);
FkRejZakupow(jpk, csv);
}
jpk.JPKMessage = (new StringBuilder()).AppendLine("Wygenerowano JPK csv").AppendLine(jpkPath).ToString();
return(true);
}
catch (Exception exp)
{
jpk.JPKMessage = exp.Message;
}
return(false);
}
private static void ProcessFile(string file)
{
if (File.Exists(file) == false)
{
_logger.Warn($"'{file}' does not exist! Skipping");
return;
}
_logger.Warn($"\r\nProcessing '{file}'...");
Stream fileS;
try
{
fileS = new FileStream(file, FileMode.Open, FileAccess.Read);
}
catch (Exception)
{
//file is in use
if (Helper.IsAdministrator() == false)
{
_logger.Fatal("\r\nAdministrator privileges not found! Exiting!!\r\n");
Environment.Exit(0);
}
_logger.Warn($"\r\n'{file}' is in use. Rerouting...");
var files = new List <string>();
files.Add(file);
var rawFiles = Helper.GetFiles(files);
fileS = rawFiles.First().FileStream;
}
try
{
var evt = new EventLog(fileS);
var seenRecords = 0;
foreach (var eventRecord in evt.GetEventRecords())
{
if (_includeIds.Count > 0)
{
if (_includeIds.Contains(eventRecord.EventId) == false)
{
//it is NOT in the list, so skip
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains(eventRecord.EventId))
{
//it IS in the list, so skip
continue;
}
}
eventRecord.SourceFile = file;
try
{
_csvWriter?.WriteRecord(eventRecord);
_csvWriter?.NextRecord();
var xml = string.Empty;
if (_swXml != null)
{
xml = eventRecord.ConvertPayloadToXml();
_swXml.WriteLine(xml);
}
if (_swJson != null)
{
JsConfig.IncludeNullValues = true;
JsConfig.DateHandler = DateHandler.ISO8601;
var jsOut = eventRecord.ToJson();
if (_fluentCommandLineParser.Object.FullJson)
{
if (xml.IsNullOrEmpty())
{
xml = eventRecord.ConvertPayloadToXml();
}
var xd = new XmlDocument();
xd.LoadXml(xml);
jsOut = JsonConvert.SerializeXmlNode(xd);
}
_swJson.WriteLine(jsOut);
}
seenRecords += 1;
}
catch (Exception e)
{
_logger.Error($"Error processing record #{eventRecord.RecordNumber}: {e.Message}");
}
}
if (evt.ErrorRecords.Count > 0)
{
_errorFiles.Add(file, evt.ErrorRecords.Count);
}
_fileCount += 1;
_logger.Info("");
_logger.Fatal("Event log details");
_logger.Info(evt);
_logger.Info($"Records processed: {seenRecords:N0} Errors: {evt.ErrorRecords.Count:N0}");
if (evt.ErrorRecords.Count > 0)
{
_logger.Warn("\r\nErrors");
foreach (var evtErrorRecord in evt.ErrorRecords)
{
_logger.Info($"Record #{evtErrorRecord.Key}: Error: {evtErrorRecord.Value}");
}
}
if (_fluentCommandLineParser.Object.Metrics && evt.EventIdMetrics.Count > 0)
{
_logger.Fatal("\r\nMetrics");
_logger.Warn("Event Id\tCount");
foreach (var esEventIdMetric in evt.EventIdMetrics.OrderBy(t => t.Key))
{
if (_includeIds.Count > 0)
{
if (_includeIds.Contains((int)esEventIdMetric.Key) == false)
{
//it is NOT in the list, so skip
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains((int)esEventIdMetric.Key))
{
//it IS in the list, so skip
continue;
}
}
_logger.Info($"{esEventIdMetric.Key}\t\t{esEventIdMetric.Value:N0}");
}
}
}
catch (Exception e)
{
if (e.Message.Contains("Invalid signature! Expected 'ElfFile"))
{
_logger.Info($"'{file}' is not an evtx file! Message: {e.Message} Skipping...");
}
else
{
_logger.Error($"Error processing '{file}'! Message: {e.Message}");
}
}
fileS?.Close();
}
public static void WriteCSV(TextWriter writer, IEnumerable <WorkorderSummary> ws)
{
var csv = new CsvHelper.CsvWriter(writer);
csv.WriteField("ID");
csv.WriteField("CompletedTimeUTC");
csv.WriteField("Part");
csv.WriteField("Quantity");
csv.WriteField("Serials");
var activeStations = new HashSet <string>();
var elapsedStations = new HashSet <string>();
foreach (var p in ws.SelectMany(w => w.Parts))
{
foreach (var k in p.ActiveStationTime.Keys)
{
activeStations.Add(k);
}
foreach (var k in p.ElapsedStationTime.Keys)
{
elapsedStations.Add(k);
}
}
var actualKeys = activeStations.OrderBy(x => x).ToList();
foreach (var k in actualKeys)
{
csv.WriteField("Active " + k + " (minutes)");
}
var plannedKeys = elapsedStations.OrderBy(x => x).ToList();
foreach (var k in plannedKeys)
{
csv.WriteField("Elapsed " + k + " (minutes)");
}
csv.NextRecord();
foreach (var w in ws)
{
foreach (var p in w.Parts)
{
csv.WriteField(w.WorkorderId);
if (w.FinalizedTimeUTC.HasValue)
{
csv.WriteField(w.FinalizedTimeUTC.Value.ToString("yyyy-MM-ddTHH:mm:ssZ"));
}
else
{
csv.WriteField("");
}
csv.WriteField(p.Part);
csv.WriteField(p.PartsCompleted);
csv.WriteField(string.Join(";", w.Serials));
foreach (var k in actualKeys)
{
if (p.ActiveStationTime.ContainsKey(k))
{
csv.WriteField(p.ActiveStationTime[k].TotalMinutes);
}
else
{
csv.WriteField(0);
}
}
foreach (var k in plannedKeys)
{
if (p.ElapsedStationTime.ContainsKey(k))
{
csv.WriteField(p.ElapsedStationTime[k].TotalMinutes);
}
else
{
csv.WriteField(0);
}
}
csv.NextRecord();
}
}
}
/// <summary>
/// Uploads book cover images to ipfs
/// Writes a mapping csv file to relate ean, image size, ipfs hash etc
/// Assumes the book cover images are held locally so....
/// Ensure the cover images have been downloaded first! (DownloadCoversAsync)
/// </summary>
/// <param name="books"></param>
/// <returns></returns>
public static async Task UploadCoversToIpfsAsync(
IEnumerable <Book> books, string coverFolder, string indexOutputFilePath, string ipfsUrl)
{
ipfsFilesLoaded = 0;
//indexOutputFilePath
try
{
// we'll parallel load images - but don't want to write to the csv in parallel
var semaphore = new SemaphoreSlim(1);
using (var textWriter = File.CreateText(indexOutputFilePath))
{
var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture);
csvWriter.WriteHeader <IpfsBookCoverMapping>();
csvWriter.NextRecord();
try
{
var ipfsService = new IpfsService(ipfsUrl);
foreach (var book in books.AsParallel())
{
if (!string.IsNullOrWhiteSpace(book.COVER_SMALL))
{
var smallCover = await UploadCoverToIpfs(coverFolder, book.EAN, "s", book.COVER_SMALL, ipfsService);
await Save(smallCover);
}
if (!string.IsNullOrWhiteSpace(book.COVER_MED))
{
var mediumCover = await UploadCoverToIpfs(coverFolder, book.EAN, "m", book.COVER_MED, ipfsService);
await Save(mediumCover);
}
if (!string.IsNullOrWhiteSpace(book.COVER_LARGE))
{
var largeCover = await UploadCoverToIpfs(coverFolder, book.EAN, "l", book.COVER_LARGE, ipfsService);
await Save(largeCover);
}
async Task Save(IpfsBookCoverMapping mapping)
{
await semaphore.WaitAsync();
try
{
csvWriter.WriteRecord(mapping);
csvWriter.NextRecord();
}
finally
{
semaphore.Release();
}
}
}
}
finally
{
await csvWriter.FlushAsync();
await textWriter.FlushAsync();
}
}
}
finally
{
// consider changing file name on completion to avoid it being overwritten accidentally
}
}
public void MarkWorkorderAsFilled(string workorderId,
DateTime filledUTC,
WorkorderResources resources)
{
if (!Directory.Exists(Path.Combine(CSVBasePath, FilledWorkordersPath)))
{
Directory.CreateDirectory(Path.Combine(CSVBasePath, FilledWorkordersPath));
}
using (var f = File.OpenWrite(Path.Combine(CSVBasePath, Path.Combine(FilledWorkordersPath, workorderId + ".csv"))))
using (var stream = new StreamWriter(f))
using (var csv = new CsvHelper.CsvWriter(stream))
{
csv.WriteField("CompletedTimeUTC");
csv.WriteField("ID");
csv.WriteField("Part");
csv.WriteField("Quantity");
csv.WriteField("Serials");
var activeStations = new HashSet <string>();
var elapsedStations = new HashSet <string>();
foreach (var p in resources.Parts)
{
foreach (var k in p.ActiveOperationTime.Keys)
{
activeStations.Add(k);
}
foreach (var k in p.ElapsedOperationTime.Keys)
{
elapsedStations.Add(k);
}
}
var actualKeys = activeStations.OrderBy(x => x).ToList();
foreach (var k in actualKeys)
{
csv.WriteField("Active " + k + " (minutes)");
}
var plannedKeys = elapsedStations.OrderBy(x => x).ToList();
foreach (var k in plannedKeys)
{
csv.WriteField("Elapsed " + k + " (minutes)");
}
csv.NextRecord();
foreach (var p in resources.Parts)
{
csv.WriteField(filledUTC.ToString("yyyy-MM-ddTHH:mm:ssZ"));
csv.WriteField(workorderId);
csv.WriteField(p.Part);
csv.WriteField(p.PartsCompleted);
csv.WriteField(string.Join(";", resources.Serials));
foreach (var k in actualKeys)
{
if (p.ActiveOperationTime.ContainsKey(k))
{
csv.WriteField(p.ActiveOperationTime[k].TotalMinutes);
}
else
{
csv.WriteField(0);
}
}
foreach (var k in plannedKeys)
{
if (p.ElapsedOperationTime.ContainsKey(k))
{
csv.WriteField(p.ElapsedOperationTime[k].TotalMinutes);
}
else
{
csv.WriteField(0);
}
}
csv.NextRecord();
}
}
}