/// <summary> /// Exports all customers for the reseller to the specified file. /// </summary> /// <param name="defaultDomain">default domain of the reseller</param> /// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param> /// <param name="key">Key for this application in Azure Active Directory</param> /// <param name="customerMicrosoftId">Microsoft Id of the customer</param> /// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param> public static void ExportCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId) { GetTokens(defaultDomain, appId, key, resellerMicrosoftId); var adCustomers = Customer.GetAllCustomers(resellerMicrosoftId, adAuthorizationToken.AccessToken); using (StreamWriter outputFile = new StreamWriter(filename)) using (var writer = new CsvHelper.CsvWriter(outputFile)) { writer.WriteField("displayName"); writer.WriteField("defaultDomainName"); writer.WriteField("cspCustomerCid"); writer.WriteField("customerContextId"); writer.NextRecord(); foreach (AzureADContract adCustomer in adCustomers) { adCustomer.cspCustomerCid = Customer.GetCustomerCid(adCustomer.customerContextId.ToString(), resellerMicrosoftId, saAuthorizationToken.AccessToken); writer.WriteField(adCustomer.displayName); writer.WriteField(adCustomer.defaultDomainName); writer.WriteField(adCustomer.cspCustomerCid); writer.WriteField(adCustomer.customerContextId); writer.NextRecord(); } } }
public void Write(IEnumerable<CsvModel> models, TextWriter writer, string delimiter) { /* get all headers row */ List<string> headers = new List<string>(); headers.Add(CsvHeaders.ObjectType); headers.Add(CsvHeaders.Class); headers.Add(CsvHeaders.Source); foreach (var model in models) { foreach (string key in model.Values.Keys) if (!headers.Contains(key)) headers.Add(key); } using (var csvWriter = new CsvHelper.CsvWriter(writer, new CsvHelper.Configuration.CsvConfiguration() { QuoteAllFields = true, Delimiter = delimiter } )) { /* write the header row */ foreach (string header in headers) csvWriter.WriteField<string>(header); csvWriter.NextRecord(); /* write the content of each csv model */ foreach (var model in models) { for (int i = 0; i < headers.Count; i++) { string value = string.Empty; switch (i) { case 0: value = model.ObjectType; break; case 1: value = model.Class; break; case 2: value = string.Join(new string (new char[] {Constants.SourceFileDelimiter }), model.SourceFiles); break; default: if (!model.Values.TryGetValue(headers[i], out value)) value = string.Empty; break; } csvWriter.WriteField<string>(value); } csvWriter.NextRecord(); } } }
internal string createCsvWithStructure(IEnumerable<dynamic> data, CsvConfiguration configuration) { var guid = Guid.NewGuid().ToString(); var builder = new StringBuilder(); var writer = new CsvHelper.CsvWriter(new StringWriter(builder), configuration); foreach (var column in Structure) writer.WriteField(column.Name); writer.NextRecord(); foreach (var datum in data) { var dict = ToDictionary(guid, datum); foreach (var column in Structure) { var value = dict[column.Name]; var str = string.IsNullOrEmpty(column.Format) ? value.ToString() : value.ToString(column.Format); writer.WriteField(str); } writer.NextRecord(); } return builder.ToString(); }
protected virtual void WriteHeader(ICsvReportConfiguration config, CsvHelper.CsvWriter writer) { if (config.Options.IsHideHeader) { return; } var memberOptions = config.MemberConfigurations.GetActiveOrderlyOptions(); foreach (var options in memberOptions) { var header = options.GetHeader(config.Options.UseHeaderHumanizer); writer.WriteField(header); } writer.NextRecord(); }
public void WriteComment(params string[] comments) { if (comments == null || comments.Length == 0) { return; } var configuration = GetConfiguration(); using (var writer = new CsvHelper.CsvWriter(TextWriter, configuration)) { foreach (var comment in comments) { writer.WriteComment(comment); writer.NextRecord(); } } }
public JsonResult _SubmiteBayEndProductListing(MarketplaceFeed model) { try { var productItems = _productService.GeteBayItemFeeds(model).ToList(); var fileFullPath = string.Format("{0}\\eBayProductEndListing_{1:yyyyMMdd_HHmmss}.csv", _systemJobsRoot, DateTime.Now); using (var streamWriter = new StreamWriter(fileFullPath)) { var writer = new CsvHelper.CsvWriter(streamWriter); foreach (var item in productItems) { writer.WriteField(item.EisSKU); writer.WriteField(item.ItemId); writer.NextRecord(); } } // create new job for eBay suggested categories var systemJob = new SystemJobDto { JobType = JobType.eBayProductsEndItem, Parameters = fileFullPath, SupportiveParameters = model.Mode, SubmittedBy = User.Identity.Name }; var jobId = _systemJobService.CreateSystemJob(systemJob); return(Json(new { Success = "eBay EndItem has been passed to EIS System Jobs for execution.", JobId = jobId }, JsonRequestBehavior.AllowGet)); } catch (Exception ex) { return(Json(new { Error = "Error in submitting eBay EndItem! - Message: " + EisHelper.GetExceptionMessage(ex) }, JsonRequestBehavior.AllowGet)); } }
private void WriteHeader(CsvHelper.CsvWriter csv) { if (!string.IsNullOrEmpty(CustomIdColumn)) { csv.WriteField("Custom Id"); } csv.WriteField("Id"); csv.WriteField("Source"); csv.WriteField("Target"); csv.WriteField("State"); if (IsLangColumnRequired) { csv.WriteField("Language"); } csv.NextRecord(); }
public HttpResponseMessage DownloadCsv(GenerateCsvModel model) { // Variables. var start = model.StartDate.HasValue ? model.StartDate.Value : SqlDateTime.MinValue.Value; var stop = model.EndDate.HasValue ? model.EndDate.Value.AddDays(1) : SqlDateTime.MaxValue.Value; // Get history items. var items = HistoryHelper.GetHistoryItems(start, stop); var config = new CsvHelper.Configuration.CsvConfiguration() { HasHeaderRecord = true, QuoteAllFields = true }; // write CSV data to memory stream using (var stream = new MemoryStream()) { using (var writer = new StreamWriter(stream, Encoding.UTF8, 1024, true)) { using (var csvWriter = new CsvHelper.CsvWriter(writer, config)) { csvWriter.WriteHeader <HistoryCsvHeader>(); foreach (var item in items) { csvWriter.WriteField(item.Timestamp.ToString()); csvWriter.WriteField(item.Message); csvWriter.NextRecord(); } } } stream.Seek(0, SeekOrigin.Begin); return(CreateFileResponse( stream.ToArray(), $"site-activity_{start.ToString("yyyy-MM-dd")}_{stop.ToString("yyyy-MM-dd")}.csv", "text/csv" )); } }
public JsonResult _GeteBayBulkSuggestedCategories(MarketplaceFeed model) { try { // get the product feed for bul eBay suggested categories var productKeywordFeeds = _productService.GeteBaySuggestedCategoryFeed(model); var fileFullPath = string.Format("{0}\\BulkeBaySuggestedCategories_{1:yyyyMMdd_HHmmss}.csv", _systemJobsRoot, DateTime.Now); using (var streamWriter = new StreamWriter(fileFullPath)) { var writer = new CsvHelper.CsvWriter(streamWriter); foreach (var item in productKeywordFeeds) { writer.WriteField(item.EisSKU); writer.WriteField(item.Keyword); writer.NextRecord(); } } // create new job for eBay suggested categories var systemJob = new SystemJobDto { JobType = JobType.BulkeBaySuggestedCategories, Parameters = fileFullPath, SubmittedBy = User.Identity.Name }; var jobId = _systemJobService.CreateSystemJob(systemJob); return(Json(new { Success = "Bulk eBay get suggested categories has been passed to EIS System Jobs for execution.", JobId = jobId }, JsonRequestBehavior.AllowGet)); } catch (Exception ex) { return(Json(new { Error = "Error in getting bulk eBay suggested categories! - Message: " + EisHelper.GetExceptionMessage(ex) }, JsonRequestBehavior.AllowGet)); } }
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer) { var szTrCode = resModel.Name; var block = resModel.Blocks[szTrCode + "OutBlock"]; writer.WriteHeader <_t8425OutBlock>(); for (var i = 0; i < query.GetBlockCount(block.Name); i++) { var result = new _t8425OutBlock() { tmcode = query.GetFieldData(block.Name, "tmcode", i), tmname = query.GetFieldData(block.Name, "tmname", i), }; writer.NextRecord(); writer.WriteRecord(result); Constants.CodeThemes.Add(result.tmcode, new CodeTheme() { Code = result.tmcode, Name = result.tmname, }); } }
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer) { var szTrCode = resModel.Name; var block = resModel.Blocks[szTrCode + "OutBlock"]; writer.WriteHeader <_t8436OutBlock>(); for (var i = 0; i < query.GetBlockCount(block.Name); i++) { var result = new _t8436OutBlock() { hname = query.GetFieldData(block.Name, "hname", i), shcode = query.GetFieldData(block.Name, "shcode", i), expcode = query.GetFieldData(block.Name, "expcode", i), etfgubun = query.GetFieldData(block.Name, "etfgubun", i), uplmtprice = long.Parse(query.GetFieldData(block.Name, "uplmtprice", i)), dnlmtprice = long.Parse(query.GetFieldData(block.Name, "dnlmtprice", i)), jnilclose = long.Parse(query.GetFieldData(block.Name, "jnilclose", i)), memedan = query.GetFieldData(block.Name, "memedan", i), recprice = long.Parse(query.GetFieldData(block.Name, "recprice", i)), gubun = query.GetFieldData(block.Name, "gubun", i), bu12gubun = query.GetFieldData(block.Name, "bu12gubun", i), spac_gubun = query.GetFieldData(block.Name, "spac_gubun", i), filler = query.GetFieldData(block.Name, "filler", i), }; writer.NextRecord(); writer.WriteRecord(result); Constants.CodeStocks.Add(result.shcode, new CodeStock() { Name = result.hname, Code = result.shcode, ExpandedCode = result.expcode, EtfGubun = result.etfgubun, SpacGubun = result.spac_gubun, MarketGubun = result.gubun, Bu12Gubun = result.bu12gubun, }); } }
/// <summary> /// execute the <see cref="ExportCommand"/> /// </summary> private void ExecuteExportCommand() { var openSaveFileDialogService = ServiceLocator.Current.GetInstance <IOpenSaveFileDialogService>(); var result = openSaveFileDialogService.GetSaveFileDialog("Untitled", ".csv", "CSV File (.csv)|*.csv", "", 0); if (string.IsNullOrEmpty(result)) { return; } using (TextWriter writer = File.CreateText(result)) { var csv = new CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture); foreach (var logInfoRowViewModel in this.LogEventInfo) { csv.WriteField(logInfoRowViewModel.TimeStamp); csv.WriteField(logInfoRowViewModel.Logger); csv.WriteField(logInfoRowViewModel.LogLevel.Name); csv.WriteField(logInfoRowViewModel.Message); csv.NextRecord(); } } }
protected virtual void WriteRecords(IEnumerable source, ICsvReportConfiguration config, TypeAccessor.TypeAccessor accessor, CsvHelper.CsvWriter writer) { foreach (var row in source) { var memberOptions = config.MemberConfigurations.GetActiveOrderlyOptions(); foreach (var column in memberOptions) { var value = accessor.GetValue(row, column.MemberName); string formattedValue = value.ToString(); if (column.Formatter != null) { formattedValue = column.Formatter.Format(value); } else if (column.FormatterExpression != null) { formattedValue = ReportHelper.ReportHelper.CreateFormatFunc(value, column.FormatterExpression)(value); } writer.WriteField(formattedValue); } writer.NextRecord(); } }
public JsonResult _DeleteBulkVendorProducts(VendorProductFilterDto model) { try { // get product's EIS SKU var eisSupplierSKUs = _vendorProductService.GetVendorProductsEisSupplierSKUs(model); var fileFullPath = string.Format("{0}\\BulkDeleteVendorProducts_{1:yyyyMMdd_HHmmss}.txt", _systemJobsRoot, DateTime.Now); using (var streamWriter = new StreamWriter(fileFullPath)) { var writer = new CsvHelper.CsvWriter(streamWriter); foreach (var sku in eisSupplierSKUs) { // just write the vendor product's Id which we need to delete writer.WriteField(sku); writer.NextRecord(); } } // create new job for the Amazon Get Info for the asins var systemJob = new SystemJobDto { JobType = JobType.BulkDeleteVendorProduct, Parameters = fileFullPath, SubmittedBy = User.Identity.Name }; _systemJobService.CreateSystemJob(systemJob); return(Json("Bulk deletion of vendor products has been passed to EIS System Jobs for execution.", JsonRequestBehavior.AllowGet)); } catch (Exception ex) { return(Json(new { Error = "Error in deleting vendor products! - Message: " + EisHelper.GetExceptionMessage(ex) }, JsonRequestBehavior.AllowGet)); } }
public static void ExportToCsv(List <List <string> > data) { var pathToFile = GetPathToFile(); using (var stream = File.OpenWrite(pathToFile)) { using (var streamWriter = new StreamWriter(stream, Encoding.UTF8)) { using (var csvWriter = new CsvHelper.CsvWriter(streamWriter, CultureInfo.InvariantCulture)) { foreach (var row in data) { foreach (var i in row) { csvWriter.WriteField(i); } csvWriter.NextRecord(); } } } } Process.Start(pathToFile); }
private static void ProcessFile(string file) { if (File.Exists(file) == false) { _logger.Warn($"'{file}' does not exist! Skipping"); return; } if (file.StartsWith(VssDir)) { _logger.Warn($"\r\nProcessing 'VSS{file.Replace($"{VssDir}\\", "")}'"); } else { _logger.Warn($"\r\nProcessing '{file}'..."); } Stream fileS; try { fileS = new FileStream(file, FileMode.Open, FileAccess.Read); } catch (Exception) { //file is in use if (Helper.IsAdministrator() == false) { _logger.Fatal("\r\nAdministrator privileges not found! Exiting!!\r\n"); Environment.Exit(0); } _logger.Warn($"\r\n'{file}' is in use. Rerouting..."); var files = new List <string>(); files.Add(file); var rawFiles = Helper.GetFiles(files); fileS = rawFiles.First().FileStream; } try { if (_fluentCommandLineParser.Object.Dedupe) { var sha = Helper.GetSha1FromStream(fileS, 0); if (_seenHashes.Contains(sha)) { _logger.Debug($"Skipping '{file}' as a file with SHA-1 '{sha}' has already been processed"); return; } _seenHashes.Add(sha); } var evt = new EventLog(fileS); var seenRecords = 0; foreach (var eventRecord in evt.GetEventRecords()) { if (_includeIds.Count > 0) { if (_includeIds.Contains(eventRecord.EventId) == false) { //it is NOT in the list, so skip _droppedEvents += 1; continue; } } else if (_excludeIds.Count > 0) { if (_excludeIds.Contains(eventRecord.EventId)) { //it IS in the list, so skip _droppedEvents += 1; continue; } } if (_startDate.HasValue) { if (eventRecord.TimeCreated < _startDate.Value) { //too old _logger.Debug($"Dropping record Id '{eventRecord.EventRecordId}' with timestamp '{eventRecord.TimeCreated.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}' as its too old."); _droppedEvents += 1; continue; } } if (_endDate.HasValue) { if (eventRecord.TimeCreated > _endDate.Value) { //too new _logger.Debug($"Dropping record Id '{eventRecord.EventRecordId}' with timestamp '{eventRecord.TimeCreated.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}' as its too new."); _droppedEvents += 1; continue; } } if (file.StartsWith(VssDir)) { eventRecord.SourceFile = $"VSS{file.Replace($"{VssDir}\\", "")}"; } else { eventRecord.SourceFile = file; } try { if (_fluentCommandLineParser.Object.PayloadAsJson) { var xdo = new XmlDocument(); xdo.LoadXml(eventRecord.Payload); var payOut = JsonConvert.SerializeXmlNode(xdo); eventRecord.Payload = payOut; } _csvWriter?.WriteRecord(eventRecord); _csvWriter?.NextRecord(); var xml = string.Empty; if (_swXml != null) { xml = eventRecord.ConvertPayloadToXml(); _swXml.WriteLine(xml); } if (_swJson != null) { var jsOut = eventRecord.ToJson(); if (_fluentCommandLineParser.Object.FullJson) { if (xml.IsNullOrEmpty()) { xml = eventRecord.ConvertPayloadToXml(); } jsOut = GetPayloadAsJson(xml); } _swJson.WriteLine(jsOut); } seenRecords += 1; } catch (Exception e) { _logger.Error($"Error processing record #{eventRecord.RecordNumber}: {e.Message}"); evt.ErrorRecords.Add(21, e.Message); } } if (evt.ErrorRecords.Count > 0) { var fn = file; if (file.StartsWith(VssDir)) { fn = $"VSS{file.Replace($"{VssDir}\\", "")}"; } _errorFiles.Add(fn, evt.ErrorRecords.Count); } _fileCount += 1; _logger.Info(""); _logger.Fatal("Event log details"); _logger.Info(evt); _logger.Info($"Records included: {seenRecords:N0} Errors: {evt.ErrorRecords.Count:N0} Events dropped: {_droppedEvents:N0}"); if (evt.ErrorRecords.Count > 0) { _logger.Warn("\r\nErrors"); foreach (var evtErrorRecord in evt.ErrorRecords) { _logger.Info($"Record #{evtErrorRecord.Key}: Error: {evtErrorRecord.Value}"); } } if (_fluentCommandLineParser.Object.Metrics && evt.EventIdMetrics.Count > 0) { _logger.Fatal("\r\nMetrics (including dropped events)"); _logger.Warn("Event Id\tCount"); foreach (var esEventIdMetric in evt.EventIdMetrics.OrderBy(t => t.Key)) { if (_includeIds.Count > 0) { if (_includeIds.Contains((int)esEventIdMetric.Key) == false) { //it is NOT in the list, so skip continue; } } else if (_excludeIds.Count > 0) { if (_excludeIds.Contains((int)esEventIdMetric.Key)) { //it IS in the list, so skip continue; } } _logger.Info($"{esEventIdMetric.Key}\t\t{esEventIdMetric.Value:N0}"); } } } catch (Exception e) { var fn = file; if (file.StartsWith(VssDir)) { fn = $"VSS{file.Replace($"{VssDir}\\", "")}"; } if (e.Message.Contains("Invalid signature! Expected 'ElfFile")) { _logger.Info($"'{fn}' is not an evtx file! Message: {e.Message} Skipping..."); } else { _logger.Error($"Error processing '{fn}'! Message: {e.Message}"); } } fileS?.Close(); }
protected override void OutBlock(ResModel resModel, IXAQuery query, CsvHelper.CsvWriter writer) { var szTrCode = resModel.Name; if (query.GetBlockCount(szTrCode + "OutBlock1") == 0) { _codeIdx++; var code = codes.ElementAt(_codeIdx); InBlock(code); return; } var block = resModel.Blocks[szTrCode + "OutBlock"]; var meta = new _t4203OutBlock() { shcode = query.GetFieldData(block.Name, "shcode", 0), jisiga = decimal.Parse(query.GetFieldData(block.Name, "jisiga", 0)), jihigh = decimal.Parse(query.GetFieldData(block.Name, "jihigh", 0)), jilow = decimal.Parse(query.GetFieldData(block.Name, "jilow", 0)), jiclose = decimal.Parse(query.GetFieldData(block.Name, "jiclose", 0)), jivolume = long.Parse(query.GetFieldData(block.Name, "jivolume", 0)), disiga = decimal.Parse(query.GetFieldData(block.Name, "disiga", 0)), dihigh = decimal.Parse(query.GetFieldData(block.Name, "dihigh", 0)), dilow = decimal.Parse(query.GetFieldData(block.Name, "dilow", 0)), diclose = decimal.Parse(query.GetFieldData(block.Name, "diclose", 0)), disvalue = long.Parse(query.GetFieldData(block.Name, "disvalue", 0)), cts_date = query.GetFieldData(block.Name, "cts_date", 0), cts_time = query.GetFieldData(block.Name, "cts_time", 0), cts_daygb = query.GetFieldData(block.Name, "cts_daygb", 0), }; var block1 = resModel.Blocks[szTrCode + "OutBlock1"]; writer.WriteHeader <_t4203OutBlock1>(); for (var i = 0; i < query.GetBlockCount(block1.Name); i++) { var result = new _t4203OutBlock1() { date = query.GetFieldData(block1.Name, "date", i), time = query.GetFieldData(block1.Name, "time", i), open = decimal.Parse(query.GetFieldData(block1.Name, "open", i)), high = decimal.Parse(query.GetFieldData(block1.Name, "high", i)), low = decimal.Parse(query.GetFieldData(block1.Name, "low", i)), close = decimal.Parse(query.GetFieldData(block1.Name, "close", i)), jdiff_vol = long.Parse(query.GetFieldData(block1.Name, "jdiff_vol", i)), value = long.Parse(query.GetFieldData(block1.Name, "value", i)), }; writer.NextRecord(); writer.WriteRecord(result); } Thread.Sleep(1000); if (_query.IsNext) { _inBlock.cts_date = meta.cts_date; _inBlock.cts_time = meta.cts_time; _inBlock.cts_daygb = meta.cts_daygb; InBlock(meta.shcode, _query.IsNext); } else { _codeIdx++; if (codes.Count > _codeIdx) { var code = codes.ElementAt(_codeIdx); InBlock(code); } else { //TODO next callback hell } } }
private static void Main(string[] args) { ExceptionlessClient.Default.Startup("Wdlq68AwLteBtuqOwNv5rgphcMxzKuHKJQAVK5JN"); SetupNLog(); _logger = LogManager.GetCurrentClassLogger(); _fluentCommandLineParser = new FluentCommandLineParser <AppArgs> { IsCaseSensitive = false }; _fluentCommandLineParser.Setup(arg => arg.File) .As('f') .WithDescription("File to process. Required"); _fluentCommandLineParser.Setup(arg => arg.CsvDirectory) .As("csv") .WithDescription( "Directory to save CSV (tab separated) formatted results to. Be sure to include the full path in double quotes"); _fluentCommandLineParser.Setup(arg => arg.JsonDirectory) .As("json") .WithDescription( "Directory to save json representation to. Use --pretty for a more human readable layout"); _fluentCommandLineParser.Setup(arg => arg.JsonPretty) .As("pretty") .WithDescription( "When exporting to json, use a more human readable layout\r\n").SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.Quiet) .As('q') .WithDescription( "Only show the filename being processed vs all output. Useful to speed up exporting to json and/or csv\r\n") .SetDefault(false); var header = $"RecentFileCacheParser version {Assembly.GetExecutingAssembly().GetName().Version}" + "\r\n\r\nAuthor: Eric Zimmerman ([email protected])" + "\r\nhttps://github.com/EricZimmerman/RecentFileCacheParser"; var footer = @"Examples: RecentFileCacheParser.exe -f ""C:\Temp\RecentFileCache.bcf"" --csv ""c:\temp""" + "\r\n\t " + @" RecentFileCacheParser.exe -f ""C:\Temp\RecentFileCache.bcf"" --json ""D:\jsonOutput"" --jsonpretty" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help") .WithHeader(header) .Callback(text => _logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _logger.Error(""); _logger.Error(result.ErrorText); _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } if (_fluentCommandLineParser.Object.File.IsNullOrEmpty()) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); _logger.Warn("-f is required. Exiting"); return; } if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false && !File.Exists(_fluentCommandLineParser.Object.File)) { _logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting"); return; } _logger.Info(header); _logger.Info(""); _logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n"); try { if (_fluentCommandLineParser.Object.Quiet == false) { _logger.Warn($"Processing '{_fluentCommandLineParser.Object.File}'"); _logger.Info(""); } var sw = new Stopwatch(); sw.Start(); var rfc = RecentFileCache.RecentFileCache.LoadFile(_fluentCommandLineParser.Object.File); if (_fluentCommandLineParser.Object.Quiet == false) { _logger.Error($"Source file: {rfc.SourceFile}"); _logger.Info($" Source created: {rfc.SourceCreated.ToString(_dateTimeFormat)} "); _logger.Info($" Source modified: {rfc.SourceModified.ToString(_dateTimeFormat)}"); _logger.Info($" Source accessed: {rfc.SourceAccessed.ToString(_dateTimeFormat)}"); _logger.Info(""); _logger.Warn("File names"); foreach (var rfcFileName in rfc.FileNames) { _logger.Info($"{rfcFileName}"); } _logger.Info(""); } sw.Stop(); if (_fluentCommandLineParser.Object.Quiet) { _logger.Info(""); } _logger.Info( $"---------- Processed '{rfc.SourceFile}' in {sw.Elapsed.TotalSeconds:N8} seconds ----------"); if (_fluentCommandLineParser.Object.Quiet == false) { _logger.Info("\r\n"); } try { StreamWriter sw1 = null; if (_fluentCommandLineParser.Object.CsvDirectory?.Length > 0) { if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false) { _logger.Warn( $"'{_fluentCommandLineParser.Object.CsvDirectory} does not exist. Creating...'"); Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory); } var outName = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_RecentFileCacheParser_Output.tsv"; var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName); _fluentCommandLineParser.Object.CsvDirectory = Path.GetFullPath(outFile); _logger.Warn( $"CSV (tab separated) output will be saved to '{Path.GetFullPath(outFile)}'"); try { sw1 = new StreamWriter(outFile); var csv = new CsvWriter(sw1); csv.Configuration.Delimiter = $"{'\t'}"; csv.Configuration.HasHeaderRecord = true; var foo = csv.Configuration.AutoMap <CsvOut>(); foo.Map(t => t.SourceAccessed) .ConvertUsing(t => t.SourceAccessed.ToString(_dateTimeFormat)); foo.Map(t => t.SourceCreated).ConvertUsing(t => t.SourceCreated.ToString(_dateTimeFormat)); foo.Map(t => t.SourceModified) .ConvertUsing(t => t.SourceModified.ToString(_dateTimeFormat)); csv.WriteHeader(typeof(CsvOut)); csv.NextRecord(); csv.WriteRecords(GetCsvFormat(rfc)); } catch (Exception ex) { _logger.Error( $"Unable to open '{outFile}' for writing. CSV export canceled. Error: {ex.Message}"); } } if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0) { if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false) { _logger.Warn( $"'{_fluentCommandLineParser.Object.JsonDirectory} does not exist. Creating...'"); Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory); } _logger.Warn($"Saving json output to '{_fluentCommandLineParser.Object.JsonDirectory}'"); SaveJson(rfc, _fluentCommandLineParser.Object.JsonPretty, _fluentCommandLineParser.Object.JsonDirectory); } //Close CSV stuff sw1?.Flush(); sw1?.Close(); } catch (Exception e) { _logger.Error( $"Error exporting data! Error: {e.Message}"); } } catch (UnauthorizedAccessException ua) { _logger.Error( $"Unable to access '{_fluentCommandLineParser.Object.File}'. Are you running as an administrator? Error: {ua.Message}"); } catch (Exception ex) { _logger.Error( $"Error processing file '{_fluentCommandLineParser.Object.File}' Please send it to [email protected]. Error: {ex.Message}"); } }
//private static readonly string VssDir = @"C:\____vssMount"; private static void Main(string[] args) { ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq"); SetupNLog(); _logger = LogManager.GetLogger("EvtxECmd"); _fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments> { IsCaseSensitive = false }; _fluentCommandLineParser.Setup(arg => arg.File) .As('f') .WithDescription("File to process. This or -d is required\r\n"); _fluentCommandLineParser.Setup(arg => arg.Directory) .As('d') .WithDescription("Directory to process that contains evtx files. This or -f is required"); _fluentCommandLineParser.Setup(arg => arg.CsvDirectory) .As("csv") .WithDescription( "Directory to save CSV formatted results to."); // This, --json, or --xml required _fluentCommandLineParser.Setup(arg => arg.CsvName) .As("csvf") .WithDescription( "File name to save CSV formatted results to. When present, overrides default name"); _fluentCommandLineParser.Setup(arg => arg.JsonDirectory) .As("json") .WithDescription( "Directory to save JSON formatted results to."); // This, --csv, or --xml required _fluentCommandLineParser.Setup(arg => arg.JsonName) .As("jsonf") .WithDescription( "File name to save JSON formatted results to. When present, overrides default name"); _fluentCommandLineParser.Setup(arg => arg.XmlDirectory) .As("xml") .WithDescription( "Directory to save XML formatted results to."); // This, --csv, or --json required _fluentCommandLineParser.Setup(arg => arg.XmlName) .As("xmlf") .WithDescription( "File name to save XML formatted results to. When present, overrides default name\r\n"); _fluentCommandLineParser.Setup(arg => arg.DateTimeFormat) .As("dt") .WithDescription( "The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff") .SetDefault("yyyy-MM-dd HH:mm:ss.fffffff"); _fluentCommandLineParser.Setup(arg => arg.IncludeIds) .As("inc") .WithDescription( "List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.ExcludeIds) .As("exc") .WithDescription( "List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.StartDate) .As("sd") .WithDescription( "Start date for including events (UTC). Anything OLDER than this is dropped. Format should match --dt") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.EndDate) .As("ed") .WithDescription( "End date for including events (UTC). Anything NEWER than this is dropped. Format should match --dt") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.FullJson) .As("fj") .WithDescription( "When true, export all available data when using --json. Default is FALSE.") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.PayloadAsJson) .As("pj") .WithDescription( "When true, include event *payload* as json. Default is TRUE.") .SetDefault(true); _fluentCommandLineParser.Setup(arg => arg.TimeDiscrepancyThreshold) .As("tdt") .WithDescription( "The number of seconds to use for time discrepancy detection. Default is 1 second") .SetDefault(1); _fluentCommandLineParser.Setup(arg => arg.Metrics) .As("met") .WithDescription( "When true, show metrics about processed event log. Default is TRUE.\r\n") .SetDefault(true); _fluentCommandLineParser.Setup(arg => arg.MapsDirectory) .As("maps") .WithDescription( "The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ") .SetDefault(Path.Combine(BaseDirectory, "Maps")); /*_fluentCommandLineParser.Setup(arg => arg.Vss) * .As("vss") * .WithDescription( * "Process all Volume Shadow Copies that exist on drive specified by -f or -d . Default is FALSE") * .SetDefault(false);*/ _fluentCommandLineParser.Setup(arg => arg.Dedupe) .As("dedupe") .WithDescription( "Deduplicate -f or -d & VSCs based on SHA-1. First file found wins. Default is TRUE\r\n") .SetDefault(true); _fluentCommandLineParser.Setup(arg => arg.Sync) .As("sync") .WithDescription( "If true, the latest maps from https://github.com/EricZimmerman/evtx/tree/master/evtx/Maps are downloaded and local maps updated. Default is FALSE\r\n") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.Debug) .As("debug") .WithDescription("Show debug information during processing").SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.Trace) .As("trace") .WithDescription("Show trace information during processing\r\n").SetDefault(false); var header = $"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" + "\n\nAuthor: Eric Zimmerman ([email protected])" + "\nhttps://github.com/EricZimmerman/evtx" + "\nLinux Port : Scott Dermott ([email protected])" + "\nhttps://github.com/ScottDermott/evtx"; var footer = @"Examples: EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out"" --csvf MyOutputFile.csv" + "\r\n\t " + @" EvtxECmd -f ""/Temp/Application.evtx"" --csv ""/temp/out""" + "\r\n\t " + @" EvtxECmd -f ""/Temp/Application.evtx"" --json ""/temp/jsonout""" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help") .WithHeader(header) .Callback(text => _logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _logger.Error(""); _logger.Error(result.ErrorText); _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } if (_fluentCommandLineParser.Object.Sync) { try { _logger.Info(header); UpdateFromRepo(); } catch (Exception e) { _logger.Error(e, $"There was an error checking for updates: {e.Message}"); } Environment.Exit(0); } if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() && _fluentCommandLineParser.Object.Directory.IsNullOrEmpty()) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); _logger.Warn("-f or -d is required. Exiting"); return; } _logger.Info(header); _logger.Info(""); _logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n"); /* if (IsAdministrator() == false) * { * _logger.Fatal("Warning: Administrator privileges not found!\r\n"); * }*/ if (_fluentCommandLineParser.Object.Debug) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug); } if (_fluentCommandLineParser.Object.Trace) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace); } LogManager.ReconfigExistingLoggers(); /* if (_fluentCommandLineParser.Object.Vss & (IsAdministrator() == false)) * { * _logger.Error("--vss is present, but administrator rights not found. Exiting\r\n"); * return; * }*/ var sw = new Stopwatch(); sw.Start(); var ts = DateTimeOffset.UtcNow; _errorFiles = new Dictionary <string, int>(); if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json"; if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName); _logger.Warn($"json output will be saved to '{outFile}'\r\n"); try { _swJson = new StreamWriter(outFile, false, Encoding.UTF8); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } JsConfig.DateHandler = DateHandler.ISO8601; } if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml"; if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName); _logger.Warn($"XML output will be saved to '{outFile}'\r\n"); try { _swXml = new StreamWriter(outFile, false, Encoding.UTF8); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } } if (_fluentCommandLineParser.Object.StartDate.IsNullOrEmpty() == false) { if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.StartDate, null, DateTimeStyles.AssumeUniversal, out var dt)) { _startDate = dt; _logger.Info($"Setting Start date to '{_startDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'"); } else { _logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.StartDate}' to a valud datetime! Events will not be filtered by Start date!"); } } if (_fluentCommandLineParser.Object.EndDate.IsNullOrEmpty() == false) { if (DateTimeOffset.TryParse(_fluentCommandLineParser.Object.EndDate, null, DateTimeStyles.AssumeUniversal, out var dt)) { _endDate = dt; _logger.Info($"Setting End date to '{_endDate.Value.ToUniversalTime().ToString(_fluentCommandLineParser.Object.DateTimeFormat)}'"); } else { _logger.Warn($"Could not parse '{_fluentCommandLineParser.Object.EndDate}' to a valud datetime! Events will not be filtered by End date!"); } } if (_startDate.HasValue || _endDate.HasValue) { _logger.Info(""); } if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv"; if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName); _logger.Warn($"CSV output will be saved to '{outFile}'\r\n"); try { _swCsv = new StreamWriter(outFile, false, Encoding.UTF8); _csvWriter = new CsvWriter(_swCsv, CultureInfo.InvariantCulture); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } var foo = _csvWriter.Configuration.AutoMap <EventRecord>(); if (_fluentCommandLineParser.Object.PayloadAsJson == false) { foo.Map(t => t.Payload).Ignore(); } else { foo.Map(t => t.Payload).Index(22); } foo.Map(t => t.RecordPosition).Ignore(); foo.Map(t => t.Size).Ignore(); foo.Map(t => t.Timestamp).Ignore(); foo.Map(t => t.RecordNumber).Index(0); foo.Map(t => t.EventRecordId).Index(1); foo.Map(t => t.TimeCreated).Index(2); foo.Map(t => t.TimeCreated).ConvertUsing(t => $"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}"); foo.Map(t => t.EventId).Index(3); foo.Map(t => t.Level).Index(4); foo.Map(t => t.Provider).Index(5); foo.Map(t => t.Channel).Index(6); foo.Map(t => t.ProcessId).Index(7); foo.Map(t => t.ThreadId).Index(8); foo.Map(t => t.Computer).Index(9); foo.Map(t => t.UserId).Index(10); foo.Map(t => t.MapDescription).Index(11); foo.Map(t => t.UserName).Index(12); foo.Map(t => t.RemoteHost).Index(13); foo.Map(t => t.PayloadData1).Index(14); foo.Map(t => t.PayloadData2).Index(15); foo.Map(t => t.PayloadData3).Index(16); foo.Map(t => t.PayloadData4).Index(17); foo.Map(t => t.PayloadData5).Index(18); foo.Map(t => t.PayloadData6).Index(19); foo.Map(t => t.ExecutableInfo).Index(20); foo.Map(t => t.SourceFile).Index(21); _csvWriter.Configuration.RegisterClassMap(foo); _csvWriter.WriteHeader <EventRecord>(); _csvWriter.NextRecord(); } if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false) { _logger.Warn( $"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!"); } else { _logger.Info($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'"); var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)); if (errors) { return; } _logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}"); } _includeIds = new HashSet <int>(); _excludeIds = new HashSet <int>(); if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false) { var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(','); foreach (var incSeg in excSegs) { if (int.TryParse(incSeg, out var goodId)) { _excludeIds.Add(goodId); } } } if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false) { _excludeIds.Clear(); var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(','); foreach (var incSeg in incSegs) { if (int.TryParse(incSeg, out var goodId)) { _includeIds.Add(goodId); } } } /*if (_fluentCommandLineParser.Object.Vss) * { * string driveLetter; * if (_fluentCommandLineParser.Object.File.IsEmpty() == false) * { * driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File)) * .Substring(0, 1); * } * else * { * driveLetter = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory)) * .Substring(0, 1); * } * * * Helper.MountVss(driveLetter,VssDir); * Console.WriteLine(); * }*/ EventLog.TimeDiscrepancyThreshold = _fluentCommandLineParser.Object.TimeDiscrepancyThreshold; if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false) { if (File.Exists(_fluentCommandLineParser.Object.File) == false) { _logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting"); return; } if (_swXml == null && _swJson == null && _swCsv == null) { //no need for maps _logger.Debug("Clearing map collection since no output specified"); EventLog.EventLogMaps.Clear(); } _fluentCommandLineParser.Object.Dedupe = false; ProcessFile(Path.GetFullPath(_fluentCommandLineParser.Object.File)); /*if (_fluentCommandLineParser.Object.Vss) * { * var vssDirs = Directory.GetDirectories(VssDir); * * var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.File)); * var stem = Path.GetFullPath(_fluentCommandLineParser.Object.File).Replace(root, ""); * * foreach (var vssDir in vssDirs) * { * var newPath = Path.Combine(vssDir, stem); * if (File.Exists(newPath)) * { * ProcessFile(newPath); * } * } * }*/ } else { _logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'"); _logger.Info(""); /* var f = new DirectoryEnumerationFilters * { * InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX", * RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink, * ErrorFilter = (errorCode, errorMessage, pathProcessed) => true * };*/ /* var dirEnumOptions = * DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive | * DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException | * DirectoryEnumerationOptions.BasicSearch;*/ var files2 = Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), "*.evtx"); if (_swXml == null && _swJson == null && _swCsv == null) { //no need for maps _logger.Debug("Clearing map collection since no output specified"); EventLog.EventLogMaps.Clear(); } foreach (var file in files2) { ProcessFile(file); } /*if (_fluentCommandLineParser.Object.Vss) * { * var vssDirs = Directory.GetDirectories(VssDir); * * Console.WriteLine(); * * foreach (var vssDir in vssDirs) * { * var root = Path.GetPathRoot(Path.GetFullPath(_fluentCommandLineParser.Object.Directory)); * var stem = Path.GetFullPath(_fluentCommandLineParser.Object.Directory).Replace(root, ""); * * var target = Path.Combine(vssDir, stem); * * _logger.Fatal($"\r\nSearching 'VSS{target.Replace($"{VssDir}\\","")}' for event logs..."); * * var vssFiles = Helper.GetFilesFromPath(target, true, "*.evtx"); * * foreach (var file in vssFiles) * { * ProcessFile(file); * } * } * * }*/ } _swCsv?.Flush(); _swCsv?.Close(); _swJson?.Flush(); _swJson?.Close(); _swXml?.Flush(); _swXml?.Close(); sw.Stop(); _logger.Info(""); var suff = string.Empty; if (_fileCount != 1) { suff = "s"; } _logger.Error( $"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n"); if (_errorFiles.Count > 0) { _logger.Info(""); _logger.Error("Files with errors"); foreach (var errorFile in _errorFiles) { _logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}"); } _logger.Info(""); } /*if (_fluentCommandLineParser.Object.Vss) * { * if (Directory.Exists(VssDir)) * { * foreach (var directory in Directory.GetDirectories(VssDir)) * { * Directory.Delete(directory); * } * Directory.Delete(VssDir,true); * } * }*/ }
private void ExportDataToFolder(string outputPath) { var metadataPane = this.ActiveDocument.MetadataPane; var exceptionFound = false; // TODO: Use async but to be well done need to apply async on the DBCommand & DBConnection // TODO: Show warning message? if (metadataPane.SelectedModel == null) { return; } if (!Directory.Exists(outputPath)) { Directory.CreateDirectory(outputPath); } ActiveDocument.QueryStopWatch.Start(); foreach (var table in metadataPane.SelectedModel.Tables) { var rows = 0; var tableCnt = 0; try { var csvFilePath = System.IO.Path.Combine(outputPath, $"{table.Name}.csv"); var daxQuery = $"EVALUATE {table.DaxName}"; var totalTables = metadataPane.SelectedModel.Tables.Count; using (var textWriter = new StreamWriter(csvFilePath, false, Encoding.UTF8)) using (var csvWriter = new CsvHelper.CsvWriter(textWriter)) using (var statusMsg = new StatusBarMessage(this.ActiveDocument, $"Exporting {table.Name}")) using (var reader = ActiveDocument.Connection.ExecuteReader(daxQuery)) { rows = 0; tableCnt++; // Write Header foreach (var colName in reader.CleanColumnNames()) { csvWriter.WriteField(colName); } csvWriter.NextRecord(); // Write data while (reader.Read()) { for (var fieldOrdinal = 0; fieldOrdinal < reader.FieldCount; fieldOrdinal++) { var fieldValue = reader[fieldOrdinal]; csvWriter.WriteField(fieldValue); } rows++; if (rows % 5000 == 0) { new StatusBarMessage(ActiveDocument, $"Exporting Table {tableCnt} of {totalTables} : {table.Name} ({rows:N0} rows)"); ActiveDocument.RefreshElapsedTime(); // if cancel has been requested do not write any more records if (CancelRequested) { break; } } csvWriter.NextRecord(); } ActiveDocument.RefreshElapsedTime(); _eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, $"Exported {rows:N0} rows to {table.Name}.csv")); // if cancel has been requested do not write any more files if (CancelRequested) { break; } } } catch (Exception ex) { exceptionFound = true; Log.Error(ex, "{class} {method} {message}", "ExportDataDialogViewModel", "ExportDataToFolder", "Error while exporting model to CSV"); _eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Error, $"Error Exporting '{table.Name}': {ex.Message}")); break; // exit from the loop if we have caught an exception } } ActiveDocument.QueryStopWatch.Stop(); // export complete if (!exceptionFound) { _eventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, $"Model Export Complete: {metadataPane.SelectedModel.Tables.Count} tables exported", ActiveDocument.QueryStopWatch.ElapsedMilliseconds)); } ActiveDocument.QueryStopWatch.Reset(); }
private void ExportDataToCSV(string outputPath) { var exceptionFound = false; // TODO: Use async but to be well done need to apply async on the DBCommand & DBConnection // TODO: Show warning message? if (string.IsNullOrEmpty(Document.Connection.SelectedModelName)) { return; } if (!Directory.Exists(outputPath)) { Directory.CreateDirectory(outputPath); } Document.QueryStopWatch.Start(); var selectedTables = Tables.Where(t => t.IsSelected); var totalTables = selectedTables.Count(); var tableCnt = 0; string decimalSep = System.Globalization.CultureInfo.CurrentUICulture.NumberFormat.CurrencyDecimalSeparator; string isoDateFormat = string.Format(Constants.IsoDateMask, decimalSep); var encoding = new UTF8Encoding(false); foreach (var table in selectedTables) { EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(table)); var rows = 0; tableCnt++; try { table.Status = ExportStatus.Exporting; var fileName = CleanNameOfIllegalChars(table.Caption); var csvFilePath = System.IO.Path.Combine(outputPath, $"{fileName}.csv"); var daxRowCount = $"EVALUATE ROW(\"RowCount\", COUNTROWS( {table.DaxName} ) )"; // get a count of the total rows in the table var connRead = Document.Connection; DataTable dtRows = connRead.ExecuteDaxQueryDataTable(daxRowCount); var totalRows = dtRows.Rows[0].Field <long?>(0) ?? 0; table.TotalRows = totalRows; StreamWriter textWriter = null; try { textWriter = new StreamWriter(csvFilePath, false, encoding); using (var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture)) using (var statusMsg = new StatusBarMessage(Document, $"Exporting {table.Caption}")) { for (long batchRows = 0; batchRows < totalRows; batchRows += maxBatchSize) { var daxQuery = $"EVALUATE {table.DaxName}"; // if the connection supports TOPNSKIP then use that to query batches of rows if (connRead.AllFunctions.Contains("TOPNSKIP")) { daxQuery = $"EVALUATE TOPNSKIP({maxBatchSize}, {batchRows}, {table.DaxName} )"; } using (var reader = connRead.ExecuteReader(daxQuery)) { rows = 0; // configure delimiter csvWriter.Configuration.Delimiter = CsvDelimiter; // output dates using ISO 8601 format csvWriter.Configuration.TypeConverterOptionsCache.AddOptions( typeof(DateTime), new CsvHelper.TypeConversion.TypeConverterOptions() { Formats = new string[] { isoDateFormat } }); // if this is the first batch of rows if (batchRows == 0) { // Write Header foreach (var colName in reader.CleanColumnNames()) { csvWriter.WriteField(colName); } csvWriter.NextRecord(); } // Write data while (reader.Read()) { for (var fieldOrdinal = 0; fieldOrdinal < reader.FieldCount; fieldOrdinal++) { var fieldValue = reader[fieldOrdinal]; // quote all string fields if (reader.GetFieldType(fieldOrdinal) == typeof(string)) { if (reader.IsDBNull(fieldOrdinal)) { csvWriter.WriteField("", this.CsvQuoteStrings); } else { csvWriter.WriteField(fieldValue.ToString(), this.CsvQuoteStrings); } } else { csvWriter.WriteField(fieldValue); } } rows++; if (rows % 5000 == 0) { table.RowCount = rows + batchRows; statusMsg.Update($"Exporting Table {tableCnt} of {totalTables} : {table.DaxName} ({rows + batchRows:N0} rows)"); Document.RefreshElapsedTime(); // if cancel has been requested do not write any more records if (CancelRequested) { table.Status = ExportStatus.Cancelled; // break out of datareader.Read() loop break; } } csvWriter.NextRecord(); } Document.RefreshElapsedTime(); table.RowCount = rows + batchRows; // if cancel has been requested do not write any more files if (CancelRequested) { EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Warning, "Data Export Cancelled")); MarkWaitingTablesAsSkipped(); // break out of foreach table loop break; } } // do not loop around if the current connection does not support TOPNSKIP if (!connRead.AllFunctions.Contains("TOPNSKIP")) { break; } } // end of batch EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, exportTableMsg.Format(rows, rows == 1 ? "":"s", table.DaxName + ".csv")));; } } finally { textWriter?.Dispose(); } table.Status = ExportStatus.Done; } catch (Exception ex) { table.Status = ExportStatus.Error; exceptionFound = true; Log.Error(ex, "{class} {method} {message}", nameof(ExportDataWizardViewModel), nameof(ExportDataToCSV), "Error while exporting model to CSV"); EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Error, $"Error Exporting '{table.DaxName}': {ex.Message}")); EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(currentTable, true)); continue; // skip to the next table if we have caught an exception } } Document.QueryStopWatch.Stop(); // export complete if (!exceptionFound) { EventAggregator.PublishOnUIThread(new OutputMessage(MessageType.Information, exportCompleteMsg.Format(tableCnt), Document.QueryStopWatch.ElapsedMilliseconds)); } EventAggregator.PublishOnUIThread(new ExportStatusUpdateEvent(currentTable, true)); Document.QueryStopWatch.Reset(); }
/// <summary> /// /// </summary> /// <param name="dataDirectory"></param> /// <param name="outputDirectory"></param> public void PostProcess(string dataDirectory, string outputDirectory) { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.QuoteAllFields = true; using (FileStream fileStream = new FileStream(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv"), FileMode.Append, FileAccess.Write, FileShare.Read)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { // Now MD5 the files foreach (string file in System.IO.Directory.EnumerateFiles(outputDirectory, "*.xml", SearchOption.AllDirectories)) { string fileName = System.IO.Path.GetFileName(file); if (fileName.StartsWith("Message.Details.") == false) { continue; } MessageDetails messageDetails = new MessageDetails(); string ret = messageDetails.Load(file); if (ret.Length == 0) { foreach (AttachmentDetails attachment in messageDetails.Attachments) { csvWriter.WriteField(attachment.Md5); csvWriter.WriteField(attachment.File); csvWriter.WriteField(messageDetails.SrcIp); csvWriter.WriteField(messageDetails.SrcPort); csvWriter.WriteField(messageDetails.DstIp); csvWriter.WriteField(messageDetails.DstPort); csvWriter.WriteField(messageDetails.To); csvWriter.WriteField(messageDetails.From); csvWriter.WriteField(messageDetails.MailFrom); csvWriter.WriteField(messageDetails.Sender); csvWriter.WriteField(messageDetails.Subject); csvWriter.WriteField(messageDetails.Date); csvWriter.NextRecord(); } } } } ProcessAttachmentHashes(outputDirectory); }
/// <summary> /// /// </summary> /// <param name="filePath">The output file name</param> /// <param name="dateFrom"></param> /// <param name="dateTo"></param> /// <param name="sid"></param> public void ExportEventsAll(string filePath, string dateFrom, string dateTo, string sid) { if (IsRunning == true) { OnExclamation("Already performing an export"); return; } IsRunning = true; new Thread(() => { try { using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection())) { string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_RULES_EVENTS_EXPORT); query = query.Replace("#WHERE#", @"WHERE exclude.id IS NULL AND event.timestamp > @0 AND event.timestamp < @1 AND signature.sig_sid = @2"); List<Event> events = db.Fetch<Event>(query, new object[] { dateFrom, dateTo, sid }); events = Helper.ProcessEventDataSet(events); CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = '\t'; using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { // Write out the file headers csvWriter.WriteField("CID"); csvWriter.WriteField("Src IP"); csvWriter.WriteField("Src Port"); csvWriter.WriteField("Dst IP"); csvWriter.WriteField("Dst Port"); csvWriter.WriteField("Protocol"); csvWriter.WriteField("Timestamp"); csvWriter.WriteField("TCP Flags"); csvWriter.WriteField("Payload (ASCII)"); csvWriter.WriteField("Payload (HEX)"); csvWriter.NextRecord(); foreach (var item in events) { csvWriter.WriteField(item.Cid); csvWriter.WriteField(item.IpSrcTxt); csvWriter.WriteField(item.SrcPort); csvWriter.WriteField(item.IpDst); csvWriter.WriteField(item.DstPort); csvWriter.WriteField(item.Protocol); csvWriter.WriteField(item.Timestamp); csvWriter.WriteField(item.TcpFlagsString); csvWriter.WriteField(item.PayloadAscii); csvWriter.WriteField( Helper.ConvertByteArrayToHexString(item.PayloadHex)); csvWriter.NextRecord(); } } OnComplete(); } } catch (Exception ex) { OnError("An error occurred whilst performing the export: " + ex.Message); } finally { IsRunning = false; } }).Start(); }
/// <summary> /// /// </summary> /// <param name="filePath"></param> /// <param name="dateFrom"></param> /// <param name="initials"></param> /// <param name="text"></param> public void ExportAcknowledgmentsFrom(string filePath, string dateFrom, string initials, bool text) { if (IsRunning == true) { OnExclamation("Already performing an export"); return; } IsRunning = true; new Thread(() => { try { if (text == true) { using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection())) using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) { string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_ACKNOWLEDGEMENT); query = query.Replace("#WHERE#", @"WHERE acknowledgment.initials = @0 AND acknowledgment.timestamp > @1"); var data = db.Fetch<Dictionary<string, object>>(query, new object[] { initials, dateFrom }); foreach (Dictionary<string, object> temp in data) { streamWriter.WriteLine("Signature: " + temp["sig_name"].ToString()); streamWriter.WriteLine("SID/GID: " + temp["sig_sid"].ToString() + "/" + temp["sig_gid"].ToString()); if (temp["successful"] == null) { streamWriter.WriteLine("Acknowledgement: " + temp["description"].ToString()); } else { streamWriter.WriteLine("Acknowledgement: " + temp["description"].ToString() + " (" + temp["successful"].ToString() + ")"); } streamWriter.WriteLine("Notes: " + temp["notes"].ToString()); streamWriter.WriteLine(string.Empty); } OnComplete(); } } else { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = '\t'; using (NPoco.Database db = new NPoco.Database(Db.GetOpenMySqlConnection())) using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { string query = _sql.GetQuery(snorbert.Configs.Sql.Query.SQL_ACKNOWLEDGEMENT); query = query.Replace("#WHERE#", @"WHERE acknowledgment.initials = @0 AND acknowledgment.timestamp > @1"); var data = db.Fetch<Dictionary<string, object>>(query, new object[] { initials, dateFrom }); // Write out the file headers csvWriter.WriteField("Sig Name"); csvWriter.WriteField("Sig SID"); csvWriter.WriteField("Sig GID"); csvWriter.WriteField("Acknowledgement"); csvWriter.WriteField("Notes"); csvWriter.NextRecord(); foreach (Dictionary<string, object> temp in data) { csvWriter.WriteField(temp["sig_name"].ToString()); csvWriter.WriteField(temp["sig_sid"].ToString()); csvWriter.WriteField(temp["sig_gid"].ToString()); if (temp["successful"] == null) { csvWriter.WriteField(temp["description"].ToString()); } else { csvWriter.WriteField(temp["description"].ToString() + " (" + temp["successful"].ToString() + ")"); } csvWriter.WriteField(temp["notes"].ToString()); csvWriter.NextRecord(); } OnComplete(); } } } catch (Exception ex) { OnError("An error occurred whilst performing the export: " + ex.Message); } finally { IsRunning = false; } }).Start(); }
/// <summary> /// /// </summary> /// <param name="signatures"></param> /// <param name="filePath">The output file name</param> public void ExportRules(List<Signature> signatures, string filePath) { if (IsRunning == true) { OnExclamation("Already performing an export"); return; } IsRunning = true; new Thread(() => { try { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = '\t'; using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { // Write out the file headers csvWriter.WriteField("SID"); csvWriter.WriteField("Name"); csvWriter.WriteField("Priority"); csvWriter.WriteField("Count"); csvWriter.NextRecord(); foreach (Signature temp in signatures) { csvWriter.WriteField(temp.Sid); csvWriter.WriteField(temp.Name); csvWriter.WriteField(temp.Priority); csvWriter.WriteField(temp.Count); csvWriter.NextRecord(); } } OnComplete(); } catch (Exception ex) { OnError("An error occurred whilst performing the export: " + ex.Message); } finally { IsRunning = false; } }).Start(); }
/// <summary> /// /// </summary> /// <param name="filePath">The output file name</param> public void ExportExcludes(string filePath) { if (IsRunning == true) { OnExclamation("Already performing an export"); return; } IsRunning = true; new Thread(() => { try { List<Exclude> excludes = new List<Exclude>(); using (NPoco.Database dbMySql = new NPoco.Database(Db.GetOpenMySqlConnection())) { var data = dbMySql.Fetch<Dictionary<string, object>>(_sql.GetQuery(snorbert.Configs.Sql.Query.SQL_EXCLUDES)); foreach (Dictionary<string, object> temp in data) { Exclude exclude = new Exclude(); exclude.Id = long.Parse(temp["id"].ToString()); exclude.SigId = long.Parse(temp["sig_id"].ToString()); exclude.SigSid = long.Parse(temp["sig_sid"].ToString()); exclude.Rule = temp["sig_name"].ToString(); exclude.SourceIpText = temp["ip_src"].ToString(); exclude.DestinationIpText = temp["ip_dst"].ToString(); if (((byte[])temp["fp"])[0] == 48) { exclude.FalsePositive = false; } else { exclude.FalsePositive = true; } exclude.Timestamp = DateTime.Parse(temp["timestamp"].ToString()); excludes.Add(exclude); } } CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = '\t'; using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { // Write out the file headers csvWriter.WriteField("Sig. ID"); csvWriter.WriteField("Source IP"); csvWriter.WriteField("Destination IP"); csvWriter.WriteField("FP"); csvWriter.WriteField("Comment"); csvWriter.WriteField("Sig. Name"); csvWriter.WriteField("Timestamp"); csvWriter.WriteField("Sig."); csvWriter.NextRecord(); foreach (var temp in excludes) { csvWriter.WriteField(temp.SigId); csvWriter.WriteField(temp.SourceIpText); csvWriter.WriteField(temp.DestinationIpText); csvWriter.WriteField(temp.FalsePositive); csvWriter.WriteField(temp.Comment); csvWriter.WriteField(temp.Rule); csvWriter.WriteField(temp.Timestamp); csvWriter.WriteField(temp.Rule); csvWriter.NextRecord(); } } OnComplete(); } catch (Exception ex) { OnError("An error occurred whilst performing the export: " + ex.Message); } finally { IsRunning = false; } }).Start(); }
private static void Main(string[] args) { ExceptionlessClient.Default.Startup("tYeWS6A5K5uItgpB44dnNy2qSb2xJxiQWRRGWebq"); SetupNLog(); _logger = LogManager.GetLogger("EvtxECmd"); _fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments> { IsCaseSensitive = false }; _fluentCommandLineParser.Setup(arg => arg.File) .As('f') .WithDescription("File to process. This or -d is required\r\n"); _fluentCommandLineParser.Setup(arg => arg.Directory) .As('d') .WithDescription("Directory to process that contains evtx files. This or -f is required"); _fluentCommandLineParser.Setup(arg => arg.CsvDirectory) .As("csv") .WithDescription( "Directory to save CSV formatted results to."); // This, --json, or --xml required _fluentCommandLineParser.Setup(arg => arg.CsvName) .As("csvf") .WithDescription( "File name to save CSV formatted results to. When present, overrides default name"); _fluentCommandLineParser.Setup(arg => arg.JsonDirectory) .As("json") .WithDescription( "Directory to save JSON formatted results to."); // This, --csv, or --xml required _fluentCommandLineParser.Setup(arg => arg.JsonName) .As("jsonf") .WithDescription( "File name to save JSON formatted results to. When present, overrides default name"); _fluentCommandLineParser.Setup(arg => arg.XmlDirectory) .As("xml") .WithDescription( "Directory to save XML formatted results to."); // This, --csv, or --json required _fluentCommandLineParser.Setup(arg => arg.XmlName) .As("xmlf") .WithDescription( "File name to save XML formatted results to. When present, overrides default name\r\n"); _fluentCommandLineParser.Setup(arg => arg.DateTimeFormat) .As("dt") .WithDescription( "The custom date/time format to use when displaying time stamps. Default is: yyyy-MM-dd HH:mm:ss.fffffff") .SetDefault("yyyy-MM-dd HH:mm:ss.fffffff"); _fluentCommandLineParser.Setup(arg => arg.IncludeIds) .As("inc") .WithDescription( "List of event IDs to process. All others are ignored. Overrides --exc Format is 4624,4625,5410") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.ExcludeIds) .As("exc") .WithDescription( "List of event IDs to IGNORE. All others are included. Format is 4624,4625,5410") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.FullJson) .As("fj") .WithDescription( "When true, export all available data when using --json. Default is FALSE.") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.Metrics) .As("met") .WithDescription( "When true, show metrics about processed event log. Default is TRUE.\r\n") .SetDefault(true); _fluentCommandLineParser.Setup(arg => arg.MapsDirectory) .As("maps") .WithDescription( "The path where event maps are located. Defaults to 'Maps' folder where program was executed\r\n ") .SetDefault(Path.Combine(BaseDirectory, "Maps")); _fluentCommandLineParser.Setup(arg => arg.Debug) .As("debug") .WithDescription("Show debug information during processing").SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.Trace) .As("trace") .WithDescription("Show trace information during processing\r\n").SetDefault(false); var header = $"EvtxECmd version {Assembly.GetExecutingAssembly().GetName().Version}" + "\r\n\r\nAuthor: Eric Zimmerman ([email protected])" + "\r\nhttps://github.com/EricZimmerman/evtx"; var footer = @"Examples: EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out"" --csvf MyOutputFile.csv" + "\r\n\t " + @" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --csv ""c:\temp\out""" + "\r\n\t " + @" EvtxECmd.exe -f ""C:\Temp\Application.evtx"" --json ""c:\temp\jsonout""" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help") .WithHeader(header) .Callback(text => _logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _logger.Error(""); _logger.Error(result.ErrorText); _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() && _fluentCommandLineParser.Object.Directory.IsNullOrEmpty()) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); _logger.Warn("-f or -d is required. Exiting"); return; } _logger.Info(header); _logger.Info(""); _logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n"); if (IsAdministrator() == false) { _logger.Fatal("Warning: Administrator privileges not found!\r\n"); } if (_fluentCommandLineParser.Object.Debug) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug); } if (_fluentCommandLineParser.Object.Trace) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Trace); } LogManager.ReconfigExistingLoggers(); var sw = new Stopwatch(); sw.Start(); var ts = DateTimeOffset.UtcNow; _errorFiles = new Dictionary <string, int>(); if (_fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.JsonDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.JsonDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json"; if (_fluentCommandLineParser.Object.JsonName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.JsonName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.JsonDirectory, outName); _logger.Warn($"json output will be saved to '{outFile}'\r\n"); try { _swJson = new StreamWriter(outFile, false, Encoding.UTF8); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } } if (_fluentCommandLineParser.Object.XmlDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.XmlDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.XmlDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.XmlDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.XmlDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml"; if (_fluentCommandLineParser.Object.XmlName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.XmlName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.XmlDirectory, outName); _logger.Warn($"XML output will be saved to '{outFile}'\r\n"); try { _swXml = new StreamWriter(outFile, false, Encoding.UTF8); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } } if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false) { if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' doesn't exist. Creating..."); try { Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory); } catch (Exception) { _logger.Fatal( $"Unable to create directory '{_fluentCommandLineParser.Object.CsvDirectory}'. Does a file with the same name exist? Exiting"); return; } } var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv"; if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false) { outName = Path.GetFileName(_fluentCommandLineParser.Object.CsvName); } var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName); _logger.Warn($"CSV output will be saved to '{outFile}'\r\n"); try { _swCsv = new StreamWriter(outFile, false, Encoding.UTF8); _csvWriter = new CsvWriter(_swCsv); } catch (Exception) { _logger.Error($"Unable to open '{outFile}'! Is it in use? Exiting!\r\n"); Environment.Exit(0); } var foo = _csvWriter.Configuration.AutoMap <EventRecord>(); foo.Map(t => t.PayloadXml).Ignore(); foo.Map(t => t.RecordPosition).Ignore(); foo.Map(t => t.Size).Ignore(); foo.Map(t => t.Timestamp).Ignore(); foo.Map(t => t.RecordNumber).Index(0); foo.Map(t => t.TimeCreated).Index(1); foo.Map(t => t.TimeCreated).ConvertUsing(t => $"{t.TimeCreated.ToString(_fluentCommandLineParser.Object.DateTimeFormat)}"); foo.Map(t => t.EventId).Index(2); foo.Map(t => t.Level).Index(3); foo.Map(t => t.Provider).Index(4); foo.Map(t => t.Channel).Index(5); foo.Map(t => t.ProcessId).Index(6); foo.Map(t => t.ThreadId).Index(7); foo.Map(t => t.Computer).Index(8); foo.Map(t => t.UserId).Index(9); foo.Map(t => t.MapDescription).Index(10); foo.Map(t => t.UserName).Index(11); foo.Map(t => t.RemoteHost).Index(12); foo.Map(t => t.PayloadData1).Index(13); foo.Map(t => t.PayloadData2).Index(14); foo.Map(t => t.PayloadData3).Index(15); foo.Map(t => t.PayloadData4).Index(16); foo.Map(t => t.PayloadData5).Index(17); foo.Map(t => t.PayloadData6).Index(18); foo.Map(t => t.ExecutableInfo).Index(19); foo.Map(t => t.SourceFile).Index(20); _csvWriter.Configuration.RegisterClassMap(foo); _csvWriter.WriteHeader <EventRecord>(); _csvWriter.NextRecord(); } if (Directory.Exists(_fluentCommandLineParser.Object.MapsDirectory) == false) { _logger.Warn( $"Maps directory '{_fluentCommandLineParser.Object.MapsDirectory}' does not exist! Event ID maps will not be loaded!!"); } else { _logger.Debug($"Loading maps from '{Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)}'"); var errors = EventLog.LoadMaps(Path.GetFullPath(_fluentCommandLineParser.Object.MapsDirectory)); if (errors) { return; } _logger.Info($"Maps loaded: {EventLog.EventLogMaps.Count:N0}"); } _includeIds = new HashSet <int>(); _excludeIds = new HashSet <int>(); if (_fluentCommandLineParser.Object.ExcludeIds.IsNullOrEmpty() == false) { var excSegs = _fluentCommandLineParser.Object.ExcludeIds.Split(','); foreach (var incSeg in excSegs) { if (int.TryParse(incSeg, out var goodId)) { _excludeIds.Add(goodId); } } } if (_fluentCommandLineParser.Object.IncludeIds.IsNullOrEmpty() == false) { _excludeIds.Clear(); var incSegs = _fluentCommandLineParser.Object.IncludeIds.Split(','); foreach (var incSeg in incSegs) { if (int.TryParse(incSeg, out var goodId)) { _includeIds.Add(goodId); } } } if (_fluentCommandLineParser.Object.File.IsNullOrEmpty() == false) { if (File.Exists(_fluentCommandLineParser.Object.File) == false) { _logger.Warn($"'{_fluentCommandLineParser.Object.File}' does not exist! Exiting"); return; } ProcessFile(_fluentCommandLineParser.Object.File); } else { _logger.Info($"Looking for event log files in '{_fluentCommandLineParser.Object.Directory}'"); _logger.Info(""); var f = new DirectoryEnumerationFilters(); f.InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX"; f.RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink; f.ErrorFilter = (errorCode, errorMessage, pathProcessed) => true; var dirEnumOptions = DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive | DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException | DirectoryEnumerationOptions.BasicSearch; var files2 = Directory.EnumerateFileSystemEntries(Path.GetFullPath(_fluentCommandLineParser.Object.Directory), dirEnumOptions, f); foreach (var file in files2) { ProcessFile(file); } } _swCsv?.Flush(); _swCsv?.Close(); _swJson?.Flush(); _swJson?.Close(); _swXml?.Flush(); _swXml?.Close(); sw.Stop(); _logger.Info(""); var suff = string.Empty; if (_fileCount != 1) { suff = "s"; } _logger.Error( $"Processed {_fileCount:N0} file{suff} in {sw.Elapsed.TotalSeconds:N4} seconds\r\n"); if (_errorFiles.Count > 0) { _logger.Info(""); _logger.Error("Files with errors"); foreach (var errorFile in _errorFiles) { _logger.Info($"'{errorFile.Key}' error count: {errorFile.Value:N0}"); } _logger.Info(""); } }
private static void Main(string[] args) { ExceptionlessClient.Default.Startup("x3MPpeQSBUUsXl3DjekRQ9kYjyN3cr5JuwdoOBpZ"); SetupNLog(); _keywords = new HashSet <string> { "temp", "tmp" }; _logger = LogManager.GetCurrentClassLogger(); if (!CheckForDotnet46()) { _logger.Warn(".net 4.6 not detected. Please install .net 4.6 and try again."); return; } _fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments> { IsCaseSensitive = false }; _fluentCommandLineParser.Setup(arg => arg.File) .As('f') .WithDescription("File to process. Either this or -d is required"); _fluentCommandLineParser.Setup(arg => arg.Directory) .As('d') .WithDescription("Directory to recursively process. Either this or -f is required"); _fluentCommandLineParser.Setup(arg => arg.Keywords) .As('k') .WithDescription( "Comma separated list of keywords to highlight in output. By default, 'temp' and 'tmp' are highlighted. Any additional keywords will be added to these."); _fluentCommandLineParser.Setup(arg => arg.OutFile) .As('o') .WithDescription( "When specified, save prefetch file bytes to the given path. Useful to look at decompressed Win10 files"); _fluentCommandLineParser.Setup(arg => arg.Quiet) .As('q') .WithDescription( "Do not dump full details about each file processed. Speeds up processing when using --json or --csv\r\n") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.JsonDirectory) .As("json") .WithDescription( "Directory to save json representation to. Use --pretty for a more human readable layout"); _fluentCommandLineParser.Setup(arg => arg.CsvDirectory) .As("csv") .WithDescription( "Directory to save CSV results to. Be sure to include the full path in double quotes"); _fluentCommandLineParser.Setup(arg => arg.xHtmlDirectory) .As("html") .WithDescription( "Directory to save xhtml formatted results to. Be sure to include the full path in double quotes"); _fluentCommandLineParser.Setup(arg => arg.JsonPretty) .As("pretty") .WithDescription( "When exporting to json, use a more human readable layout\r\n").SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.CsvSeparator) .As("cs") .WithDescription( "When true, use comma instead of tab for field separator. Default is true").SetDefault(true); _fluentCommandLineParser.Setup(arg => arg.DateTimeFormat) .As("dt") .WithDescription( "The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss") .SetDefault("yyyy-MM-dd HH:mm:ss"); _fluentCommandLineParser.Setup(arg => arg.PreciseTimestamps) .As("mp") .WithDescription( "When true, display higher precision for timestamps. Default is false").SetDefault(false); var header = $"PECmd version {Assembly.GetExecutingAssembly().GetName().Version}" + "\r\n\r\nAuthor: Eric Zimmerman ([email protected])" + "\r\nhttps://github.com/EricZimmerman/PECmd"; var footer = @"Examples: PECmd.exe -f ""C:\Temp\CALC.EXE-3FBEF7FD.pf""" + "\r\n\t " + @" PECmd.exe -f ""C:\Temp\CALC.EXE-3FBEF7FD.pf"" --json ""D:\jsonOutput"" --jsonpretty" + "\r\n\t " + @" PECmd.exe -d ""C:\Temp"" -k ""system32, fonts""" + "\r\n\t " + @" PECmd.exe -d ""C:\Temp"" --csv ""c:\temp"" --json c:\temp\json" + "\r\n\t " + @" PECmd.exe -d ""C:\Windows\Prefetch""" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help") .WithHeader(header) .Callback(text => _logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _logger.Error(""); _logger.Error(result.ErrorText); _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.File) && UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.Directory)) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); _logger.Warn("Either -f or -d is required. Exiting"); return; } if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.File) == false && !File.Exists(_fluentCommandLineParser.Object.File)) { _logger.Warn($"File '{_fluentCommandLineParser.Object.File}' not found. Exiting"); return; } if (UsefulExtension.IsNullOrEmpty(_fluentCommandLineParser.Object.Directory) == false && !Directory.Exists(_fluentCommandLineParser.Object.Directory)) { _logger.Warn($"Directory '{_fluentCommandLineParser.Object.Directory}' not found. Exiting"); return; } if (_fluentCommandLineParser.Object.Keywords?.Length > 0) { var kws = _fluentCommandLineParser.Object.Keywords.ToLowerInvariant().Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var kw in kws) { _keywords.Add(kw.Trim()); } } if (_fluentCommandLineParser.Object.CsvSeparator) { _exportExt = "csv"; } _logger.Info(header); _logger.Info(""); _logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}"); if (IsAdministrator() == false) { _logger.Fatal("\r\nWarning: Administrator privileges not found!"); } _logger.Info(""); _logger.Info($"Keywords: {string.Join(", ", _keywords)}"); _logger.Info(""); if (_fluentCommandLineParser.Object.PreciseTimestamps) { _fluentCommandLineParser.Object.DateTimeFormat = _preciseTimeFormat; } _processedFiles = new List <IPrefetch>(); _failedFiles = new List <string>(); if (_fluentCommandLineParser.Object.File?.Length > 0) { IPrefetch pf = null; try { pf = LoadFile(_fluentCommandLineParser.Object.File); if (pf != null) { if (_fluentCommandLineParser.Object.OutFile.IsNullOrEmpty() == false) { try { if (Directory.Exists(Path.GetDirectoryName(_fluentCommandLineParser.Object.OutFile)) == false) { Directory.CreateDirectory( Path.GetDirectoryName(_fluentCommandLineParser.Object.OutFile)); } PrefetchFile.SavePrefetch(_fluentCommandLineParser.Object.OutFile, pf); _logger.Info($"Saved prefetch bytes to '{_fluentCommandLineParser.Object.OutFile}'"); } catch (Exception e) { _logger.Error($"Unable to save prefetch file. Error: {e.Message}"); } } _processedFiles.Add(pf); } } catch (UnauthorizedAccessException ex) { _logger.Error( $"Unable to access '{_fluentCommandLineParser.Object.File}'. Are you running as an administrator? Error: {ex.Message}"); } catch (Exception ex) { _logger.Error( $"Error getting prefetch files in '{_fluentCommandLineParser.Object.Directory}'. Error: {ex.Message}"); } } else { _logger.Info($"Looking for prefetch files in '{_fluentCommandLineParser.Object.Directory}'"); _logger.Info(""); string[] pfFiles = null; try { pfFiles = Directory.GetFiles(_fluentCommandLineParser.Object.Directory, "*.pf", SearchOption.AllDirectories); } catch (UnauthorizedAccessException ua) { _logger.Error( $"Unable to access '{_fluentCommandLineParser.Object.Directory}'. Are you running as an administrator? Error: {ua.Message}"); return; } catch (Exception ex) { _logger.Error( $"Error getting prefetch files in '{_fluentCommandLineParser.Object.Directory}'. Error: {ex.Message}"); return; } _logger.Info($"Found {pfFiles.Length:N0} Prefetch files"); _logger.Info(""); var sw = new Stopwatch(); sw.Start(); foreach (var file in pfFiles) { var pf = LoadFile(file); if (pf != null) { _processedFiles.Add(pf); } } sw.Stop(); if (_fluentCommandLineParser.Object.Quiet) { _logger.Info(""); } _logger.Info( $"Processed {pfFiles.Length - _failedFiles.Count:N0} out of {pfFiles.Length:N0} files in {sw.Elapsed.TotalSeconds:N4} seconds"); if (_failedFiles.Count > 0) { _logger.Info(""); _logger.Warn("Failed files"); foreach (var failedFile in _failedFiles) { _logger.Info($" {failedFile}"); } } } if (_processedFiles.Count > 0) { _logger.Info(""); try { CsvWriter csv = null; StreamWriter streamWriter = null; CsvWriter csvTl = null; StreamWriter streamWriterTl = null; if (_fluentCommandLineParser.Object.CsvDirectory?.Length > 0) { var outName = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_PECmd_Output.{_exportExt}"; var outNameTl = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_PECmd_Output_Timeline.{_exportExt}"; var outFile = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outName); var outFileTl = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outNameTl); if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false) { _logger.Warn( $"Path to '{_fluentCommandLineParser.Object.CsvDirectory}' does not exist. Creating..."); Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory); } _logger.Warn($"CSV output will be saved to '{outFile}'"); _logger.Warn($"CSV time line output will be saved to '{outFileTl}'"); try { streamWriter = new StreamWriter(outFile); csv = new CsvWriter(streamWriter); if (_fluentCommandLineParser.Object.CsvSeparator == false) { csv.Configuration.Delimiter = "\t"; } csv.WriteHeader(typeof(CsvOut)); csv.NextRecord(); streamWriterTl = new StreamWriter(outFileTl); csvTl = new CsvWriter(streamWriterTl); if (_fluentCommandLineParser.Object.CsvSeparator == false) { csvTl.Configuration.Delimiter = "\t"; } csvTl.WriteHeader(typeof(CsvOutTl)); csvTl.NextRecord(); } catch (Exception ex) { _logger.Error( $"Unable to open '{outFile}' for writing. CSV export canceled. Error: {ex.Message}"); } } if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0) { if (Directory.Exists(_fluentCommandLineParser.Object.JsonDirectory) == false) { _logger.Warn( $"'{_fluentCommandLineParser.Object.JsonDirectory} does not exist. Creating...'"); Directory.CreateDirectory(_fluentCommandLineParser.Object.JsonDirectory); } _logger.Warn($"Saving json output to '{_fluentCommandLineParser.Object.JsonDirectory}'"); } XmlTextWriter xml = null; if (_fluentCommandLineParser.Object.xHtmlDirectory?.Length > 0) { if (Directory.Exists(_fluentCommandLineParser.Object.xHtmlDirectory) == false) { _logger.Warn( $"'{_fluentCommandLineParser.Object.xHtmlDirectory} does not exist. Creating...'"); Directory.CreateDirectory(_fluentCommandLineParser.Object.xHtmlDirectory); } var outDir = Path.Combine(_fluentCommandLineParser.Object.xHtmlDirectory, $"{DateTimeOffset.UtcNow:yyyyMMddHHmmss}_PECmd_Output_for_{_fluentCommandLineParser.Object.xHtmlDirectory.Replace(@":\", "_").Replace(@"\", "_")}"); if (Directory.Exists(outDir) == false) { Directory.CreateDirectory(outDir); } var styleDir = Path.Combine(outDir, "styles"); if (Directory.Exists(styleDir) == false) { Directory.CreateDirectory(styleDir); } File.WriteAllText(Path.Combine(styleDir, "normalize.css"), Resources.normalize); File.WriteAllText(Path.Combine(styleDir, "style.css"), Resources.style); Resources.directories.Save(Path.Combine(styleDir, "directories.png")); Resources.filesloaded.Save(Path.Combine(styleDir, "filesloaded.png")); var outFile = Path.Combine(_fluentCommandLineParser.Object.xHtmlDirectory, outDir, "index.xhtml"); _logger.Warn($"Saving HTML output to '{outFile}'"); xml = new XmlTextWriter(outFile, Encoding.UTF8) { Formatting = Formatting.Indented, Indentation = 4 }; xml.WriteStartDocument(); xml.WriteProcessingInstruction("xml-stylesheet", "href=\"styles/normalize.css\""); xml.WriteProcessingInstruction("xml-stylesheet", "href=\"styles/style.css\""); xml.WriteStartElement("document"); } if (_fluentCommandLineParser.Object.CsvDirectory.IsNullOrEmpty() == false || _fluentCommandLineParser.Object.JsonDirectory.IsNullOrEmpty() == false || _fluentCommandLineParser.Object.xHtmlDirectory.IsNullOrEmpty() == false) { foreach (var processedFile in _processedFiles) { var o = GetCsvFormat(processedFile); try { foreach (var dateTimeOffset in processedFile.LastRunTimes) { var t = new CsvOutTl(); var exePath = processedFile.Filenames.FirstOrDefault( y => y.EndsWith(processedFile.Header.ExecutableFilename)); if (exePath == null) { exePath = processedFile.Header.ExecutableFilename; } t.ExecutableName = exePath; t.RunTime = dateTimeOffset.ToString(_fluentCommandLineParser.Object.DateTimeFormat); csvTl?.WriteRecord(t); csvTl?.NextRecord(); } } catch (Exception ex) { _logger.Error( $"Error getting time line record for '{processedFile.SourceFilename}' to '{_fluentCommandLineParser.Object.CsvDirectory}'. Error: {ex.Message}"); } try { csv?.WriteRecord(o); csv?.NextRecord(); } catch (Exception ex) { _logger.Error( $"Error writing CSV record for '{processedFile.SourceFilename}' to '{_fluentCommandLineParser.Object.CsvDirectory}'. Error: {ex.Message}"); } if (_fluentCommandLineParser.Object.JsonDirectory?.Length > 0) { SaveJson(processedFile, _fluentCommandLineParser.Object.JsonPretty, _fluentCommandLineParser.Object.JsonDirectory); } //XHTML xml?.WriteStartElement("Container"); xml?.WriteElementString("SourceFile", o.SourceFilename); xml?.WriteElementString("SourceCreated", o.SourceCreated); xml?.WriteElementString("SourceModified", o.SourceModified); xml?.WriteElementString("SourceAccessed", o.SourceAccessed); xml?.WriteElementString("LastRun", o.LastRun); xml?.WriteElementString("PreviousRun0", $"{o.PreviousRun0}"); xml?.WriteElementString("PreviousRun1", $"{o.PreviousRun1}"); xml?.WriteElementString("PreviousRun2", $"{o.PreviousRun2}"); xml?.WriteElementString("PreviousRun3", $"{o.PreviousRun3}"); xml?.WriteElementString("PreviousRun4", $"{o.PreviousRun4}"); xml?.WriteElementString("PreviousRun5", $"{o.PreviousRun5}"); xml?.WriteElementString("PreviousRun6", $"{o.PreviousRun6}"); xml?.WriteStartElement("ExecutableName"); xml?.WriteAttributeString("title", "Note: The name of the executable tracked by the pf file"); xml?.WriteString(o.ExecutableName); xml?.WriteEndElement(); xml?.WriteElementString("RunCount", $"{o.RunCount}"); xml?.WriteStartElement("Size"); xml?.WriteAttributeString("title", "Note: The size of the executable in bytes"); xml?.WriteString(o.Size); xml?.WriteEndElement(); xml?.WriteStartElement("Hash"); xml?.WriteAttributeString("title", "Note: The calculated hash for the pf file that should match the hash in the source file name"); xml?.WriteString(o.Hash); xml?.WriteEndElement(); xml?.WriteStartElement("Version"); xml?.WriteAttributeString("title", "Note: The operating system that generated the prefetch file"); xml?.WriteString(o.Version); xml?.WriteEndElement(); xml?.WriteElementString("Note", o.Note); xml?.WriteElementString("Volume0Name", o.Volume0Name); xml?.WriteElementString("Volume0Serial", o.Volume0Serial); xml?.WriteElementString("Volume0Created", o.Volume0Created); xml?.WriteElementString("Volume1Name", o.Volume1Name); xml?.WriteElementString("Volume1Serial", o.Volume1Serial); xml?.WriteElementString("Volume1Created", o.Volume1Created); xml?.WriteStartElement("Directories"); xml?.WriteAttributeString("title", "A comma separated list of all directories accessed by the executable"); xml?.WriteString(o.Directories); xml?.WriteEndElement(); xml?.WriteStartElement("FilesLoaded"); xml?.WriteAttributeString("title", "A comma separated list of all files that were loaded by the executable"); xml?.WriteString(o.FilesLoaded); xml?.WriteEndElement(); xml?.WriteEndElement(); } //Close CSV stuff streamWriter?.Flush(); streamWriter?.Close(); streamWriterTl?.Flush(); streamWriterTl?.Close(); //Close XML xml?.WriteEndElement(); xml?.WriteEndDocument(); xml?.Flush(); } } catch (Exception ex) { _logger.Error($"Error exporting data! Error: {ex.Message}"); } } }
public Task OutputResultsAsync(IQueryRunner runner) { var dlg = new Microsoft.Win32.SaveFileDialog { DefaultExt = ".txt", Filter = "Tab separated text file|*.txt|Comma separated text file - UTF8|*.csv|Comma separated text file - Unicode|*.csv|Custom Export Format (Configure in Options)|*.csv" }; string fileName = ""; long durationMs = 0; // Show save file dialog box var result = dlg.ShowDialog(); // Process save file dialog box results if (result == true) { // Save document fileName = dlg.FileName; return(Task.Run(() => { try { runner.OutputMessage("Query Started"); var sw = Stopwatch.StartNew(); string sep = "\t"; bool shouldQuoteStrings = true; //default to quoting all string fields string decimalSep = System.Globalization.CultureInfo.CurrentUICulture.NumberFormat.CurrencyDecimalSeparator; string isoDateFormat = string.Format(Constants.IsoDateMask, decimalSep); var enc = Encoding.UTF8; switch (dlg.FilterIndex) { case 1: // tab separated sep = "\t"; break; case 2: // utf-8 csv sep = System.Globalization.CultureInfo.CurrentUICulture.TextInfo.ListSeparator; break; case 3: //unicode csv enc = Encoding.Unicode; sep = System.Globalization.CultureInfo.CurrentUICulture.TextInfo.ListSeparator; break; case 4: // TODO - custom export format sep = runner.Options.GetCustomCsvDelimiter(); shouldQuoteStrings = runner.Options.CustomCsvQuoteStringFields; break; } var daxQuery = runner.QueryText; var reader = runner.ExecuteDataReaderQuery(daxQuery); try { if (reader != null) { int iFileCnt = 1; var outputFilename = fileName; runner.OutputMessage("Command Complete, writing output file"); bool moreResults = true; while (moreResults) { int iMaxCol = reader.FieldCount - 1; int iRowCnt = 0; if (iFileCnt > 1) { outputFilename = AddFileCntSuffix(fileName, iFileCnt); } using (var textWriter = new System.IO.StreamWriter(outputFilename, false, enc)) { using (var csvWriter = new CsvHelper.CsvWriter(textWriter)) { // CSV Writer config csvWriter.Configuration.Delimiter = sep; // Datetime as ISOFormat csvWriter.Configuration.TypeConverterOptionsCache.AddOptions( typeof(DateTime), new CsvHelper.TypeConversion.TypeConverterOptions() { Formats = new string[] { isoDateFormat } }); // write out clean column names foreach (var colName in reader.CleanColumnNames()) { csvWriter.WriteField(colName); } csvWriter.NextRecord(); while (reader.Read()) { iRowCnt++; for (int iCol = 0; iCol < reader.FieldCount; iCol++) { var fieldValue = reader[iCol]; // quote all string fields if (reader.GetFieldType(iCol) == typeof(string)) { if (reader.IsDBNull(iCol)) { csvWriter.WriteField("", shouldQuoteStrings); } else { csvWriter.WriteField(fieldValue.ToString(), shouldQuoteStrings); } } else { csvWriter.WriteField(fieldValue); } } csvWriter.NextRecord(); if (iRowCnt % 1000 == 0) { runner.NewStatusBarMessage(string.Format("Written {0:n0} rows to the file output", iRowCnt)); } } } } runner.OutputMessage( string.Format("Query {2} Completed ({0:N0} row{1} returned)" , iRowCnt , iRowCnt == 1 ? "" : "s", iFileCnt) ); runner.RowCount = iRowCnt; moreResults = reader.NextResult(); iFileCnt++; } sw.Stop(); durationMs = sw.ElapsedMilliseconds; runner.SetResultsMessage("Query results written to file", OutputTargets.File); runner.ActivateOutput(); } else { runner.OutputError("Query Batch Completed with errors", durationMs); } } finally { if (reader != null) { reader.Dispose(); } } } catch (Exception ex) { runner.ActivateOutput(); runner.OutputError(ex.Message); #if DEBUG runner.OutputError(ex.StackTrace); #endif } finally { runner.OutputMessage("Query Batch Completed", durationMs); runner.QueryCompleted(); } })); } // else dialog was cancelled so return an empty task. return(Task.Run(() => { })); }
/// <summary> /// /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void contextExportUniqueSourceIp_Click(object sender, EventArgs e) { SaveFileDialog saveFileDialog = new SaveFileDialog(); saveFileDialog.Title = "Select the export CSV"; saveFileDialog.Filter = "TSV Files|*.tsv"; if (saveFileDialog.ShowDialog(this) == DialogResult.Cancel) { return; } List<Session> sessions = listSession.Objects.Cast<Session>().ToList(); var unique = sessions.Select(s => s.SrcIpText).Distinct(); CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = "\t"; using (FileStream fileStream = new FileStream(saveFileDialog.FileName, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { csvWriter.WriteField("Source IP"); csvWriter.NextRecord(); foreach (var ip in unique) { csvWriter.WriteField(ip); csvWriter.NextRecord(); } } }
/// <summary> /// /// </summary> /// <param name="events"></param> /// <param name="filePath">The output file name</param> public void ExportEventCurrent(List<Event> events, string filePath) { if (IsRunning == true) { OnExclamation("Already performing an export"); return; } IsRunning = true; new Thread(() => { try { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = '\t'; using (FileStream fileStream = new FileStream(filePath, FileMode.Append, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { // Write out the file headers csvWriter.WriteField("CID"); csvWriter.WriteField("Src IP"); csvWriter.WriteField("Src Port"); csvWriter.WriteField("Dst IP"); csvWriter.WriteField("Dst Port"); csvWriter.WriteField("Protocol"); csvWriter.WriteField("Timestamp"); csvWriter.WriteField("TCP Flags"); csvWriter.WriteField("Payload (ASCII)"); csvWriter.NextRecord(); foreach (Event temp in events) { csvWriter.WriteField(temp.Cid); csvWriter.WriteField(temp.IpSrcTxt); csvWriter.WriteField(temp.SrcPort); csvWriter.WriteField(temp.IpDstTxt); csvWriter.WriteField(temp.DstPort); csvWriter.WriteField(temp.Protocol); csvWriter.WriteField(temp.Timestamp); csvWriter.WriteField(temp.TcpFlagsString); csvWriter.WriteField(temp.PayloadAscii); csvWriter.NextRecord(); } } OnComplete(); } catch (Exception ex) { OnError("An error occurred whilst performing the export: " + ex.Message); } finally { IsRunning = false; } }).Start(); }
/// <summary> /// /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void menuFileExportUrls_Click(object sender, EventArgs e) { FolderBrowserDialog fbd = new FolderBrowserDialog(); fbd.Description = "Select the export directory"; if (fbd.ShowDialog(this) == DialogResult.Cancel) { return; } string directory = fbd.SelectedPath; string fileName = System.IO.Path.Combine(directory, "urls.tsv"); new Thread(() => { MethodInvoker methodInvoker = delegate { using (new HourGlass(this)) { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.Delimiter = "\t"; using (FileStream fileStream = new FileStream(fileName, FileMode.Create, FileAccess.Write)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { csvWriter.WriteField("Host"); csvWriter.WriteField("Destination IP"); csvWriter.WriteField("URL"); csvWriter.NextRecord(); List<string> temp = new List<string>(); foreach (Session session in listSession.Objects) { if (File.Exists(System.IO.Path.Combine(this.dataDirectory, session.Guid + ".urls")) == false) { continue; } string[] urls = File.ReadAllLines(System.IO.Path.Combine(this.dataDirectory, session.Guid + ".urls")); foreach (string url in urls) { string tempUrl = url.Trim(); if (tempUrl.Length == 0) { continue; } if (temp.Contains(session.HttpHost + tempUrl) == false) { temp.Add(session.HttpHost + tempUrl); } csvWriter.WriteField(session.HttpHost); csvWriter.WriteField(session.DstIpText); csvWriter.WriteField(tempUrl); csvWriter.NextRecord(); } } temp.Sort(); foreach (string url in temp) { IO.WriteUnicodeTextToFile(url, System.IO.Path.Combine(directory, "urls.txt"), true); } } } }; if (this.InvokeRequired == true) { this.BeginInvoke(methodInvoker); } else { methodInvoker.Invoke(); } }).Start(); }
internal string createCsvWithoutStructure(IEnumerable<dynamic> data, CsvConfiguration configuration) { var guid = Guid.NewGuid().ToString(); var builder = new StringBuilder(); var writer = new CsvHelper.CsvWriter(new StringWriter(builder), configuration); if (data.Count() == 0) return null; var first = ToDictionary(guid, data.First()); foreach (var name in first.Keys) writer.WriteField(name); writer.NextRecord(); foreach (var datum in data) { var dict = ToDictionary(guid, datum); foreach (var key in first.Keys) writer.WriteField(dict[key].ToString()); writer.NextRecord(); } return builder.ToString(); }
static async Task Main() { var connectionString = "Data Source=/home/daniel/Downloads/unterdb_DE.db;Cache=Shared"; var query = "select id, tags from locations"; var hashSeparator = "|"; var nodesFilePath = "/home/daniel/Desktop/tag_nodes.csv"; var edgesFilePath = "/home/daniel/Desktop/tag_edges.csv"; var minCoOccurrence = 50; var index = new Index(); var posts = new SqlitePostReader( connectionString, query, hashSeparator); await Read(posts, to : index); var foundEdges = await CountCoOccurrences(index); var usedTags = new HashSet <string>(); var utf8WithoutBom = new UTF8Encoding(false); await using var edgesStream = File.Create(edgesFilePath); await using var edgesWriter = new StreamWriter(edgesStream, utf8WithoutBom); await using var edgesCsvWriter = new CsvHelper.CsvWriter( edgesWriter, new CsvConfiguration(CultureInfo.InvariantCulture)); edgesCsvWriter.WriteField("Source", true); edgesCsvWriter.WriteField("Target", true); edgesCsvWriter.WriteField("Weight", true); edgesCsvWriter.NextRecord(); foreach (var edge in foundEdges) { if (edge.Value.Item2 < minCoOccurrence) { continue; } usedTags.Add(edge.Key); usedTags.Add(edge.Value.Item1); edgesCsvWriter.WriteField(edge.Key, true); edgesCsvWriter.WriteField(edge.Value.Item1, true); edgesCsvWriter.WriteField(edge.Value.Item2.ToString(), true); edgesCsvWriter.NextRecord(); } await using var nodesStream = File.Create(nodesFilePath); await using var nodesWriter = new StreamWriter(nodesStream, utf8WithoutBom); await using var nodesCsvWriter = new CsvHelper.CsvWriter( nodesWriter, new CsvConfiguration(CultureInfo.InvariantCulture)); nodesCsvWriter.WriteField("Id", true); nodesCsvWriter.WriteField("Label", true); nodesCsvWriter.WriteField("Weight", true); nodesCsvWriter.NextRecord(); foreach (var tag in index.TagValuesByTagHash) { if (!usedTags.Contains(tag.Key)) // Skip all tags which have no links { continue; } var totalOccurrences = index.TotalOccurrencesByTagHash[tag.Key]; nodesCsvWriter.WriteField(tag.Key, true); nodesCsvWriter.WriteField(tag.Value, true); nodesCsvWriter.WriteField(totalOccurrences.ToString(), true); nodesCsvWriter.NextRecord(); } }
private static void Main(string[] args) { ExceptionlessClient.Default.Startup("7iL4b0Me7W8PbFflftqWgfQCIdf55flrT2O11zIP"); SetupNLog(); var logger = LogManager.GetCurrentClassLogger(); if (!CheckForDotnet46()) { logger.Warn(".net 4.6 not detected. Please install .net 4.6 and try again."); return; } _fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>(); _fluentCommandLineParser.Setup(arg => arg.SaveTo) .As("csv") .WithDescription("Directory to save results. Required") .Required(); _fluentCommandLineParser.Setup(arg => arg.HiveFile) .As('f') .WithDescription( "Full path to SYSTEM hive to process. If this option is not specified, the live Registry will be used") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.SortTimestamps) .As('t') .WithDescription("Sorts last modified timestamps in descending order\r\n") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.ControlSet) .As('c') .WithDescription("The ControlSet to parse. Default is to extract all control sets.") .SetDefault(-1); _fluentCommandLineParser.Setup(arg => arg.Debug) .As('d') .WithDescription("Debug mode") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.DateTimeFormat) .As("dt") .WithDescription( "The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss") .SetDefault("yyyy-MM-dd HH:mm:ss"); var header = $"AppCompatCache Parser version {Assembly.GetExecutingAssembly().GetName().Version}" + $"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" + $"\r\nhttps://github.com/EricZimmerman/AppCompatCacheParser"; var footer = @"Examples: AppCompatCacheParser.exe --csv c:\temp -t -c 2" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help").WithHeader(header).Callback(text => logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } var hiveToProcess = "Live Registry"; if (_fluentCommandLineParser.Object.HiveFile?.Length > 0) { hiveToProcess = _fluentCommandLineParser.Object.HiveFile; } logger.Info(header); logger.Info(""); logger.Info($"Processing hive '{hiveToProcess}'"); logger.Info(""); if (_fluentCommandLineParser.Object.Debug) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug); } try { var appCompat = new AppCompatCache.AppCompatCache(_fluentCommandLineParser.Object.HiveFile, _fluentCommandLineParser.Object.ControlSet); var outFileBase = string.Empty; if (_fluentCommandLineParser.Object.HiveFile?.Length > 0) { if (_fluentCommandLineParser.Object.ControlSet >= 0) { outFileBase = $"{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_ControlSet00{_fluentCommandLineParser.Object.ControlSet}_AppCompatCache.tsv"; } else { outFileBase = $"{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_AppCompatCache.tsv"; } } else { outFileBase = $"{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.tsv"; } if (Directory.Exists(_fluentCommandLineParser.Object.SaveTo) == false) { Directory.CreateDirectory(_fluentCommandLineParser.Object.SaveTo); } var outFilename = Path.Combine(_fluentCommandLineParser.Object.SaveTo, outFileBase); var sw = new StreamWriter(outFilename); var csv = new CsvWriter(sw); csv.Configuration.HasHeaderRecord = true; csv.Configuration.Delimiter = "\t"; var foo = csv.Configuration.AutoMap <CacheEntry>(); var o = new TypeConverterOptions { DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal }; csv.Configuration.TypeConverterOptionsCache.AddOptions <CacheEntry>(o); foo.Map(t => t.LastModifiedTimeUTC).ConvertUsing(t => t.LastModifiedTimeUTC.ToString(_fluentCommandLineParser.Object.DateTimeFormat)); foo.Map(t => t.CacheEntrySize).Ignore(); foo.Map(t => t.Data).Ignore(); foo.Map(t => t.InsertFlags).Ignore(); foo.Map(t => t.DataSize).Ignore(); foo.Map(t => t.LastModifiedFILETIMEUTC).Ignore(); foo.Map(t => t.PathSize).Ignore(); foo.Map(t => t.Signature).Ignore(); foo.Map(t => t.ControlSet).Index(0); foo.Map(t => t.CacheEntryPosition).Index(1); foo.Map(t => t.Path).Index(2); foo.Map(t => t.LastModifiedTimeUTC).Index(3); foo.Map(t => t.Executed).Index(4); csv.WriteHeader <CacheEntry>(); csv.NextRecord(); logger.Debug($"**** Found {appCompat.Caches.Count} caches"); if (appCompat.Caches.Any()) { foreach (var appCompatCach in appCompat.Caches) { if (_fluentCommandLineParser.Object.Debug) { appCompatCach.PrintDump(); } try { logger.Info( $"Found {appCompatCach.Entries.Count:N0} cache entries for {appCompat.OperatingSystem} in ControlSet00{appCompatCach.ControlSet}"); if (_fluentCommandLineParser.Object.SortTimestamps) { csv.WriteRecords(appCompatCach.Entries.OrderByDescending(t => t.LastModifiedTimeUTC)); } else { csv.WriteRecords(appCompatCach.Entries); } } catch (Exception ex) { logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}"); try { appCompatCach.PrintDump(); } catch (Exception ex1) { logger.Error($"Couldn't PrintDump {ex1.Message} Stack: {ex1.StackTrace}"); } } } sw.Flush(); sw.Close(); logger.Warn($"\r\nResults saved to '{outFilename}'\r\n"); } else { logger.Warn($"\r\nNo caches were found!\r\n"); } } catch (Exception ex) { logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}"); } }
private static void Main(string[] args) { ExceptionlessClient.Default.Startup("7iL4b0Me7W8PbFflftqWgfQCIdf55flrT2O11zIP"); SetupNLog(); var logger = LogManager.GetCurrentClassLogger(); _fluentCommandLineParser = new FluentCommandLineParser <ApplicationArguments>(); _fluentCommandLineParser.Setup(arg => arg.CsvDirectory) .As("csv") .WithDescription("Directory to save CSV formatted results to. Required") .Required(); _fluentCommandLineParser.Setup(arg => arg.CsvName) .As("csvf") .WithDescription("File name to save CSV formatted results to. When present, overrides default name\r\n"); _fluentCommandLineParser.Setup(arg => arg.HiveFile) .As('f') .WithDescription( "Full path to SYSTEM hive to process. If this option is not specified, the live Registry will be used") .SetDefault(string.Empty); _fluentCommandLineParser.Setup(arg => arg.SortTimestamps) .As('t') .WithDescription("Sorts last modified timestamps in descending order\r\n") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.ControlSet) .As('c') .WithDescription("The ControlSet to parse. Default is to extract all control sets.") .SetDefault(-1); _fluentCommandLineParser.Setup(arg => arg.Debug) .As("debug") .WithDescription("Debug mode") .SetDefault(false); _fluentCommandLineParser.Setup(arg => arg.DateTimeFormat) .As("dt") .WithDescription( "The custom date/time format to use when displaying timestamps. See https://goo.gl/CNVq0k for options. Default is: yyyy-MM-dd HH:mm:ss") .SetDefault("yyyy-MM-dd HH:mm:ss"); _fluentCommandLineParser.Setup(arg => arg.NoTransLogs) .As("nl") .WithDescription( "When true, ignore transaction log files for dirty hives. Default is FALSE").SetDefault(false); var header = $"AppCompatCache Parser version {Assembly.GetExecutingAssembly().GetName().Version}" + $"\r\n\r\nAuthor: Eric Zimmerman ([email protected])" + $"\r\nhttps://github.com/EricZimmerman/AppCompatCacheParser"; var footer = @"Examples: AppCompatCacheParser.exe --csv c:\temp -t -c 2" + "\r\n\t " + @" AppCompatCacheParser.exe --csv c:\temp --csvf results.csv" + "\r\n\t " + "\r\n\t" + " Short options (single letter) are prefixed with a single dash. Long commands are prefixed with two dashes\r\n"; _fluentCommandLineParser.SetupHelp("?", "help").WithHeader(header).Callback(text => logger.Info(text + "\r\n" + footer)); var result = _fluentCommandLineParser.Parse(args); if (result.HelpCalled) { return; } if (result.HasErrors) { _fluentCommandLineParser.HelpOption.ShowHelp(_fluentCommandLineParser.Options); return; } var hiveToProcess = "Live Registry"; if (_fluentCommandLineParser.Object.HiveFile?.Length > 0) { hiveToProcess = _fluentCommandLineParser.Object.HiveFile; } logger.Info(header); logger.Info(""); logger.Info($"Command line: {string.Join(" ", Environment.GetCommandLineArgs().Skip(1))}\r\n"); if (IsAdministrator() == false) { logger.Fatal($"Warning: Administrator privileges not found!\r\n"); } logger.Info($"Processing hive '{hiveToProcess}'"); logger.Info(""); if (_fluentCommandLineParser.Object.Debug) { LogManager.Configuration.LoggingRules.First().EnableLoggingForLevel(LogLevel.Debug); } try { var appCompat = new AppCompatCache.AppCompatCache(_fluentCommandLineParser.Object.HiveFile, _fluentCommandLineParser.Object.ControlSet, _fluentCommandLineParser.Object.NoTransLogs); var outFileBase = string.Empty; var ts1 = DateTime.Now.ToString("yyyyMMddHHmmss"); if (_fluentCommandLineParser.Object.HiveFile?.Length > 0) { if (_fluentCommandLineParser.Object.ControlSet >= 0) { outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_ControlSet00{_fluentCommandLineParser.Object.ControlSet}_AppCompatCache.csv"; } else { outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(_fluentCommandLineParser.Object.HiveFile)}_AppCompatCache.csv"; } } else { outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.csv"; } if (_fluentCommandLineParser.Object.CsvName.IsNullOrEmpty() == false) { outFileBase = Path.GetFileName(_fluentCommandLineParser.Object.CsvName); } if (Directory.Exists(_fluentCommandLineParser.Object.CsvDirectory) == false) { Directory.CreateDirectory(_fluentCommandLineParser.Object.CsvDirectory); } var outFilename = Path.Combine(_fluentCommandLineParser.Object.CsvDirectory, outFileBase); var sw = new StreamWriter(outFilename); var csv = new CsvWriter(sw, CultureInfo.InvariantCulture); var foo = csv.Context.AutoMap <CacheEntry>(); var o = new TypeConverterOptions { DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal }; csv.Context.TypeConverterOptionsCache.AddOptions <CacheEntry>(o); foo.Map(t => t.LastModifiedTimeUTC).Convert(t => t.Value.LastModifiedTimeUTC.HasValue ? t.Value.LastModifiedTimeUTC.Value.ToString(_fluentCommandLineParser.Object.DateTimeFormat): ""); foo.Map(t => t.CacheEntrySize).Ignore(); foo.Map(t => t.Data).Ignore(); foo.Map(t => t.InsertFlags).Ignore(); foo.Map(t => t.DataSize).Ignore(); foo.Map(t => t.LastModifiedFILETIMEUTC).Ignore(); foo.Map(t => t.PathSize).Ignore(); foo.Map(t => t.Signature).Ignore(); foo.Map(t => t.ControlSet).Index(0); foo.Map(t => t.CacheEntryPosition).Index(1); foo.Map(t => t.Path).Index(2); foo.Map(t => t.LastModifiedTimeUTC).Index(3); foo.Map(t => t.Executed).Index(4); foo.Map(t => t.Duplicate).Index(5); foo.Map(t => t.SourceFile).Index(6); csv.WriteHeader <CacheEntry>(); csv.NextRecord(); logger.Debug($"**** Found {appCompat.Caches.Count} caches"); var cacheKeys = new HashSet <string>(); if (appCompat.Caches.Any()) { foreach (var appCompatCach in appCompat.Caches) { if (_fluentCommandLineParser.Object.Debug) { appCompatCach.PrintDump(); } try { logger.Info( $"Found {appCompatCach.Entries.Count:N0} cache entries for {appCompat.OperatingSystem} in ControlSet00{appCompatCach.ControlSet}"); if (_fluentCommandLineParser.Object.SortTimestamps) { // csv.WriteRecords(appCompatCach.Entries.OrderByDescending(t => t.LastModifiedTimeUTC)); foreach (var cacheEntry in appCompatCach.Entries) { cacheEntry.SourceFile = hiveToProcess; cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey()); cacheKeys.Add(cacheEntry.GetKey()); csv.WriteRecord(cacheEntry); csv.NextRecord(); } } else { foreach (var cacheEntry in appCompatCach.Entries) { cacheEntry.SourceFile = hiveToProcess; cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey()); cacheKeys.Add(cacheEntry.GetKey()); csv.WriteRecord(cacheEntry); csv.NextRecord(); } //csv.WriteRecords(appCompatCach.Entries); } } catch (Exception ex) { logger.Error($"There was an error: Error message: {ex.Message} Stack: {ex.StackTrace}"); try { appCompatCach.PrintDump(); } catch (Exception ex1) { logger.Error($"Couldn't PrintDump {ex1.Message} Stack: {ex1.StackTrace}"); } } } sw.Flush(); sw.Close(); logger.Warn($"\r\nResults saved to '{outFilename}'\r\n"); } else { logger.Warn($"\r\nNo caches were found!\r\n"); } } catch (Exception ex) { if (ex.Message.Contains("Sequence numbers do not match and transaction logs were not found in the same direct") == false) { if (ex.Message.Contains("Administrator privileges not found")) { logger.Fatal($"Could not access '{_fluentCommandLineParser.Object.HiveFile}'. Does it exist?"); logger.Error(""); logger.Fatal("Rerun the program with Administrator privileges to try again\r\n"); } else if (ex.Message.Contains("Invalid diskName:")) { logger.Fatal($"Could not access '{_fluentCommandLineParser.Object.HiveFile}'. Invalid disk!"); logger.Error(""); } else { logger.Error($"There was an error: {ex.Message}"); logger.Error($"Stacktrace: {ex.StackTrace}"); logger.Info(""); } } } }
public async Task GenerateAsync(IEnumerable <PackageInfo> packageList, Dictionary <string, LicenseRow> cachedLicenses, SpdxLicenseData spdxLicenseData) { var licenses = (await ToLicenseRowsAsync(packageList, cachedLicenses, spdxLicenseData)).OrderBy(p => p.Value.Id); switch (_options.FileType) { case FileType.Csv: using (var write = new StreamWriter(_options.Path)) { using (var helper = new CsvHelper.CsvWriter(write)) { helper.Configuration.SanitizeForInjection = true; helper.Configuration.QuoteAllFields = true; //Write headers foreach (var property in _options.Columns) { helper.WriteField(property, true); } helper.NextRecord(); var properties = typeof(LicenseRow).GetProperties().ToList(); //Write values foreach (var license in licenses) { //Get rid of new lines as they are bad in CSV and other report layouts. license.Value.LicenseText = license.Value?.LicenseText.Replace("\r", "").Replace("\n", "").Replace(",", " "); // if (license.Value?.LicenseText?.Length >= 35000) { license.Value.LicenseText = license.Value.LicenseText.Substring(0, 35000); } foreach (var propertyName in _options.Columns) { var property = properties.FirstOrDefault(p => p.Name == propertyName); var value = property?.GetValue(license.Value, null); helper.WriteField(value?.ToString()); } helper.NextRecord(); } } } break; case FileType.Html: throw new NotImplementedException(); case FileType.Pdf: throw new NotImplementedException(); case FileType.Word: throw new NotImplementedException(); default: throw new ArgumentOutOfRangeException(); } var proc = new Process(); var finfo = new FileInfo(_options.Path); if (finfo.Exists) { proc.StartInfo.FileName = finfo.FullName; proc.Start(); } }
/// <summary> /// /// </summary> /// <param name="dataDirectory"></param> /// <param name="outputDirectory"></param> public void PostProcess(string dataDirectory, string outputDirectory) { CsvConfiguration csvConfiguration = new CsvConfiguration(); csvConfiguration.QuoteAllFields = true; using (FileStream fileStream = new FileStream(System.IO.Path.Combine(outputDirectory, "Urls.csv"), FileMode.Append, FileAccess.Write, FileShare.Read)) using (StreamWriter streamWriter = new StreamWriter(fileStream)) using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvConfiguration)) { foreach (string file in System.IO.Directory.EnumerateFiles(outputDirectory, "*.xml", SearchOption.AllDirectories)) { string fileName = System.IO.Path.GetFileName(file); if (fileName.StartsWith("Url.Details.") == false) { continue; } UrlDetails urlDetails = new UrlDetails(); string ret = urlDetails.Load(file); if (ret.Length == 0) { foreach (string url in urlDetails.Urls) { csvWriter.WriteField(url); csvWriter.WriteField(urlDetails.SrcIp); csvWriter.WriteField(urlDetails.SrcPort); csvWriter.WriteField(urlDetails.DstIp); csvWriter.WriteField(urlDetails.DstPort); csvWriter.NextRecord(); } } } } }
public bool CreateCSvFile(Ijpk jpk) { string jpkPath = System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "JPK", "jpk.csv"); if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(jpkPath))) { System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(jpkPath)); } try { using (System.IO.TextWriter writeFile = new System.IO.StreamWriter(jpkPath)) { CsvHelper.CsvWriter csv = new CsvHelper.CsvWriter(writeFile); csv.Configuration.Delimiter = ";"; var fields = header.Split(";".ToCharArray()); foreach (string field in fields) { csv.WriteField(field); } csv.NextRecord(); csv.WriteField(jpk.KodFormularza); csv.WriteField(jpk.kodSystemowy); csv.WriteField(jpk.wersjaSchemy); csv.WriteField(jpk.WariantFormularza); csv.WriteField(jpk.CelZlozenia); csv.WriteField(jpk.DataWytworzeniaJPK.ToJPKDataUtworzenia()); csv.WriteField(jpk.DataOd.ToJPK()); csv.WriteField(jpk.DataDo.ToJPK()); csv.WriteField("My JPK"); //csv.WriteField(jpk.KodUrzedu); for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("NazwaSystemu"); i++) { csv.WriteField(""); } csv.NextRecord(); for (int i = 0; i < fields.ToList().IndexOf("NIP"); i++) { csv.WriteField(""); } csv.WriteField(jpk.NIP); csv.WriteField(jpk.PelnaNazwa); csv.WriteField(jpk.REGON); for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("Email"); i++) { csv.WriteField(""); } csv.NextRecord(); //for (int i = 0; i < fields.ToList().IndexOf("KodKraju"); i++) csv.WriteField(""); //csv.WriteField(jpk.KodKraju); //csv.WriteField(jpk.Wojewodztwo); //csv.WriteField(jpk.Powiat); //csv.WriteField(jpk.Gmina); //csv.WriteField(jpk.Ulica); //csv.WriteField(jpk.NrDomu); //csv.WriteField(jpk.NrLokalu); //csv.WriteField(jpk.Miejscowosc); //csv.WriteField(jpk.KodPocztowy); //csv.WriteField(jpk.Poczta); //for (int i = 0; i < fields.Count() - fields.ToList().IndexOf("Poczta"); i++) csv.WriteField(""); //csv.NextRecord(); FkRejSprzedazy(jpk, csv); FkRejZakupow(jpk, csv); } jpk.JPKMessage = (new StringBuilder()).AppendLine("Wygenerowano JPK csv").AppendLine(jpkPath).ToString(); return(true); } catch (Exception exp) { jpk.JPKMessage = exp.Message; } return(false); }
private static void ProcessFile(string file) { if (File.Exists(file) == false) { _logger.Warn($"'{file}' does not exist! Skipping"); return; } _logger.Warn($"\r\nProcessing '{file}'..."); Stream fileS; try { fileS = new FileStream(file, FileMode.Open, FileAccess.Read); } catch (Exception) { //file is in use if (Helper.IsAdministrator() == false) { _logger.Fatal("\r\nAdministrator privileges not found! Exiting!!\r\n"); Environment.Exit(0); } _logger.Warn($"\r\n'{file}' is in use. Rerouting..."); var files = new List <string>(); files.Add(file); var rawFiles = Helper.GetFiles(files); fileS = rawFiles.First().FileStream; } try { var evt = new EventLog(fileS); var seenRecords = 0; foreach (var eventRecord in evt.GetEventRecords()) { if (_includeIds.Count > 0) { if (_includeIds.Contains(eventRecord.EventId) == false) { //it is NOT in the list, so skip continue; } } else if (_excludeIds.Count > 0) { if (_excludeIds.Contains(eventRecord.EventId)) { //it IS in the list, so skip continue; } } eventRecord.SourceFile = file; try { _csvWriter?.WriteRecord(eventRecord); _csvWriter?.NextRecord(); var xml = string.Empty; if (_swXml != null) { xml = eventRecord.ConvertPayloadToXml(); _swXml.WriteLine(xml); } if (_swJson != null) { JsConfig.IncludeNullValues = true; JsConfig.DateHandler = DateHandler.ISO8601; var jsOut = eventRecord.ToJson(); if (_fluentCommandLineParser.Object.FullJson) { if (xml.IsNullOrEmpty()) { xml = eventRecord.ConvertPayloadToXml(); } var xd = new XmlDocument(); xd.LoadXml(xml); jsOut = JsonConvert.SerializeXmlNode(xd); } _swJson.WriteLine(jsOut); } seenRecords += 1; } catch (Exception e) { _logger.Error($"Error processing record #{eventRecord.RecordNumber}: {e.Message}"); } } if (evt.ErrorRecords.Count > 0) { _errorFiles.Add(file, evt.ErrorRecords.Count); } _fileCount += 1; _logger.Info(""); _logger.Fatal("Event log details"); _logger.Info(evt); _logger.Info($"Records processed: {seenRecords:N0} Errors: {evt.ErrorRecords.Count:N0}"); if (evt.ErrorRecords.Count > 0) { _logger.Warn("\r\nErrors"); foreach (var evtErrorRecord in evt.ErrorRecords) { _logger.Info($"Record #{evtErrorRecord.Key}: Error: {evtErrorRecord.Value}"); } } if (_fluentCommandLineParser.Object.Metrics && evt.EventIdMetrics.Count > 0) { _logger.Fatal("\r\nMetrics"); _logger.Warn("Event Id\tCount"); foreach (var esEventIdMetric in evt.EventIdMetrics.OrderBy(t => t.Key)) { if (_includeIds.Count > 0) { if (_includeIds.Contains((int)esEventIdMetric.Key) == false) { //it is NOT in the list, so skip continue; } } else if (_excludeIds.Count > 0) { if (_excludeIds.Contains((int)esEventIdMetric.Key)) { //it IS in the list, so skip continue; } } _logger.Info($"{esEventIdMetric.Key}\t\t{esEventIdMetric.Value:N0}"); } } } catch (Exception e) { if (e.Message.Contains("Invalid signature! Expected 'ElfFile")) { _logger.Info($"'{file}' is not an evtx file! Message: {e.Message} Skipping..."); } else { _logger.Error($"Error processing '{file}'! Message: {e.Message}"); } } fileS?.Close(); }
public static void WriteCSV(TextWriter writer, IEnumerable <WorkorderSummary> ws) { var csv = new CsvHelper.CsvWriter(writer); csv.WriteField("ID"); csv.WriteField("CompletedTimeUTC"); csv.WriteField("Part"); csv.WriteField("Quantity"); csv.WriteField("Serials"); var activeStations = new HashSet <string>(); var elapsedStations = new HashSet <string>(); foreach (var p in ws.SelectMany(w => w.Parts)) { foreach (var k in p.ActiveStationTime.Keys) { activeStations.Add(k); } foreach (var k in p.ElapsedStationTime.Keys) { elapsedStations.Add(k); } } var actualKeys = activeStations.OrderBy(x => x).ToList(); foreach (var k in actualKeys) { csv.WriteField("Active " + k + " (minutes)"); } var plannedKeys = elapsedStations.OrderBy(x => x).ToList(); foreach (var k in plannedKeys) { csv.WriteField("Elapsed " + k + " (minutes)"); } csv.NextRecord(); foreach (var w in ws) { foreach (var p in w.Parts) { csv.WriteField(w.WorkorderId); if (w.FinalizedTimeUTC.HasValue) { csv.WriteField(w.FinalizedTimeUTC.Value.ToString("yyyy-MM-ddTHH:mm:ssZ")); } else { csv.WriteField(""); } csv.WriteField(p.Part); csv.WriteField(p.PartsCompleted); csv.WriteField(string.Join(";", w.Serials)); foreach (var k in actualKeys) { if (p.ActiveStationTime.ContainsKey(k)) { csv.WriteField(p.ActiveStationTime[k].TotalMinutes); } else { csv.WriteField(0); } } foreach (var k in plannedKeys) { if (p.ElapsedStationTime.ContainsKey(k)) { csv.WriteField(p.ElapsedStationTime[k].TotalMinutes); } else { csv.WriteField(0); } } csv.NextRecord(); } } }
/// <summary> /// Uploads book cover images to ipfs /// Writes a mapping csv file to relate ean, image size, ipfs hash etc /// Assumes the book cover images are held locally so.... /// Ensure the cover images have been downloaded first! (DownloadCoversAsync) /// </summary> /// <param name="books"></param> /// <returns></returns> public static async Task UploadCoversToIpfsAsync( IEnumerable <Book> books, string coverFolder, string indexOutputFilePath, string ipfsUrl) { ipfsFilesLoaded = 0; //indexOutputFilePath try { // we'll parallel load images - but don't want to write to the csv in parallel var semaphore = new SemaphoreSlim(1); using (var textWriter = File.CreateText(indexOutputFilePath)) { var csvWriter = new CsvHelper.CsvWriter(textWriter, CultureInfo.InvariantCulture); csvWriter.WriteHeader <IpfsBookCoverMapping>(); csvWriter.NextRecord(); try { var ipfsService = new IpfsService(ipfsUrl); foreach (var book in books.AsParallel()) { if (!string.IsNullOrWhiteSpace(book.COVER_SMALL)) { var smallCover = await UploadCoverToIpfs(coverFolder, book.EAN, "s", book.COVER_SMALL, ipfsService); await Save(smallCover); } if (!string.IsNullOrWhiteSpace(book.COVER_MED)) { var mediumCover = await UploadCoverToIpfs(coverFolder, book.EAN, "m", book.COVER_MED, ipfsService); await Save(mediumCover); } if (!string.IsNullOrWhiteSpace(book.COVER_LARGE)) { var largeCover = await UploadCoverToIpfs(coverFolder, book.EAN, "l", book.COVER_LARGE, ipfsService); await Save(largeCover); } async Task Save(IpfsBookCoverMapping mapping) { await semaphore.WaitAsync(); try { csvWriter.WriteRecord(mapping); csvWriter.NextRecord(); } finally { semaphore.Release(); } } } } finally { await csvWriter.FlushAsync(); await textWriter.FlushAsync(); } } } finally { // consider changing file name on completion to avoid it being overwritten accidentally } }
public void MarkWorkorderAsFilled(string workorderId, DateTime filledUTC, WorkorderResources resources) { if (!Directory.Exists(Path.Combine(CSVBasePath, FilledWorkordersPath))) { Directory.CreateDirectory(Path.Combine(CSVBasePath, FilledWorkordersPath)); } using (var f = File.OpenWrite(Path.Combine(CSVBasePath, Path.Combine(FilledWorkordersPath, workorderId + ".csv")))) using (var stream = new StreamWriter(f)) using (var csv = new CsvHelper.CsvWriter(stream)) { csv.WriteField("CompletedTimeUTC"); csv.WriteField("ID"); csv.WriteField("Part"); csv.WriteField("Quantity"); csv.WriteField("Serials"); var activeStations = new HashSet <string>(); var elapsedStations = new HashSet <string>(); foreach (var p in resources.Parts) { foreach (var k in p.ActiveOperationTime.Keys) { activeStations.Add(k); } foreach (var k in p.ElapsedOperationTime.Keys) { elapsedStations.Add(k); } } var actualKeys = activeStations.OrderBy(x => x).ToList(); foreach (var k in actualKeys) { csv.WriteField("Active " + k + " (minutes)"); } var plannedKeys = elapsedStations.OrderBy(x => x).ToList(); foreach (var k in plannedKeys) { csv.WriteField("Elapsed " + k + " (minutes)"); } csv.NextRecord(); foreach (var p in resources.Parts) { csv.WriteField(filledUTC.ToString("yyyy-MM-ddTHH:mm:ssZ")); csv.WriteField(workorderId); csv.WriteField(p.Part); csv.WriteField(p.PartsCompleted); csv.WriteField(string.Join(";", resources.Serials)); foreach (var k in actualKeys) { if (p.ActiveOperationTime.ContainsKey(k)) { csv.WriteField(p.ActiveOperationTime[k].TotalMinutes); } else { csv.WriteField(0); } } foreach (var k in plannedKeys) { if (p.ElapsedOperationTime.ContainsKey(k)) { csv.WriteField(p.ElapsedOperationTime[k].TotalMinutes); } else { csv.WriteField(0); } } csv.NextRecord(); } } }