/// <summary>
///
/// </summary>
/// <param name="bindings"></param>
private static void OutputToFile(List <Binding> bindings)
{
using (FileStream fileStream = new FileStream(Path.Combine(fclp.Object.Output, "wmi-parser.tsv"), FileMode.Create, FileAccess.Write))
using (StreamWriter streamWriter = new StreamWriter(fileStream))
using (CsvWriter cw = new CsvHelper.CsvWriter(streamWriter))
{
cw.Configuration.Delimiter = "\t";
// Write out the file headers
cw.WriteField("Name");
cw.WriteField("Type");
cw.WriteField("Arguments");
cw.WriteField("Filter Name");
cw.WriteField("Filter Query");
cw.NextRecord();
foreach (var b in bindings)
{
cw.WriteField(b.Name);
cw.WriteField(b.Type);
cw.WriteField(b.Arguments);
cw.WriteField(b.Filter);
cw.WriteField(b.Query);
cw.NextRecord();;
}
}
}
/// <summary>
/// Write database records to text based file format.
/// </summary>
/// <param name="path">The fields file path.</param>
/// <param name="fs">The file system.</param>
/// <param name="database">The source records database.</param>
void ITextFieldWriter <XDatabase> .Write(string path, IFileSystem fs, XDatabase database)
{
using (var writer = new System.IO.StringWriter())
{
var configuration = new CSV.Configuration.CsvConfiguration();
configuration.Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator;
configuration.CultureInfo = CultureInfo.CurrentCulture;
using (var csv = new CSV.CsvWriter(writer, configuration))
{
// columns
csv.WriteField(database.IdColumnName);
foreach (var column in database.Columns)
{
csv.WriteField(column.Name);
}
csv.NextRecord();
// records
foreach (var record in database.Records)
{
csv.WriteField(record.Id.ToString());
foreach (var value in record.Values)
{
csv.WriteField(value.Content);
}
csv.NextRecord();
}
}
fs.WriteUtf8Text(path, writer.ToString());
}
}
public ActionResult DownloadCareManagers()
{
using (var writer = new System.IO.StreamWriter(Response.OutputStream, Encoding.GetEncoding("Shift-JIS")))
using (var csv = new CsvHelper.CsvWriter(writer))
{
csv.WriteField("事業所ID");
csv.WriteField("ケアマネID");
csv.WriteField("氏名");
csv.WriteField("登録者メールアドレス");
csv.WriteField("生年月日");
csv.WriteField("性別");
csv.WriteField("資格取得年月日");
csv.NextRecord();
foreach (var m in db.CareManagers.Where(m => m.UserId != null && m.Email != null).Include(m => m.CareHome).Include(m => m.User))
{
csv.WriteField(m.CareHome.CareHomeCode);
csv.WriteField(m.CareManagerId);
csv.WriteField(m.Name);
csv.WriteField(m.Email);
csv.WriteField(m.Birthday);
csv.WriteField(m.Gender);
csv.WriteField(m.Licensed);
//csv.WriteField(m.Licenses);
csv.NextRecord();
}
}
Response.ContentType = "text/csv";
Log(LogType.Admin, "CSVでケアマネ一覧をダウンロードしました。");
return(null);
}
protected override void DoPostWorkCompleted()
{
// write the item feed status into the CSV file
var resultFilePath = string.Format("{0}\\eBayProductsEndItemResults{1}.csv", _resultFileDirecctory, _systemJob.Id);
// write into the file
using (var streamWriter = new StreamWriter(resultFilePath))
{
var writer = new CsvHelper.CsvWriter(streamWriter);
// write the column headers
writer.WriteField("EisSKU");
writer.WriteField("ItemId");
writer.WriteField("Status");
writer.WriteField("Message");
writer.NextRecord();
foreach (var item in _itemFeeds)
{
writer.WriteField(item.EisSKU);
writer.WriteField(item.ItemId);
writer.WriteField(item.Status);
writer.WriteField(item.Message);
// move to the next row
writer.NextRecord();
}
}
// update the system job parameters out for the file path of the result file
_jobRepository.UpdateSystemJobParametersOut(_systemJob.Id, resultFilePath);
}
/// <summary>
/// Creates CSV-file to given path using Exportmodel as configurations and measurementModels as data.
/// </summary>
/// <param name="fullPath"></param>
/// <param name="options"></param>
/// <param name="measurements"></param>
public void CreateCsvFile(string filename, string directoryPath, ExportModel options, List <MeasurementModel> measurements)
{
_measurements = measurements;
try
{
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
using (TextWriter tw = new StreamWriter(directoryPath + filename))
{
using (var writer = new CsvHelper.CsvWriter(tw))
{
if (!string.IsNullOrEmpty(options.ColumnDelimeter))
{
writer.Configuration.Delimiter = options.ColumnDelimeter;
}
writer.Configuration.HasHeaderRecord = options.ShowHeaders;
var selected = CreateSelectedValueList(options);
if (options.ShowHeaders)
{
var headers = CreateHeaders(measurements, selected);
foreach (var h in headers)
{
writer.WriteField(h);
}
writer.NextRecord();
}
var functions = CreateDataValueFunctions(selected);
for (mIndex = 0; mIndex < measurements.Count; mIndex++)
{
writer.WriteField(measurements[mIndex].Object);
writer.WriteField(measurements[mIndex].Tag);
writer.WriteField(measurements[mIndex].Timestamp.ToString(options.Datetimeformat));
writer.WriteField(measurements[mIndex] != null ? measurements[mIndex].Location?.ToString() : "");
writer.WriteField(measurements[mIndex].Note);
for (dIndex = 0; dIndex < measurements[mIndex].Data.Count; dIndex++)
{
foreach (var f in functions)
{
var result = f.Invoke();
writer.WriteField((object)result ?? (object)string.Empty);
}
}
writer.NextRecord();
}
}
}
}
catch (Exception)
{
throw;
}
}
public static string CreateCSV(IEnumerable <StringTableEntry> entries)
{
using (var textWriter = new System.IO.StringWriter())
{
// Generate the localised .csv file
// Use the invariant culture when writing the CSV
var csv = new CsvHelper.CsvWriter(
textWriter, // write into this stream
GetConfiguration() // use this configuration
);
var fieldNames = new[] {
"language",
"id",
"text",
"file",
"node",
"lineNumber",
"lock",
"comment",
};
foreach (var field in fieldNames)
{
csv.WriteField(field);
}
csv.NextRecord();
foreach (var entry in entries)
{
var values = new[] {
entry.Language,
entry.ID,
entry.Text,
entry.File,
entry.Node,
entry.LineNumber,
entry.Lock,
entry.Comment,
};
foreach (var value in values)
{
csv.WriteField(value);
}
csv.NextRecord();
}
return(textWriter.ToString());
}
}
public string GetAsCSV()
{
System.Text.StringBuilder sb = new System.Text.StringBuilder();
var m = new StringWriter(sb);
TextWriter tw = m;
var config = new CsvConfiguration(CultureInfo.CurrentCulture);
config.Delimiter = ColumnDelimiter;
var p = new CsvHelper.CsvWriter(tw, config);
//CSVLine Line;
foreach (var line in Lines)
{
foreach (var c in line.Columns)
{
p.WriteField(c);
}
p.NextRecord();
}
p.Flush();
return(sb.ToString());
}
public void EscreverArquivoCSV(string local, List <Estudante> estudantes)
{
using (StreamWriter sw = new StreamWriter(local, false, new UTF8Encoding(true)))
{
using (CsvWriter csvWriter = new CsvHelper.CsvWriter(sw, System.Globalization.CultureInfo.InvariantCulture))
{
csvWriter.WriteHeader <Estudante>();
csvWriter.NextRecord();
foreach (Estudante est in estudantes)
{
csvWriter.WriteRecord <Estudante>(est);
csvWriter.NextRecord();
}
}
}
}
bool ExportPrices(Message message)
{
try
{
string fileName = $"C:\\temp\\price_{DateTime.Now.ToString("ddMMyyyyhh")}.csv";
using (var sw = new StreamWriter(fileName, File.Exists(fileName)))
{
var writer = new CsvHelper.CsvWriter(sw, _csvConfig);
var result = JsonConvert.DeserializeObject <PDLPRCP>(Encoding.UTF8.GetString(message.Body));
writer.WriteField(result.StyleNumber);
writer.WriteField(result.ColourCode);
writer.WriteField(result.SizeCode);
writer.WriteField(result.CountryCode);
writer.WriteField(result.BaseCcyCurrentRet);
writer.WriteField(result.BaseCcyOriginalRet);
writer.NextRecord();
}
//if (!File.Exists(fileName))
//{
// using (var sw = new StreamWriter(fileName))
// {
// var writer = new CsvHelper.CsvWriter(sw, _csvConfig);
// var result = JsonConvert.DeserializeObject<PDLPRCP>(Encoding.UTF8.GetString(message.Body));
// writer.WriteField(result.StyleNumber);
// writer.WriteField(result.ColourCode);
// writer.WriteField(result.SizeCode);
// writer.WriteField(result.CountryCode);
// writer.WriteField(result.BaseCcyCurrentRet);
// writer.WriteField(result.BaseCcyOriginalRet);
// writer.NextRecord();
// }
//}
//else
//{
// using (var sw = new StreamWriter(fileName,true))
// {
// var writer = new CsvHelper.CsvWriter(sw, _csvConfig);
// writer.NextRecord();
// var result = JsonConvert.DeserializeObject<PDLPRCP>(Encoding.UTF8.GetString(message.Body));
// writer.WriteField(result.StyleNumber);
// writer.WriteField(result.ColourCode);
// writer.WriteField(result.SizeCode);
// writer.WriteField(result.CountryCode);
// writer.WriteField(result.BaseCcyCurrentRet);
// writer.WriteField(result.BaseCcyOriginalRet);
// writer.NextRecord();
// }
//}
return(true);
}
catch (Exception ex)
{
log.Error("Export prices: ", ex);
}
return(false);
}
/// <summary>
/// Writes a csv header from a datatable
/// </summary>
/// <param name="queue"></param>
/// <param name="csvWriter"></param>
public static void WriteCSVHeader(DataTable queue, CsvHelper.CsvWriter csvWriter)
{
foreach (DataColumn column in queue.Columns)
{
csvWriter.WriteField(column.ColumnName);
}
csvWriter.NextRecord();
}
public void WriteCSVFile(string path, List <Movie> mv)
{
using (StreamWriter sw = new StreamWriter(path, false, new UTF8Encoding(true)))
{
using (var cw = new CsvHelper.CsvWriter(sw, System.Globalization.CultureInfo.CurrentCulture))
{
cw.WriteHeader <Movie>();
cw.NextRecord();
foreach (Movie stu in mv)
{
cw.WriteRecord <Movie>(stu);
cw.NextRecord();
}
}
}
}
public static DataWriter <T> Create(string path)
{
var streamWriter = new StreamWriter(path);
var csvWriter = new CsvHelper.CsvWriter(streamWriter);
csvWriter.WriteHeader <T>();
csvWriter.NextRecord();
return(new DataWriter <T>(csvWriter));
}
public void Write(Stream stream, DatabaseViewModel?database)
{
if (database is null)
{
return;
}
using var writer = new StringWriter();
var configuration = new CSV.Configuration.CsvConfiguration(CultureInfo.CurrentCulture)
{
Delimiter = CultureInfo.CurrentCulture.TextInfo.ListSeparator
};
using (var csvWriter = new CSV.CsvWriter(writer, configuration))
{
// Columns
csvWriter.WriteField(database.IdColumnName);
foreach (var column in database.Columns)
{
csvWriter.WriteField(column.Name);
}
csvWriter.NextRecord();
// Records
foreach (var record in database.Records)
{
csvWriter.WriteField(record.Id);
foreach (var value in record.Values)
{
csvWriter.WriteField(value.Content);
}
csvWriter.NextRecord();
}
}
var csv = writer.ToString();
var fileSystem = _serviceProvider.GetService <IFileSystem>();
fileSystem?.WriteUtf8Text(stream, csv);
}
private static void SaveToCsv()
{
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
using (var tw = File.CreateText($"d:\\training_data.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<MetsDataRecord>();
csv.NextRecord();
csv.WriteRecords(metsRecords);
}
}
public static void WriteCollectionToCsv(IEnumerable records, Type classtype, string filename)
{
using (var sw = new StreamWriter(filename))
{
var csvHelper = new CsvHelper.CsvWriter(sw);
csvHelper.WriteHeader(classtype);
csvHelper.NextRecord();
csvHelper.WriteRecords(records);
sw.Flush();
csvHelper.Flush();
}
}
static void SkapaIntCSV(List <OmniArtikel> artiklar, string path)
{
using (var stream = File.OpenWrite(path))
using (var writer = new StreamWriter(stream, Encoding.UTF8))
using (var csvWriter = new CsvHelper.CsvWriter(writer))
{
csvWriter.Configuration.Delimiter = ",";
csvWriter.Configuration.HasHeaderRecord = false;
// Headers
csvWriter.WriteField("x");
csvWriter.WriteField("y");
csvWriter.NextRecord();
foreach (var artikel in artiklar)
{
csvWriter.WriteField(TextTillIndexArray(artikel.Text.Trim()));
csvWriter.WriteField(KategorierTillIndexArray(artikel.Taggar.Select(t => t.TopicId).ToArray()));
csvWriter.NextRecord();
}
}
}
public static void WriteObjectToCsv(Metrics metrics, Type classtype, string filename)
{
using (var sw = new StreamWriter(filename))
{
var csvHelper = new CsvHelper.CsvWriter(sw);
csvHelper.WriteHeader(classtype);
csvHelper.NextRecord();
csvHelper.WriteField(metrics.Distance);
csvHelper.WriteField(metrics.Number);
sw.Flush();
csvHelper.Flush();
}
}
public ActionResult DownloadCareHomes()
{
using (var writer = new System.IO.StreamWriter(Response.OutputStream, Encoding.GetEncoding("Shift-JIS")))
using (var csv = new CsvHelper.CsvWriter(writer))
{
csv.WriteField("事業所ID");
csv.WriteField("事業所名");
csv.WriteField("登録者氏名");
csv.WriteField("登録者メールアドレス");
csv.NextRecord();
foreach (var c in db.CareHomes.Where(h => h.UserId != null && !h.Deactivated && h.Email != null).Include(h => h.User))
{
csv.WriteField(c.CareHomeCode);
csv.WriteField(c.Name);
csv.WriteField(c.User.Name);
csv.WriteField(c.Email);
csv.NextRecord();
}
}
Response.ContentType = "text/csv";
Log(LogType.Admin, "CSVで事業所一覧をダウンロードしました。");
return(null);
}
private string writeSuggestesCategoriesToFile(List <eBayCategoryResult> categoryResults)
{
// create the file to save the product categories
var fileFullPath = string.Format("{0}\\eBaySuggestedCateories_{1:yyyyMMdd_HHmmss}.csv", _systemJobDirectory, DateTime.Now);
// write into the file
using (var streamWriter = new StreamWriter(fileFullPath))
{
var writer = new CsvHelper.CsvWriter(streamWriter);
// let's write first the column's header for the file
writer.WriteField("EisSKU");
for (var i = 1; i <= 10; i++)
{
writer.WriteField(string.Format("eBayCategory{0}", i));
writer.WriteField(string.Format("FullPath{0}", i));
}
// let's move to next row
writer.NextRecord();
// then le't write its content
foreach (var result in categoryResults)
{
writer.WriteField(result.EisSKU);
foreach (var category in result.Categories)
{
writer.WriteField(category.Id);
writer.WriteField(category.OptionName);
}
writer.NextRecord();
}
}
return(fileFullPath);
}
public void CsvHelper()
{
using (var str = new StringWriter())
{
using (var csv = new CH.CsvWriter(str))
{
csv.WriteHeader <Row>();
csv.NextRecord();
for (var i = 0; i < Repeat; i++)
{
csv.WriteRecords(ToWrite);
}
}
GC.KeepAlive(str.ToString());
}
}
public string GetCSVLine(string Delimiter = ",")
{
var sb = new StringBuilder();
var m = new StringWriter(sb);
TextWriter tw = m; // new StreamWriter(m);
var config = new CsvConfiguration(CultureInfo.CurrentCulture); //.Configuration();
config.Delimiter = Delimiter;
var p = new CsvHelper.CsvWriter(tw, config);
foreach (var c in Columns)
{
p.WriteField(c);
}
p.NextRecord();
p.Flush();
return(sb.ToString());
//var dbq = "\"";
//System.Text.StringBuilder sb = new System.Text.StringBuilder();
////string s;
//bool First = true;
//if (m_Columns == null)
//{
// m_Columns = new List<string>();
//}
//foreach (var s in Columns)
//{
// if (!First)
// sb.Append(Delimiter);
// else
// First = false;
// sb.Append(dbq + s.Replace(dbq, dbq + dbq) + dbq);
//}
//return sb.ToString();
}
private static void DoWork(string f, string csv, string csvf, int c, bool t, string dt, bool nl, bool debug, bool trace)
{
var levelSwitch = new LoggingLevelSwitch();
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
var hiveToProcess = "Live Registry";
if (f?.Length > 0)
{
hiveToProcess = f;
if (!File.Exists(f))
{
Log.Warning("'{F}' not found. Exiting", f);
return;
}
}
else
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Log.Fatal("Live Registry support is not available on non-Windows systems");
Environment.Exit(0);
//throw new NotSupportedException("Live Registry support is not available on non-Windows systems");
}
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", _args));
Console.WriteLine();
if (IsAdministrator() == false)
{
Log.Fatal($"Warning: Administrator privileges not found!");
Console.WriteLine();
}
Log.Information("Processing hive '{HiveToProcess}'", hiveToProcess);
Console.WriteLine();
try
{
var appCompat = new AppCompatCache.AppCompatCache(f,
c, nl);
string outFileBase;
var ts1 = DateTime.Now.ToString("yyyyMMddHHmmss");
if (f?.Length > 0)
{
if (c >= 0)
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(f)}_ControlSet00{c}_AppCompatCache.csv";
}
else
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(f)}_AppCompatCache.csv";
}
}
else
{
outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.csv";
}
if (csvf.IsNullOrEmpty() == false)
{
outFileBase = Path.GetFileName(csvf);
}
if (Directory.Exists(csv) == false)
{
Directory.CreateDirectory(csv !);
}
var outFilename = Path.Combine(csv, outFileBase);
var sw = new StreamWriter(outFilename);
var csvWriter = new CsvWriter(sw, CultureInfo.InvariantCulture);
var foo = csvWriter.Context.AutoMap <CacheEntry>();
var o = new TypeConverterOptions
{
DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal
};
csvWriter.Context.TypeConverterOptionsCache.AddOptions <CacheEntry>(o);
foo.Map(entry => entry.LastModifiedTimeUTC).Convert(entry => entry.Value.LastModifiedTimeUTC.HasValue ? entry.Value.LastModifiedTimeUTC.Value.ToString(dt): "");
foo.Map(entry => entry.CacheEntrySize).Ignore();
foo.Map(entry => entry.Data).Ignore();
foo.Map(entry => entry.InsertFlags).Ignore();
foo.Map(entry => entry.DataSize).Ignore();
foo.Map(entry => entry.LastModifiedFILETIMEUTC).Ignore();
foo.Map(entry => entry.PathSize).Ignore();
foo.Map(entry => entry.Signature).Ignore();
foo.Map(entry => entry.ControlSet).Index(0);
foo.Map(entry => entry.CacheEntryPosition).Index(1);
foo.Map(entry => entry.Path).Index(2);
foo.Map(entry => entry.LastModifiedTimeUTC).Index(3);
foo.Map(entry => entry.Executed).Index(4);
foo.Map(entry => entry.Duplicate).Index(5);
foo.Map(entry => entry.SourceFile).Index(6);
csvWriter.WriteHeader <CacheEntry>();
csvWriter.NextRecord();
Log.Debug("**** Found {Count} caches", appCompat.Caches.Count);
var cacheKeys = new HashSet <string>();
if (appCompat.Caches.Any())
{
foreach (var appCompatCach in appCompat.Caches)
{
Log.Verbose("Dumping cache details: {@Details}", appCompat);
try
{
Log.Information(
"Found {Count:N0} cache entries for {OperatingSystem} in {ControlSet}", appCompatCach.Entries.Count, appCompat.OperatingSystem, $"ControlSet00{appCompatCach.ControlSet}");
if (t)
{
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csvWriter.WriteRecord(cacheEntry);
csvWriter.NextRecord();
}
}
else
{
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csvWriter.WriteRecord(cacheEntry);
csvWriter.NextRecord();
}
}
}
catch (Exception ex)
{
Log.Error(ex, "There was an error: Error message: {Message}", ex.Message);
try
{
appCompatCach.PrintDump();
}
catch (Exception ex1)
{
Log.Error(ex1, "Couldn't PrintDump: {Message}", ex1.Message);
}
}
}
sw.Flush();
sw.Close();
Console.WriteLine();
Log.Warning("Results saved to '{OutFilename}'", outFilename);
Console.WriteLine();
}
else
{
Console.WriteLine();
Log.Warning("No caches were found!");
Console.WriteLine();
}
}
catch (Exception ex)
{
if (ex.Message.Contains("Sequence numbers do not match and transaction logs were not found in the same direct") == false)
{
if (ex.Message.Contains("Administrator privileges not found"))
{
Log.Fatal("Could not access '{F}'. Does it exist?", f);
Console.WriteLine();
Log.Fatal("Rerun the program with Administrator privileges to try again");
Console.WriteLine();
}
else if (ex.Message.Contains("Invalid diskName:"))
{
Log.Fatal("Could not access '{F}'. Invalid disk!", f);
Console.WriteLine();
}
else
{
Log.Error(ex, "There was an error: {Message}", ex.Message);
Console.WriteLine();
}
}
}
}
private static void Experiment2(int neuronCountPPV, int neuronCountNPV, int repeats)
{
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
var results = new List<ExperimentPredictiveValues>();
DateTime et = DateTime.Now;
for (int i = 0; i < repeats; i++)
{
var es = DateTime.Now;
var data = CreateData(metsRecords, 0.8, 0.1);
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" EXPERIMENT #{i:000}");
Console.Out.WriteLine("================================");
Console.Out.WriteLine();
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN PPV, NC:{neuronCountPPV}");
Console.Out.WriteLine("================================");
var ann_PPV_PV = Ann(data, neuronCountPPV, null, out var ann);
LogAPV(ann_PPV_PV);
var ann_NPV_PV = new AlgoritamPredictiveValues();
if (neuronCountNPV != neuronCountPPV)
{
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN NPV, NC:{neuronCountNPV}");
Console.Out.WriteLine("================================");
ann_NPV_PV = Ann(data, neuronCountNPV, null, out ann);
LogAPV(ann_NPV_PV);
}
var experimentPV = new ExperimentPredictiveValues()
{
AnnPpvLearnNPV = ann_PPV_PV.LearnNPV,
AnnPpvLearnPPV = ann_PPV_PV.LearnPPV,
AnnPpvLearnSENS = ann_PPV_PV.LearnSENS,
AnnPpvLearnSPEC = ann_PPV_PV.LearnSPEC,
AnnPpvValidateNPV = ann_PPV_PV.ValidateNPV,
AnnPpvValidatePPV = ann_PPV_PV.ValidatePPV,
AnnPpvValidateSENS = ann_PPV_PV.ValidateSENS,
AnnPpvValidateSPEC = ann_PPV_PV.ValidateSPEC,
AnnPpvTestNPV = ann_PPV_PV.TestNPV,
AnnPpvTestPPV = ann_PPV_PV.TestPPV,
AnnPpvTestSENS = ann_PPV_PV.TestSENS,
AnnPpvTestSPEC = ann_PPV_PV.TestSPEC,
AnnNpvLearnNPV = ann_NPV_PV.LearnNPV,
AnnNpvLearnPPV = ann_NPV_PV.LearnPPV,
AnnNpvLearnSENS = ann_NPV_PV.LearnSENS,
AnnNpvLearnSPEC = ann_NPV_PV.LearnSPEC,
AnnNpvValidateNPV = ann_NPV_PV.ValidateNPV,
AnnNpvValidatePPV = ann_NPV_PV.ValidatePPV,
AnnNpvValidateSENS = ann_NPV_PV.ValidateSENS,
AnnNpvValidateSPEC = ann_NPV_PV.ValidateSPEC,
AnnNpvTestNPV = ann_NPV_PV.TestNPV,
AnnNpvTestPPV = ann_NPV_PV.TestPPV,
AnnNpvTestSENS = ann_NPV_PV.TestSENS,
AnnNpvTestSPEC = ann_NPV_PV.TestSPEC,
};
LogEPV(experimentPV);
results.Add(experimentPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
LogAvgEPV(results);
using (var tw = File.CreateText($"d:\\mets_experiment_2_{neuronCountPPV}_{neuronCountNPV}_R{repeats}{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<ExperimentPredictiveValues>();
csv.NextRecord();
csv.WriteRecords(results);
}
}
private static void ProcessFile(string file, bool dedupe, bool fj, bool met)
{
if (File.Exists(file) == false)
{
Log.Warning("{File} does not exist! Skipping", file);
return;
}
if (file.StartsWith(VssDir))
{
Console.WriteLine();
Log.Information("Processing {Vss}", $"VSS{file.Replace($"{VssDir}\\", "")}");
}
else
{
Console.WriteLine();
Log.Information("Processing {File}...", file);
}
Stream fileS;
try
{
fileS = new FileStream(file, FileMode.Open, FileAccess.Read);
}
catch (Exception)
{
//file is in use
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Console.WriteLine();
Log.Fatal("Raw disk reads not supported on non-Windows platforms! Exiting!!");
Console.WriteLine();
Environment.Exit(0);
}
if (Helper.IsAdministrator() == false)
{
Console.WriteLine();
Log.Fatal("Administrator privileges not found! Exiting!!");
Console.WriteLine();
Environment.Exit(0);
}
if (file.StartsWith("\\\\"))
{
Log.Fatal($"Raw access across UNC shares is not supported! Run locally on the system or extract logs via other means and try again. Exiting");
Environment.Exit(0);
}
Console.WriteLine();
Log.Warning("{File} is in use. Rerouting...", file);
var files = new List <string>
{
file
};
var rawFiles = Helper.GetRawFiles(files);
fileS = rawFiles.First().FileStream;
}
try
{
if (dedupe)
{
var sha = Helper.GetSha1FromStream(fileS, 0);
fileS.Seek(0, SeekOrigin.Begin);
if (SeenHashes.Contains(sha))
{
Log.Debug("Skipping {File} as a file with SHA-1 {Sha} has already been processed", file, sha);
return;
}
SeenHashes.Add(sha);
}
EventLog.LastSeenTicks = 0;
var evt = new EventLog(fileS);
Log.Information("Chunk count: {ChunkCount:N0}, Iterating records...", evt.ChunkCount);
var seenRecords = 0;
var fsw = new Stopwatch();
fsw.Start();
foreach (var eventRecord in evt.GetEventRecords())
{
if (seenRecords % 10 == 0)
{
Console.Title = $"Processing chunk {eventRecord.ChunkNumber:N0} of {evt.ChunkCount} % complete: {(eventRecord.ChunkNumber / (double)evt.ChunkCount):P} Records found: {seenRecords:N0}";
}
if (_includeIds.Count > 0)
{
if (_includeIds.Contains(eventRecord.EventId) == false)
{
//it is NOT in the list, so skip
_droppedEvents += 1;
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains(eventRecord.EventId))
{
//it IS in the list, so skip
_droppedEvents += 1;
continue;
}
}
if (_startDate.HasValue)
{
if (eventRecord.TimeCreated < _startDate.Value)
{
//too old
Log.Debug("Dropping record Id {EventRecordId} with timestamp {TimeCreated} as its too old", eventRecord.EventRecordId, eventRecord.TimeCreated);
_droppedEvents += 1;
continue;
}
}
if (_endDate.HasValue)
{
if (eventRecord.TimeCreated > _endDate.Value)
{
//too new
Log.Debug("Dropping record Id {EventRecordId} with timestamp {TimeCreated} as its too new", eventRecord.EventRecordId, eventRecord.TimeCreated);
_droppedEvents += 1;
continue;
}
}
if (file.StartsWith(VssDir))
{
eventRecord.SourceFile = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
else
{
eventRecord.SourceFile = file;
}
try
{
var xdo = new XmlDocument();
xdo.LoadXml(eventRecord.Payload);
var payOut = JsonConvert.SerializeXmlNode(xdo);
eventRecord.Payload = payOut;
_csvWriter?.WriteRecord(eventRecord);
_csvWriter?.NextRecord();
var xml = string.Empty;
if (_swXml != null)
{
xml = eventRecord.ConvertPayloadToXml();
_swXml.WriteLine(xml);
}
if (_swJson != null)
{
var jsOut = eventRecord.ToJson();
if (fj)
{
if (xml.IsNullOrEmpty())
{
xml = eventRecord.ConvertPayloadToXml();
}
jsOut = GetPayloadAsJson(xml);
}
_swJson.WriteLine(jsOut);
}
seenRecords += 1;
}
catch (Exception e)
{
Log.Error("Error processing record #{RecordNumber}: {Message}", eventRecord.RecordNumber, e.Message);
evt.ErrorRecords.Add(21, e.Message);
}
}
fsw.Stop();
if (evt.ErrorRecords.Count > 0)
{
var fn = file;
if (file.StartsWith(VssDir))
{
fn = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
_errorFiles.Add(fn, evt.ErrorRecords.Count);
}
_fileCount += 1;
Console.WriteLine();
Log.Information("Event log details");
Log.Information("{Evt}", evt);
Log.Information("Records included: {SeenRecords:N0} Errors: {ErrorRecordsCount:N0} Events dropped: {DroppedEvents:N0}", seenRecords, evt.ErrorRecords.Count, _droppedEvents);
if (evt.ErrorRecords.Count > 0)
{
Console.WriteLine();
Log.Information("Errors");
foreach (var evtErrorRecord in evt.ErrorRecords)
{
Log.Information("Record #{Key}: Error: {Value}", evtErrorRecord.Key, evtErrorRecord.Value);
}
}
if (met && evt.EventIdMetrics.Count > 0)
{
Console.WriteLine();
Log.Information("Metrics (including dropped events)");
Log.Information("Event ID\tCount");
foreach (var esEventIdMetric in evt.EventIdMetrics.OrderBy(t => t.Key))
{
if (_includeIds.Count > 0)
{
if (_includeIds.Contains((int)esEventIdMetric.Key) == false)
{
//it is NOT in the list, so skip
continue;
}
}
else if (_excludeIds.Count > 0)
{
if (_excludeIds.Contains((int)esEventIdMetric.Key))
{
//it IS in the list, so skip
continue;
}
}
Log.Information("{Key}\t\t{Value:N0}", esEventIdMetric.Key, esEventIdMetric.Value);
}
}
}
catch (Exception e)
{
var fn = file;
if (file.StartsWith(VssDir))
{
fn = $"VSS{file.Replace($"{VssDir}\\", "")}";
}
if (e.Message.Contains("Invalid signature! Expected 'ElfFile"))
{
Log.Information("{Fn} is not an evtx file! Message: {Message} Skipping...", fn, e.Message);
}
else
{
Log.Error("Error processing {Fn}! Message: {Message}", fn, e.Message);
}
}
fileS?.Close();
}
/// <summary>
///
/// </summary>
/// <param name="options"></param>
/// <param name="cacheEntries"></param>
private static void PrintCacheEntries(Options options, List<CacheEntry> cacheEntries)
{
try
{
CsvHelper.CsvWriterOptions csvWriterOptions = SetCsvWriterConfig(options);
using (MemoryStream memoryStream = new MemoryStream())
using (StreamWriter streamWriter = new StreamWriter(memoryStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvWriterOptions))
{
// Write out the file headers
csvWriter.WriteField("File Path");
csvWriter.WriteField("File Name");
csvWriter.WriteField("URL");
csvWriter.WriteField("Codebase IP");
csvWriter.WriteField("Headers");
csvWriter.WriteField("Content Length");
csvWriter.WriteField("Modified");
csvWriter.WriteField("Expiration");
csvWriter.WriteField("Validation");
csvWriter.NextRecord();
var sorted = GetSortedCacheEntries(options, cacheEntries);
StringBuilder text = new StringBuilder();
foreach (CacheEntry cacheEntry in sorted)
{
text.AppendFormat("File Path: {0}{1}", cacheEntry.FilePath, Environment.NewLine);
text.AppendFormat("File Name: {0}{1}", cacheEntry.FileName, Environment.NewLine);
text.AppendFormat("URL: {0}{1}", cacheEntry.Url, Environment.NewLine);
text.AppendFormat("Codebase IP: {0}{1}", cacheEntry.CodebaseIp, Environment.NewLine);
text.AppendFormat("Headers: {1}{0}", cacheEntry.HeadersText, Environment.NewLine);
text.AppendFormat("Content Length: {0}{1}", cacheEntry.ContentLength, Environment.NewLine);
if (cacheEntry.LastModified.Year == 1970 | cacheEntry.LastModified.Year == 0001)
{
text.AppendFormat("Modified: {0}{1}", string.Empty, Environment.NewLine);
}
else
{
text.AppendFormat("Modified: {0}{1}", cacheEntry.LastModified.ToShortDateString() + " " + cacheEntry.LastModified.ToShortTimeString(), Environment.NewLine);
}
if (cacheEntry.ExpirationDate.Year == 1970 | cacheEntry.ExpirationDate.Year == 0001)
{
text.AppendFormat("Expiration: {0}{1}", string.Empty, Environment.NewLine);
}
else
{
text.AppendFormat("Expiration: {0}{1}", cacheEntry.ExpirationDate.ToShortDateString() + " " + cacheEntry.ExpirationDate.ToShortTimeString(), Environment.NewLine);
}
if (cacheEntry.ValidationTimestamp.Year == 1970 | cacheEntry.ValidationTimestamp.Year == 0001)
{
text.AppendFormat("Validation: {0}{1}", string.Empty, Environment.NewLine);
}
else
{
text.AppendFormat("Validation: {0}{1}", cacheEntry.ValidationTimestamp.ToShortDateString() + " " + cacheEntry.ValidationTimestamp.ToShortTimeString(), Environment.NewLine);
}
text.AppendLine(string.Empty);
csvWriter.WriteField(cacheEntry.FilePath);
csvWriter.WriteField(cacheEntry.FileName);
csvWriter.WriteField(cacheEntry.CodebaseIp);
csvWriter.WriteField(cacheEntry.Url);
csvWriter.WriteField(cacheEntry.HeadersText);
csvWriter.WriteField(cacheEntry.ContentLength);
if (cacheEntry.LastModified.Year == 1970 | cacheEntry.LastModified.Year == 0001)
{
csvWriter.WriteField(string.Empty);
}
else
{
csvWriter.WriteField(cacheEntry.LastModified.ToShortDateString() + " " + cacheEntry.LastModified.ToShortTimeString());
}
if (cacheEntry.ExpirationDate.Year == 1970 | cacheEntry.ExpirationDate.Year == 0001)
{
csvWriter.WriteField(string.Empty);
}
else
{
csvWriter.WriteField(cacheEntry.ExpirationDate.ToShortDateString() + " " + cacheEntry.ExpirationDate.ToShortTimeString());
}
if (cacheEntry.ValidationTimestamp.Year == 1970 | cacheEntry.ValidationTimestamp.Year == 0001)
{
csvWriter.WriteField(string.Empty);
}
else
{
csvWriter.WriteField(cacheEntry.ValidationTimestamp.ToShortDateString() + " " + cacheEntry.ValidationTimestamp.ToShortTimeString());
}
csvWriter.NextRecord();
}
string output = string.Empty;
memoryStream.Position = 0;
using (StreamReader streamReader = new StreamReader(memoryStream))
{
output = streamReader.ReadToEnd();
}
Console.Write(text);
if (options.Output.Length > 0)
{
string ret = IO.WriteTextToFile(output, options.Output, false);
if (ret.Length > 0)
{
Console.Write("An error occurred whilst outputting the CSV/TSV file: " + ret);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred: " + ex.Message);
}
}
private static Experiment3Output Experiment3(int neuronCount, decimal minAnnPPV, decimal minAnnNPV, int treeCount, decimal minForestPPV, decimal minForestNPV)
{
var result = new Experiment3Output();
result.NeuronCount = neuronCount;
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
DateTime et = DateTime.Now;
var currentMinPPV = 0m;
var currentMinNPV = 0m;
Data data = null;
var learningData = new List<AnnLearningRecord>();
var annPV = new AlgoritamPredictiveValues();
var forestPV = new AlgoritamPredictiveValues();
while (currentMinPPV < minAnnPPV || currentMinNPV < minAnnNPV || currentMinPPV <= currentMinNPV || currentMinPPV < minForestPPV || currentMinNPV < minForestNPV)
{
var es = DateTime.Now;
data = CreateData(metsRecords, 0.8, 0.1);
result.Means = data.Means;
result.StandardDeviations = data.StandardDeviations;
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN PPV, NC:{neuronCount}");
Console.Out.WriteLine("================================");
learningData.Clear();
annPV = Ann(data, neuronCount, learningData, out var ann);
currentMinPPV = annPV.TestPPV;
currentMinNPV = annPV.TestNPV;
result.Ann = ann;
LogAPV(annPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
var experimentPV = new ExperimentPredictiveValues()
{
AnnPpvLearnNPV = annPV.LearnNPV,
AnnPpvLearnPPV = annPV.LearnPPV,
AnnPpvValidateNPV = annPV.ValidateNPV,
AnnPpvValidatePPV = annPV.ValidatePPV,
AnnPpvTestNPV = annPV.TestNPV,
AnnPpvTestPPV = annPV.TestPPV,
};
using (var tw = File.CreateText($"d:\\mets_experiment_3_{neuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}_rndfrst.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<AnnLearningRecord>();
csv.NextRecord();
csv.WriteRecords(learningData);
}
using (var tw = File.CreateText($"d:\\mets_experiment_3_PV_{neuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}_rndfrst.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<ExperimentPredictiveValues>();
csv.NextRecord();
csv.WriteRecords(new ExperimentPredictiveValues[] { experimentPV });
}
return result;
}
private static void DoWork(string d, string f, bool q, string csv, string csvf, string dt, bool debug, bool trace)
{
ActiveDateTimeFormat = dt;
var formatter =
new DateTimeOffsetFormatter(CultureInfo.CurrentCulture);
var levelSwitch = new LoggingLevelSwitch();
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template, formatProvider: formatter)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
if (f.IsNullOrEmpty() &&
d.IsNullOrEmpty())
{
var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth);
var hc = new HelpContext(helpBld, _rootCommand, Console.Out);
helpBld.Write(hc);
Log.Warning("Either -f or -d is required. Exiting");
return;
}
if (f.IsNullOrEmpty() == false &&
!File.Exists(f))
{
Log.Warning("File {F} not found. Exiting", f);
return;
}
if (d.IsNullOrEmpty() == false &&
!Directory.Exists(d))
{
Log.Warning("Directory {D} not found. Exiting", d);
return;
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", Environment.GetCommandLineArgs().Skip(1)));
if (IsAdministrator() == false)
{
Log.Warning("Warning: Administrator privileges not found!");
Console.WriteLine();
}
_csvOuts = new List <CsvOut>();
_failedFiles = new List <string>();
var files = new List <string>();
var sw = new Stopwatch();
sw.Start();
if (f?.Length > 0)
{
files.Add(f);
}
else
{
Console.WriteLine();
Log.Information("Looking for files in {Dir}", d);
if (!q)
{
Console.WriteLine();
}
files = GetRecycleBinFiles(d);
}
Log.Information("Found {Count:N0} files. Processing...", files.Count);
if (!q)
{
Console.WriteLine();
}
foreach (var file in files)
{
ProcessFile(file, q, dt);
}
sw.Stop();
Console.WriteLine();
Log.Information(
"Processed {FailedFilesCount:N0} out of {Count:N0} files in {TotalSeconds:N4} seconds", files.Count - _failedFiles.Count, files.Count, sw.Elapsed.TotalSeconds);
Console.WriteLine();
if (_failedFiles.Count > 0)
{
Console.WriteLine();
Log.Information("Failed files");
foreach (var failedFile in _failedFiles)
{
Log.Information(" {FailedFile}", failedFile);
}
}
if (csv.IsNullOrEmpty() == false && files.Count > 0)
{
if (Directory.Exists(csv) == false)
{
Log.Information("{Csv} does not exist. Creating...", csv);
Directory.CreateDirectory(csv);
}
var outName = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_RBCmd_Output.csv";
if (csvf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(csvf);
}
var outFile = Path.Combine(csv, outName);
outFile =
Path.GetFullPath(outFile);
Log.Warning("CSV output will be saved to {Path}", Path.GetFullPath(outFile));
try
{
var sw1 = new StreamWriter(outFile);
var csvWriter = new CsvWriter(sw1, CultureInfo.InvariantCulture);
csvWriter.WriteHeader(typeof(CsvOut));
csvWriter.NextRecord();
foreach (var csvOut in _csvOuts)
{
csvWriter.WriteRecord(csvOut);
csvWriter.NextRecord();
}
sw1.Flush();
sw1.Close();
}
catch (Exception ex)
{
Log.Error(ex,
"Unable to open {OutFile} for writing. CSV export canceled. Error: {Message}", outFile, ex.Message);
}
}
}
/// <summary>
///
/// </summary>
/// <param name="options"></param>
/// <param name="hits"></param>
private static void PrintHits(Options options, List<Hit> hits)
{
try
{
CsvHelper.CsvWriterOptions csvWriterOptions = SetCsvWriterConfig(options);
using (MemoryStream memoryStream = new MemoryStream())
using (StreamWriter streamWriter = new StreamWriter(memoryStream))
using (CsvHelper.CsvWriter csvWriter = new CsvHelper.CsvWriter(streamWriter, csvWriterOptions))
{
// Write out the file headers
csvWriter.WriteField("Last Modified");
csvWriter.WriteField("Last Update");
csvWriter.WriteField("Path");
csvWriter.WriteField("File Size");
csvWriter.WriteField("Process Exec Flag");
csvWriter.NextRecord();
var sorted = GetSortedHits(options, hits);
foreach (Hit hit in sorted)
{
switch (hit.Type)
{
case Global.CacheType.CacheEntryXp: // Windows XP Shim Cache
csvWriter.WriteField(hit.LastModified.ToShortDateString() + " " + hit.LastModified.ToShortTimeString());
csvWriter.WriteField(hit.LastUpdate.ToShortDateString() + " " + hit.LastUpdate.ToShortTimeString());
csvWriter.WriteField(hit.Path);
csvWriter.WriteField(hit.FileSize.ToString());
csvWriter.WriteField(hit.ProcessExecFlag);
csvWriter.NextRecord();
break;
case Global.CacheType.CacheEntryNt5: // Windows 2k3/Vista/2k8 Shim Cache
case Global.CacheType.CacheEntryNt6: // Windows 7/2k8-R2 Shim Cache
csvWriter.WriteField(hit.LastModified.ToShortDateString() + " " + hit.LastModified.ToShortTimeString());
csvWriter.WriteField("N/A");
csvWriter.WriteField(hit.Path);
csvWriter.WriteField("N/A");
csvWriter.WriteField(hit.ProcessExecFlag);
csvWriter.NextRecord();
break;
}
}
string output = string.Empty;
memoryStream.Position = 0;
using (StreamReader streamReader = new StreamReader(memoryStream))
{
output = streamReader.ReadToEnd();
}
Console.Write(output);
if (options.Output.Length > 0)
{
string ret = IO.WriteUnicodeTextToFile(output, options.Output, false);
if (ret.Length > 0)
{
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred: " + ex.Message);
}
}
public void ClassMap_ClassMapTest()
{
var sb = new StringBuilder()
.AppendLine("fuga,Hega")
//.AppendLine("124 ,,\"1\"")
//.AppendLine(",sss,\"2\"")
//.AppendLine()
.AppendLine("123,\"3,4\"")
;
Console.WriteLine(sb.ToString());
//using (var sr = new StringReader(sb.ToString()))
//using (var csv = new CsvHelper.CsvReader(sr))
//{
// csv.Configuration.PrepareHeaderForMatch = (header, index) => header.Trim().ToLower();
// csv.Configuration.RegisterClassMap<ClassMapImpl>();
// csv.Read();
// csv.ReadHeader();
// while (csv.Read())
// {
// var d = csv.GetRecord<Hoge>();
// Console.WriteLine("-" + d.Fuga.Code + "-");
// foreach (var h in csv.Context.HeaderRecord)
// {
// Console.WriteLine($"{h}, {csv.GetField(h)}");
// }
// }
// var records = csv.GetRecords<Hoge>();
// //foreach (var data in records)
// //{
// // Console.WriteLine(data.Fuga.Code);
// //}
//}
var srr = new StringWriter();
using (var csv = new CsvHelper.CsvWriter(srr))
{
csv.Configuration.RegisterClassMap <ClassMapImpl>();
var hoges = new List <Hoge>
{
new Hoge {
Fuga = new Fuga {
Code = "code"
}, Hega = new List <int> {
2, 3,
}
},
//new Hoge{ },
//new Hoge{Fuga = new Fuga{ Code = "11,333"} },
//new Hoge{Fuga = new Fuga{ Code = "hega\r\nfuga"} },
//new Hoge{Fuga = new Fuga{ Code = "11"} },
};
csv.WriteHeader <Hoge>();
csv.NextRecord();
foreach (var hoge in hoges)
{
csv.WriteRecord(hoge);
}
}
Console.WriteLine(srr.ToString());
}
private static void Experiment1()
{
Console.WriteLine("Start neuron count:");
var startNeuronCount = int.Parse(Console.ReadLine());
Console.WriteLine("End neuron count:");
var endNeuronCount = int.Parse(Console.ReadLine());
Console.WriteLine("Repeats:");
var repeats = int.Parse(Console.ReadLine());
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
var results = new List<Experiment2Results>();
DateTime et = DateTime.Now;
for (int neuronCount = startNeuronCount; neuronCount <= endNeuronCount; neuronCount++)
{
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" NEURON COUNT #{neuronCount:000}");
Console.Out.WriteLine("================================");
var iterationResults = new List<AlgoritamPredictiveValues>();
for (int i = 0; i < repeats; i++)
{
var es = DateTime.Now;
var data = CreateData(metsRecords, 0.8, 0.1);
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" EXPERIMENT #{i + 1:000}, NC{neuronCount:000}");
Console.Out.WriteLine("================================");
Console.Out.WriteLine();
var annPV = Ann(data, neuronCount, null, out var ann);
LogAPV(annPV);
iterationResults.Add(annPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
results.Add(new Experiment2Results()
{
NeuronCount = neuronCount,
LearnNPV = iterationResults.Average(x => x.LearnNPV),
LearnPPV = iterationResults.Average(x => x.LearnPPV),
ValidateNPV = iterationResults.Average(x => x.ValidateNPV),
ValidatePPV = iterationResults.Average(x => x.ValidatePPV),
TestNPV = iterationResults.Average(x => x.TestNPV),
TestPPV = iterationResults.Average(x => x.TestPPV),
});
}
using (var tw = File.CreateText($"d:\\mets_experiment_2_{startNeuronCount}_{endNeuronCount}_R{repeats}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment2Results>();
csv.NextRecord();
csv.WriteRecords(results);
}
}
private static void Experiment4(Experiment3Output exp3)
{
var metsRecords = LoadDataRecords(TEST_DATA_FILE, 0);
for (int i = 0; i < metsRecords.Count; i++)
{
var mets = CalculateMets(metsRecords[i]);
if (metsRecords[i].METS != mets)
throw new Exception("Invalid data");
}
if (metsRecords.Count > 100)
{
var mr = new List<MetsDataRecord>();
var rnd = new Random();
while (mr.Count < 100)
{
var index = rnd.Next(metsRecords.Count);
mr.Add(metsRecords[index]);
metsRecords.RemoveAt(index);
}
metsRecords = mr;
}
var inputs = CreateInputs(metsRecords).ToArray();
inputs = inputs.ZScores(exp3.Means, exp3.StandardDeviations);
var outputs = CreateOutputs(metsRecords).ToArray();
var annPredictions = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
{
var annr = exp3.Ann.Compute(inputs[i])[0];
var to = Math.Round(outputs[i][0], 0);
var anno = Math.Round(annr, 0);
annPredictions[i] = new double[] { anno };
}
var annPV = CalculatePredictiveValues(outputs, annPredictions);
var results = new List<Experiment4ResultRecord>();
for (int i = 0; i < metsRecords.Count; i++)
{
results.Add(new Experiment4ResultRecord()
{
AGE = metsRecords[i].AGE,
BMI = metsRecords[i].BMI,
DBP = metsRecords[i].DBP,
FPG = metsRecords[i].FPG,
GEN = metsRecords[i].GEN,
HDL = metsRecords[i].HDL,
HT = metsRecords[i].HT,
METS = metsRecords[i].METS,
METS_PR_ANN = (int)Math.Round(annPredictions[i][0]),
SBP = metsRecords[i].SBP,
TG = metsRecords[i].TG,
WC = metsRecords[i].WC,
WHtR = metsRecords[i].WHtR,
WT = metsRecords[i].WT
});
}
var experimentPV = new Experiment4PredictiveValues()
{
AnnNPV = annPV.NPV,
AnnPPV = annPV.PPV,
};
using (var tw = File.CreateText($"d:\\mets_experiment_4_{exp3.NeuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment4ResultRecord>();
csv.NextRecord();
csv.WriteRecords(results);
}
using (var tw = File.CreateText($"d:\\mets_experiment_4_PV_{exp3.NeuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment4PredictiveValues>();
csv.NextRecord();
csv.WriteRecords(new Experiment4PredictiveValues[] { experimentPV });
}
}
private static void DoWork(string f, string d, string csv, string csvf, string json, string jsonf, string xml, string xmlf, string dt, string inc, string exc, string sd, string ed, bool fj, int tdt, bool met, string maps, bool vss, bool dedupe, bool sync, bool debug, bool trace)
{
var levelSwitch = new LoggingLevelSwitch();
_activeDateTimeFormat = dt;
var formatter =
new DateTimeOffsetFormatter(CultureInfo.CurrentCulture);
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template, formatProvider: formatter)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
if (sync)
{
try
{
Log.Information("{Header}", Header);
UpdateFromRepo();
}
catch (Exception e)
{
Log.Error(e, "There was an error checking for updates: {Message}", e.Message);
}
Environment.Exit(0);
}
if (f.IsNullOrEmpty() &&
d.IsNullOrEmpty())
{
var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth);
var hc = new HelpContext(helpBld, _rootCommand, Console.Out);
helpBld.Write(hc);
Log.Warning("-f or -d is required. Exiting");
Console.WriteLine();
return;
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", Environment.GetCommandLineArgs().Skip(1)));
Console.WriteLine();
if (IsAdministrator() == false)
{
Log.Warning("Warning: Administrator privileges not found!");
Console.WriteLine();
}
if (vss & !RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
vss = false;
Log.Warning("{Vss} not supported on non-Windows platforms. Disabling...", "--vss");
Console.WriteLine();
}
if (vss & (IsAdministrator() == false))
{
Log.Error("{Vss} is present, but administrator rights not found. Exiting", "--vss");
Console.WriteLine();
return;
}
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (json.IsNullOrEmpty() == false)
{
if (Directory.Exists(json) == false)
{
Log.Information("Path to {Json} doesn't exist. Creating...", json);
try
{
Directory.CreateDirectory(json);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Json}. Does a file with the same name exist? Exiting", json);
Console.WriteLine();
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (jsonf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(jsonf);
}
var outFile = Path.Combine(json, outName);
Log.Information("json output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (xml.IsNullOrEmpty() == false)
{
if (Directory.Exists(xml) == false)
{
Log.Information("Path to {Xml} doesn't exist. Creating...", xml);
try
{
Directory.CreateDirectory(xml);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Xml}. Does a file with the same name exist? Exiting", xml);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (xmlf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(xmlf);
}
var outFile = Path.Combine(xml, outName);
Log.Information("XML output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
}
if (sd.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(sd, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_startDate = dateTimeOffset;
Log.Information("Setting Start date to {StartDate}", _startDate.Value);
}
else
{
Log.Warning("Could not parse {Sd} to a valid datetime! Events will not be filtered by Start date!", sd);
}
}
if (ed.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(ed, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_endDate = dateTimeOffset;
Log.Information("Setting End date to {EndDate}", _endDate.Value);
}
else
{
Log.Warning("Could not parse {Ed} to a valid datetime! Events will not be filtered by End date!", ed);
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
Console.WriteLine();
}
if (csv.IsNullOrEmpty() == false)
{
if (Directory.Exists(csv) == false)
{
Log.Information(
"Path to {Csv} doesn't exist. Creating...", csv);
try
{
Directory.CreateDirectory(csv);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Csv}. Does a file with the same name exist? Exiting", csv);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (csvf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(csvf);
}
var outFile = Path.Combine(csv, outName);
Log.Information("CSV output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
var opt = new CsvConfiguration(CultureInfo.InvariantCulture)
{
ShouldUseConstructorParameters = _ => false
};
_csvWriter = new CsvWriter(_swCsv, opt);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
var foo = _csvWriter.Context.AutoMap <EventRecord>();
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).Convert(t =>
$"{t.Value.TimeCreated.ToString(dt)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.HiddenRecord).Index(21);
foo.Map(t => t.SourceFile).Index(22);
foo.Map(t => t.Keywords).Index(23);
foo.Map(t => t.Payload).Index(24);
_csvWriter.Context.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(maps) == false)
{
Log.Warning("Maps directory {Maps} does not exist! Event ID maps will not be loaded!!", maps);
}
else
{
Log.Debug("Loading maps from {Path}", Path.GetFullPath(maps));
var errors = EventLog.LoadMaps(Path.GetFullPath(maps));
if (errors)
{
return;
}
Log.Information("Maps loaded: {Count:N0}", EventLog.EventLogMaps.Count);
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (exc.IsNullOrEmpty() == false)
{
var excSegs = exc.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (inc.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = inc.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (vss)
{
string driveLetter;
if (f.IsEmpty() == false)
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(f))
.Substring(0, 1);
}
else
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(d))
.Substring(0, 1);
}
Helper.MountVss(driveLetter, VssDir);
Console.WriteLine();
}
EventLog.TimeDiscrepancyThreshold = tdt;
if (f.IsNullOrEmpty() == false)
{
if (File.Exists(f) == false)
{
Log.Warning("\t{F} does not exist! Exiting", f);
Console.WriteLine();
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
dedupe = false;
ProcessFile(Path.GetFullPath(f), dedupe, fj, met);
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
var root = Path.GetPathRoot(Path.GetFullPath(f));
var stem = Path.GetFullPath(f).Replace(root, "");
foreach (var vssDir in vssDirs)
{
var newPath = Path.Combine(vssDir, stem);
if (File.Exists(newPath))
{
ProcessFile(newPath, dedupe, fj, met);
}
}
}
}
else
{
if (Directory.Exists(d) == false)
{
Log.Warning("\t{D} does not exist! Exiting", d);
Console.WriteLine();
return;
}
Log.Information("Looking for event log files in {D}", d);
Console.WriteLine();
#if !NET6_0
var directoryEnumerationFilters = new DirectoryEnumerationFilters
{
InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
};
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(d), dirEnumOptions, directoryEnumerationFilters);
#else
var enumerationOptions = new EnumerationOptions
{
IgnoreInaccessible = true,
MatchCasing = MatchCasing.CaseInsensitive,
RecurseSubdirectories = true,
AttributesToSkip = 0
};
var files2 =
Directory.EnumerateFileSystemEntries(d, "*.evtx", enumerationOptions);
#endif
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file, dedupe, fj, met);
}
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
Console.WriteLine();
foreach (var vssDir in vssDirs)
{
var root = Path.GetPathRoot(Path.GetFullPath(d));
var stem = Path.GetFullPath(d).Replace(root, "");
var target = Path.Combine(vssDir, stem);
Console.WriteLine();
Log.Information("Searching {Vss} for event logs...", $"VSS{target.Replace($"{VssDir}\\", "")}");
var vssFiles = Helper.GetFilesFromPath(target, "*.evtx", true);
foreach (var file in vssFiles)
{
ProcessFile(file, dedupe, fj, met);
}
}
}
}
try
{
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
}
catch (Exception e)
{
Log.Error(e, "Error when flushing output files to disk! Error message: {Message}", e.Message);
}
sw.Stop();
Console.WriteLine();
if (_fileCount == 1)
{
Log.Information("Processed {FileCount:N0} file in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
else
{
Log.Information("Processed {FileCount:N0} files in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
Console.WriteLine();
if (_errorFiles.Count > 0)
{
Console.WriteLine();
Log.Information("Files with errors");
foreach (var errorFile in _errorFiles)
{
Log.Information("{Key} error count: {Value:N0}", errorFile.Key, errorFile.Value);
}
Console.WriteLine();
}
if (vss)
{
if (Directory.Exists(VssDir))
{
foreach (var directory in Directory.GetDirectories(VssDir))
{
Directory.Delete(directory);
}
#if !NET6_0
Directory.Delete(VssDir, true, true);
#else
Directory.Delete(VssDir, true);
#endif
}
}
}