public static DataWriter <T> Create(string path)
{
var streamWriter = new StreamWriter(path);
var csvWriter = new CsvHelper.CsvWriter(streamWriter);
csvWriter.WriteHeader <T>();
csvWriter.NextRecord();
return(new DataWriter <T>(csvWriter));
}
public void WriteOutputFile <T>(IEnumerable <T> records, string outputFile)
{
using (var fileWritter = File.CreateText(outputFile))
{
CsvHelper.CsvWriter w = new CsvHelper.CsvWriter(fileWritter);
w.WriteHeader <T>();
w.WriteRecords(records);
}
}
private static void SaveToCsv()
{
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
using (var tw = File.CreateText($"d:\\training_data.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<MetsDataRecord>();
csv.NextRecord();
csv.WriteRecords(metsRecords);
}
}
public static void WriteCollectionToCsv(IEnumerable records, Type classtype, string filename)
{
using (var sw = new StreamWriter(filename))
{
var csvHelper = new CsvHelper.CsvWriter(sw);
csvHelper.WriteHeader(classtype);
csvHelper.NextRecord();
csvHelper.WriteRecords(records);
sw.Flush();
csvHelper.Flush();
}
}
public static void WriteObjectToCsv(Metrics metrics, Type classtype, string filename)
{
using (var sw = new StreamWriter(filename))
{
var csvHelper = new CsvHelper.CsvWriter(sw);
csvHelper.WriteHeader(classtype);
csvHelper.NextRecord();
csvHelper.WriteField(metrics.Distance);
csvHelper.WriteField(metrics.Number);
sw.Flush();
csvHelper.Flush();
}
}
public void EscreverArquivoCSV(string local, List <Estudante> estudantes)
{
using (StreamWriter sw = new StreamWriter(local, false, new UTF8Encoding(true)))
{
using (CsvWriter csvWriter = new CsvHelper.CsvWriter(sw, System.Globalization.CultureInfo.InvariantCulture))
{
csvWriter.WriteHeader <Estudante>();
csvWriter.NextRecord();
foreach (Estudante est in estudantes)
{
csvWriter.WriteRecord <Estudante>(est);
csvWriter.NextRecord();
}
}
}
}
public void WriteCSVFile(string path, List <Movie> mv)
{
using (StreamWriter sw = new StreamWriter(path, false, new UTF8Encoding(true)))
{
using (var cw = new CsvHelper.CsvWriter(sw, System.Globalization.CultureInfo.CurrentCulture))
{
cw.WriteHeader <Movie>();
cw.NextRecord();
foreach (Movie stu in mv)
{
cw.WriteRecord <Movie>(stu);
cw.NextRecord();
}
}
}
}
public void CsvHelper()
{
using (var str = new StringWriter())
{
using (var csv = new CH.CsvWriter(str))
{
csv.WriteHeader <Row>();
csv.NextRecord();
for (var i = 0; i < Repeat; i++)
{
csv.WriteRecords(ToWrite);
}
}
GC.KeepAlive(str.ToString());
}
}
private static void DoWork(string f, string csv, string csvf, int c, bool t, string dt, bool nl, bool debug, bool trace)
{
var levelSwitch = new LoggingLevelSwitch();
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
var hiveToProcess = "Live Registry";
if (f?.Length > 0)
{
hiveToProcess = f;
if (!File.Exists(f))
{
Log.Warning("'{F}' not found. Exiting", f);
return;
}
}
else
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Log.Fatal("Live Registry support is not available on non-Windows systems");
Environment.Exit(0);
//throw new NotSupportedException("Live Registry support is not available on non-Windows systems");
}
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", _args));
Console.WriteLine();
if (IsAdministrator() == false)
{
Log.Fatal($"Warning: Administrator privileges not found!");
Console.WriteLine();
}
Log.Information("Processing hive '{HiveToProcess}'", hiveToProcess);
Console.WriteLine();
try
{
var appCompat = new AppCompatCache.AppCompatCache(f,
c, nl);
string outFileBase;
var ts1 = DateTime.Now.ToString("yyyyMMddHHmmss");
if (f?.Length > 0)
{
if (c >= 0)
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(f)}_ControlSet00{c}_AppCompatCache.csv";
}
else
{
outFileBase =
$"{ts1}_{appCompat.OperatingSystem}_{Path.GetFileNameWithoutExtension(f)}_AppCompatCache.csv";
}
}
else
{
outFileBase = $"{ts1}_{appCompat.OperatingSystem}_{Environment.MachineName}_AppCompatCache.csv";
}
if (csvf.IsNullOrEmpty() == false)
{
outFileBase = Path.GetFileName(csvf);
}
if (Directory.Exists(csv) == false)
{
Directory.CreateDirectory(csv !);
}
var outFilename = Path.Combine(csv, outFileBase);
var sw = new StreamWriter(outFilename);
var csvWriter = new CsvWriter(sw, CultureInfo.InvariantCulture);
var foo = csvWriter.Context.AutoMap <CacheEntry>();
var o = new TypeConverterOptions
{
DateTimeStyle = DateTimeStyles.AssumeUniversal & DateTimeStyles.AdjustToUniversal
};
csvWriter.Context.TypeConverterOptionsCache.AddOptions <CacheEntry>(o);
foo.Map(entry => entry.LastModifiedTimeUTC).Convert(entry => entry.Value.LastModifiedTimeUTC.HasValue ? entry.Value.LastModifiedTimeUTC.Value.ToString(dt): "");
foo.Map(entry => entry.CacheEntrySize).Ignore();
foo.Map(entry => entry.Data).Ignore();
foo.Map(entry => entry.InsertFlags).Ignore();
foo.Map(entry => entry.DataSize).Ignore();
foo.Map(entry => entry.LastModifiedFILETIMEUTC).Ignore();
foo.Map(entry => entry.PathSize).Ignore();
foo.Map(entry => entry.Signature).Ignore();
foo.Map(entry => entry.ControlSet).Index(0);
foo.Map(entry => entry.CacheEntryPosition).Index(1);
foo.Map(entry => entry.Path).Index(2);
foo.Map(entry => entry.LastModifiedTimeUTC).Index(3);
foo.Map(entry => entry.Executed).Index(4);
foo.Map(entry => entry.Duplicate).Index(5);
foo.Map(entry => entry.SourceFile).Index(6);
csvWriter.WriteHeader <CacheEntry>();
csvWriter.NextRecord();
Log.Debug("**** Found {Count} caches", appCompat.Caches.Count);
var cacheKeys = new HashSet <string>();
if (appCompat.Caches.Any())
{
foreach (var appCompatCach in appCompat.Caches)
{
Log.Verbose("Dumping cache details: {@Details}", appCompat);
try
{
Log.Information(
"Found {Count:N0} cache entries for {OperatingSystem} in {ControlSet}", appCompatCach.Entries.Count, appCompat.OperatingSystem, $"ControlSet00{appCompatCach.ControlSet}");
if (t)
{
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csvWriter.WriteRecord(cacheEntry);
csvWriter.NextRecord();
}
}
else
{
foreach (var cacheEntry in appCompatCach.Entries)
{
cacheEntry.SourceFile = hiveToProcess;
cacheEntry.Duplicate = cacheKeys.Contains(cacheEntry.GetKey());
cacheKeys.Add(cacheEntry.GetKey());
csvWriter.WriteRecord(cacheEntry);
csvWriter.NextRecord();
}
}
}
catch (Exception ex)
{
Log.Error(ex, "There was an error: Error message: {Message}", ex.Message);
try
{
appCompatCach.PrintDump();
}
catch (Exception ex1)
{
Log.Error(ex1, "Couldn't PrintDump: {Message}", ex1.Message);
}
}
}
sw.Flush();
sw.Close();
Console.WriteLine();
Log.Warning("Results saved to '{OutFilename}'", outFilename);
Console.WriteLine();
}
else
{
Console.WriteLine();
Log.Warning("No caches were found!");
Console.WriteLine();
}
}
catch (Exception ex)
{
if (ex.Message.Contains("Sequence numbers do not match and transaction logs were not found in the same direct") == false)
{
if (ex.Message.Contains("Administrator privileges not found"))
{
Log.Fatal("Could not access '{F}'. Does it exist?", f);
Console.WriteLine();
Log.Fatal("Rerun the program with Administrator privileges to try again");
Console.WriteLine();
}
else if (ex.Message.Contains("Invalid diskName:"))
{
Log.Fatal("Could not access '{F}'. Invalid disk!", f);
Console.WriteLine();
}
else
{
Log.Error(ex, "There was an error: {Message}", ex.Message);
Console.WriteLine();
}
}
}
}
public void ClassMap_ClassMapTest()
{
var sb = new StringBuilder()
.AppendLine("fuga,Hega")
//.AppendLine("124 ,,\"1\"")
//.AppendLine(",sss,\"2\"")
//.AppendLine()
.AppendLine("123,\"3,4\"")
;
Console.WriteLine(sb.ToString());
//using (var sr = new StringReader(sb.ToString()))
//using (var csv = new CsvHelper.CsvReader(sr))
//{
// csv.Configuration.PrepareHeaderForMatch = (header, index) => header.Trim().ToLower();
// csv.Configuration.RegisterClassMap<ClassMapImpl>();
// csv.Read();
// csv.ReadHeader();
// while (csv.Read())
// {
// var d = csv.GetRecord<Hoge>();
// Console.WriteLine("-" + d.Fuga.Code + "-");
// foreach (var h in csv.Context.HeaderRecord)
// {
// Console.WriteLine($"{h}, {csv.GetField(h)}");
// }
// }
// var records = csv.GetRecords<Hoge>();
// //foreach (var data in records)
// //{
// // Console.WriteLine(data.Fuga.Code);
// //}
//}
var srr = new StringWriter();
using (var csv = new CsvHelper.CsvWriter(srr))
{
csv.Configuration.RegisterClassMap <ClassMapImpl>();
var hoges = new List <Hoge>
{
new Hoge {
Fuga = new Fuga {
Code = "code"
}, Hega = new List <int> {
2, 3,
}
},
//new Hoge{ },
//new Hoge{Fuga = new Fuga{ Code = "11,333"} },
//new Hoge{Fuga = new Fuga{ Code = "hega\r\nfuga"} },
//new Hoge{Fuga = new Fuga{ Code = "11"} },
};
csv.WriteHeader <Hoge>();
csv.NextRecord();
foreach (var hoge in hoges)
{
csv.WriteRecord(hoge);
}
}
Console.WriteLine(srr.ToString());
}
private static void DoWork(string d, string f, bool q, string csv, string csvf, string dt, bool debug, bool trace)
{
ActiveDateTimeFormat = dt;
var formatter =
new DateTimeOffsetFormatter(CultureInfo.CurrentCulture);
var levelSwitch = new LoggingLevelSwitch();
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template, formatProvider: formatter)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
if (f.IsNullOrEmpty() &&
d.IsNullOrEmpty())
{
var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth);
var hc = new HelpContext(helpBld, _rootCommand, Console.Out);
helpBld.Write(hc);
Log.Warning("Either -f or -d is required. Exiting");
return;
}
if (f.IsNullOrEmpty() == false &&
!File.Exists(f))
{
Log.Warning("File {F} not found. Exiting", f);
return;
}
if (d.IsNullOrEmpty() == false &&
!Directory.Exists(d))
{
Log.Warning("Directory {D} not found. Exiting", d);
return;
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", Environment.GetCommandLineArgs().Skip(1)));
if (IsAdministrator() == false)
{
Log.Warning("Warning: Administrator privileges not found!");
Console.WriteLine();
}
_csvOuts = new List <CsvOut>();
_failedFiles = new List <string>();
var files = new List <string>();
var sw = new Stopwatch();
sw.Start();
if (f?.Length > 0)
{
files.Add(f);
}
else
{
Console.WriteLine();
Log.Information("Looking for files in {Dir}", d);
if (!q)
{
Console.WriteLine();
}
files = GetRecycleBinFiles(d);
}
Log.Information("Found {Count:N0} files. Processing...", files.Count);
if (!q)
{
Console.WriteLine();
}
foreach (var file in files)
{
ProcessFile(file, q, dt);
}
sw.Stop();
Console.WriteLine();
Log.Information(
"Processed {FailedFilesCount:N0} out of {Count:N0} files in {TotalSeconds:N4} seconds", files.Count - _failedFiles.Count, files.Count, sw.Elapsed.TotalSeconds);
Console.WriteLine();
if (_failedFiles.Count > 0)
{
Console.WriteLine();
Log.Information("Failed files");
foreach (var failedFile in _failedFiles)
{
Log.Information(" {FailedFile}", failedFile);
}
}
if (csv.IsNullOrEmpty() == false && files.Count > 0)
{
if (Directory.Exists(csv) == false)
{
Log.Information("{Csv} does not exist. Creating...", csv);
Directory.CreateDirectory(csv);
}
var outName = $"{DateTimeOffset.Now:yyyyMMddHHmmss}_RBCmd_Output.csv";
if (csvf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(csvf);
}
var outFile = Path.Combine(csv, outName);
outFile =
Path.GetFullPath(outFile);
Log.Warning("CSV output will be saved to {Path}", Path.GetFullPath(outFile));
try
{
var sw1 = new StreamWriter(outFile);
var csvWriter = new CsvWriter(sw1, CultureInfo.InvariantCulture);
csvWriter.WriteHeader(typeof(CsvOut));
csvWriter.NextRecord();
foreach (var csvOut in _csvOuts)
{
csvWriter.WriteRecord(csvOut);
csvWriter.NextRecord();
}
sw1.Flush();
sw1.Close();
}
catch (Exception ex)
{
Log.Error(ex,
"Unable to open {OutFile} for writing. CSV export canceled. Error: {Message}", outFile, ex.Message);
}
}
}
private static Experiment3Output Experiment3(int neuronCount, decimal minAnnPPV, decimal minAnnNPV, int treeCount, decimal minForestPPV, decimal minForestNPV)
{
var result = new Experiment3Output();
result.NeuronCount = neuronCount;
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
DateTime et = DateTime.Now;
var currentMinPPV = 0m;
var currentMinNPV = 0m;
Data data = null;
var learningData = new List<AnnLearningRecord>();
var annPV = new AlgoritamPredictiveValues();
var forestPV = new AlgoritamPredictiveValues();
while (currentMinPPV < minAnnPPV || currentMinNPV < minAnnNPV || currentMinPPV <= currentMinNPV || currentMinPPV < minForestPPV || currentMinNPV < minForestNPV)
{
var es = DateTime.Now;
data = CreateData(metsRecords, 0.8, 0.1);
result.Means = data.Means;
result.StandardDeviations = data.StandardDeviations;
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN PPV, NC:{neuronCount}");
Console.Out.WriteLine("================================");
learningData.Clear();
annPV = Ann(data, neuronCount, learningData, out var ann);
currentMinPPV = annPV.TestPPV;
currentMinNPV = annPV.TestNPV;
result.Ann = ann;
LogAPV(annPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
var experimentPV = new ExperimentPredictiveValues()
{
AnnPpvLearnNPV = annPV.LearnNPV,
AnnPpvLearnPPV = annPV.LearnPPV,
AnnPpvValidateNPV = annPV.ValidateNPV,
AnnPpvValidatePPV = annPV.ValidatePPV,
AnnPpvTestNPV = annPV.TestNPV,
AnnPpvTestPPV = annPV.TestPPV,
};
using (var tw = File.CreateText($"d:\\mets_experiment_3_{neuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}_rndfrst.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<AnnLearningRecord>();
csv.NextRecord();
csv.WriteRecords(learningData);
}
using (var tw = File.CreateText($"d:\\mets_experiment_3_PV_{neuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}_rndfrst.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<ExperimentPredictiveValues>();
csv.NextRecord();
csv.WriteRecords(new ExperimentPredictiveValues[] { experimentPV });
}
return result;
}
private static void Experiment2(int neuronCountPPV, int neuronCountNPV, int repeats)
{
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
var results = new List<ExperimentPredictiveValues>();
DateTime et = DateTime.Now;
for (int i = 0; i < repeats; i++)
{
var es = DateTime.Now;
var data = CreateData(metsRecords, 0.8, 0.1);
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" EXPERIMENT #{i:000}");
Console.Out.WriteLine("================================");
Console.Out.WriteLine();
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN PPV, NC:{neuronCountPPV}");
Console.Out.WriteLine("================================");
var ann_PPV_PV = Ann(data, neuronCountPPV, null, out var ann);
LogAPV(ann_PPV_PV);
var ann_NPV_PV = new AlgoritamPredictiveValues();
if (neuronCountNPV != neuronCountPPV)
{
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" ANN NPV, NC:{neuronCountNPV}");
Console.Out.WriteLine("================================");
ann_NPV_PV = Ann(data, neuronCountNPV, null, out ann);
LogAPV(ann_NPV_PV);
}
var experimentPV = new ExperimentPredictiveValues()
{
AnnPpvLearnNPV = ann_PPV_PV.LearnNPV,
AnnPpvLearnPPV = ann_PPV_PV.LearnPPV,
AnnPpvLearnSENS = ann_PPV_PV.LearnSENS,
AnnPpvLearnSPEC = ann_PPV_PV.LearnSPEC,
AnnPpvValidateNPV = ann_PPV_PV.ValidateNPV,
AnnPpvValidatePPV = ann_PPV_PV.ValidatePPV,
AnnPpvValidateSENS = ann_PPV_PV.ValidateSENS,
AnnPpvValidateSPEC = ann_PPV_PV.ValidateSPEC,
AnnPpvTestNPV = ann_PPV_PV.TestNPV,
AnnPpvTestPPV = ann_PPV_PV.TestPPV,
AnnPpvTestSENS = ann_PPV_PV.TestSENS,
AnnPpvTestSPEC = ann_PPV_PV.TestSPEC,
AnnNpvLearnNPV = ann_NPV_PV.LearnNPV,
AnnNpvLearnPPV = ann_NPV_PV.LearnPPV,
AnnNpvLearnSENS = ann_NPV_PV.LearnSENS,
AnnNpvLearnSPEC = ann_NPV_PV.LearnSPEC,
AnnNpvValidateNPV = ann_NPV_PV.ValidateNPV,
AnnNpvValidatePPV = ann_NPV_PV.ValidatePPV,
AnnNpvValidateSENS = ann_NPV_PV.ValidateSENS,
AnnNpvValidateSPEC = ann_NPV_PV.ValidateSPEC,
AnnNpvTestNPV = ann_NPV_PV.TestNPV,
AnnNpvTestPPV = ann_NPV_PV.TestPPV,
AnnNpvTestSENS = ann_NPV_PV.TestSENS,
AnnNpvTestSPEC = ann_NPV_PV.TestSPEC,
};
LogEPV(experimentPV);
results.Add(experimentPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
LogAvgEPV(results);
using (var tw = File.CreateText($"d:\\mets_experiment_2_{neuronCountPPV}_{neuronCountNPV}_R{repeats}{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<ExperimentPredictiveValues>();
csv.NextRecord();
csv.WriteRecords(results);
}
}
private static void Experiment1()
{
Console.WriteLine("Start neuron count:");
var startNeuronCount = int.Parse(Console.ReadLine());
Console.WriteLine("End neuron count:");
var endNeuronCount = int.Parse(Console.ReadLine());
Console.WriteLine("Repeats:");
var repeats = int.Parse(Console.ReadLine());
var metsRecords = LoadDataRecords(TRAINING_DATA_FILE, 0);
var results = new List<Experiment2Results>();
DateTime et = DateTime.Now;
for (int neuronCount = startNeuronCount; neuronCount <= endNeuronCount; neuronCount++)
{
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" NEURON COUNT #{neuronCount:000}");
Console.Out.WriteLine("================================");
var iterationResults = new List<AlgoritamPredictiveValues>();
for (int i = 0; i < repeats; i++)
{
var es = DateTime.Now;
var data = CreateData(metsRecords, 0.8, 0.1);
Console.Out.WriteLine("================================");
Console.Out.WriteLine($" EXPERIMENT #{i + 1:000}, NC{neuronCount:000}");
Console.Out.WriteLine("================================");
Console.Out.WriteLine();
var annPV = Ann(data, neuronCount, null, out var ann);
LogAPV(annPV);
iterationResults.Add(annPV);
LogMessage($"Experiment time: {DateTime.Now - es}, total time {DateTime.Now - et}");
}
results.Add(new Experiment2Results()
{
NeuronCount = neuronCount,
LearnNPV = iterationResults.Average(x => x.LearnNPV),
LearnPPV = iterationResults.Average(x => x.LearnPPV),
ValidateNPV = iterationResults.Average(x => x.ValidateNPV),
ValidatePPV = iterationResults.Average(x => x.ValidatePPV),
TestNPV = iterationResults.Average(x => x.TestNPV),
TestPPV = iterationResults.Average(x => x.TestPPV),
});
}
using (var tw = File.CreateText($"d:\\mets_experiment_2_{startNeuronCount}_{endNeuronCount}_R{repeats}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment2Results>();
csv.NextRecord();
csv.WriteRecords(results);
}
}
private static void Experiment4(Experiment3Output exp3)
{
var metsRecords = LoadDataRecords(TEST_DATA_FILE, 0);
for (int i = 0; i < metsRecords.Count; i++)
{
var mets = CalculateMets(metsRecords[i]);
if (metsRecords[i].METS != mets)
throw new Exception("Invalid data");
}
if (metsRecords.Count > 100)
{
var mr = new List<MetsDataRecord>();
var rnd = new Random();
while (mr.Count < 100)
{
var index = rnd.Next(metsRecords.Count);
mr.Add(metsRecords[index]);
metsRecords.RemoveAt(index);
}
metsRecords = mr;
}
var inputs = CreateInputs(metsRecords).ToArray();
inputs = inputs.ZScores(exp3.Means, exp3.StandardDeviations);
var outputs = CreateOutputs(metsRecords).ToArray();
var annPredictions = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
{
var annr = exp3.Ann.Compute(inputs[i])[0];
var to = Math.Round(outputs[i][0], 0);
var anno = Math.Round(annr, 0);
annPredictions[i] = new double[] { anno };
}
var annPV = CalculatePredictiveValues(outputs, annPredictions);
var results = new List<Experiment4ResultRecord>();
for (int i = 0; i < metsRecords.Count; i++)
{
results.Add(new Experiment4ResultRecord()
{
AGE = metsRecords[i].AGE,
BMI = metsRecords[i].BMI,
DBP = metsRecords[i].DBP,
FPG = metsRecords[i].FPG,
GEN = metsRecords[i].GEN,
HDL = metsRecords[i].HDL,
HT = metsRecords[i].HT,
METS = metsRecords[i].METS,
METS_PR_ANN = (int)Math.Round(annPredictions[i][0]),
SBP = metsRecords[i].SBP,
TG = metsRecords[i].TG,
WC = metsRecords[i].WC,
WHtR = metsRecords[i].WHtR,
WT = metsRecords[i].WT
});
}
var experimentPV = new Experiment4PredictiveValues()
{
AnnNPV = annPV.NPV,
AnnPPV = annPV.PPV,
};
using (var tw = File.CreateText($"d:\\mets_experiment_4_{exp3.NeuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment4ResultRecord>();
csv.NextRecord();
csv.WriteRecords(results);
}
using (var tw = File.CreateText($"d:\\mets_experiment_4_PV_{exp3.NeuronCount}_{DateTime.Now:yyyy_MM_dd_hh_mm}.csv"))
{
var csv = new CsvHelper.CsvWriter(tw);
csv.WriteHeader<Experiment4PredictiveValues>();
csv.NextRecord();
csv.WriteRecords(new Experiment4PredictiveValues[] { experimentPV });
}
}
private static void DoWork(string f, string d, string csv, string csvf, string json, string jsonf, string xml, string xmlf, string dt, string inc, string exc, string sd, string ed, bool fj, int tdt, bool met, string maps, bool vss, bool dedupe, bool sync, bool debug, bool trace)
{
var levelSwitch = new LoggingLevelSwitch();
_activeDateTimeFormat = dt;
var formatter =
new DateTimeOffsetFormatter(CultureInfo.CurrentCulture);
var template = "{Message:lj}{NewLine}{Exception}";
if (debug)
{
levelSwitch.MinimumLevel = LogEventLevel.Debug;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
if (trace)
{
levelSwitch.MinimumLevel = LogEventLevel.Verbose;
template = "[{Timestamp:HH:mm:ss.fff} {Level:u3}] {Message:lj}{NewLine}{Exception}";
}
var conf = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: template, formatProvider: formatter)
.MinimumLevel.ControlledBy(levelSwitch);
Log.Logger = conf.CreateLogger();
if (sync)
{
try
{
Log.Information("{Header}", Header);
UpdateFromRepo();
}
catch (Exception e)
{
Log.Error(e, "There was an error checking for updates: {Message}", e.Message);
}
Environment.Exit(0);
}
if (f.IsNullOrEmpty() &&
d.IsNullOrEmpty())
{
var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth);
var hc = new HelpContext(helpBld, _rootCommand, Console.Out);
helpBld.Write(hc);
Log.Warning("-f or -d is required. Exiting");
Console.WriteLine();
return;
}
Log.Information("{Header}", Header);
Console.WriteLine();
Log.Information("Command line: {Args}", string.Join(" ", Environment.GetCommandLineArgs().Skip(1)));
Console.WriteLine();
if (IsAdministrator() == false)
{
Log.Warning("Warning: Administrator privileges not found!");
Console.WriteLine();
}
if (vss & !RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
vss = false;
Log.Warning("{Vss} not supported on non-Windows platforms. Disabling...", "--vss");
Console.WriteLine();
}
if (vss & (IsAdministrator() == false))
{
Log.Error("{Vss} is present, but administrator rights not found. Exiting", "--vss");
Console.WriteLine();
return;
}
var sw = new Stopwatch();
sw.Start();
var ts = DateTimeOffset.UtcNow;
_errorFiles = new Dictionary <string, int>();
if (json.IsNullOrEmpty() == false)
{
if (Directory.Exists(json) == false)
{
Log.Information("Path to {Json} doesn't exist. Creating...", json);
try
{
Directory.CreateDirectory(json);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Json}. Does a file with the same name exist? Exiting", json);
Console.WriteLine();
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.json";
if (jsonf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(jsonf);
}
var outFile = Path.Combine(json, outName);
Log.Information("json output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swJson = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
JsConfig.DateHandler = DateHandler.ISO8601;
}
if (xml.IsNullOrEmpty() == false)
{
if (Directory.Exists(xml) == false)
{
Log.Information("Path to {Xml} doesn't exist. Creating...", xml);
try
{
Directory.CreateDirectory(xml);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Xml}. Does a file with the same name exist? Exiting", xml);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.xml";
if (xmlf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(xmlf);
}
var outFile = Path.Combine(xml, outName);
Log.Information("XML output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swXml = new StreamWriter(outFile, false, Encoding.UTF8);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
}
if (sd.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(sd, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_startDate = dateTimeOffset;
Log.Information("Setting Start date to {StartDate}", _startDate.Value);
}
else
{
Log.Warning("Could not parse {Sd} to a valid datetime! Events will not be filtered by Start date!", sd);
}
}
if (ed.IsNullOrEmpty() == false)
{
if (DateTimeOffset.TryParse(ed, null, DateTimeStyles.AssumeUniversal, out var dateTimeOffset))
{
_endDate = dateTimeOffset;
Log.Information("Setting End date to {EndDate}", _endDate.Value);
}
else
{
Log.Warning("Could not parse {Ed} to a valid datetime! Events will not be filtered by End date!", ed);
}
}
if (_startDate.HasValue || _endDate.HasValue)
{
Console.WriteLine();
}
if (csv.IsNullOrEmpty() == false)
{
if (Directory.Exists(csv) == false)
{
Log.Information(
"Path to {Csv} doesn't exist. Creating...", csv);
try
{
Directory.CreateDirectory(csv);
}
catch (Exception ex)
{
Log.Fatal(ex,
"Unable to create directory {Csv}. Does a file with the same name exist? Exiting", csv);
return;
}
}
var outName = $"{ts:yyyyMMddHHmmss}_EvtxECmd_Output.csv";
if (csvf.IsNullOrEmpty() == false)
{
outName = Path.GetFileName(csvf);
}
var outFile = Path.Combine(csv, outName);
Log.Information("CSV output will be saved to {OutFile}", outFile);
Console.WriteLine();
try
{
_swCsv = new StreamWriter(outFile, false, Encoding.UTF8);
var opt = new CsvConfiguration(CultureInfo.InvariantCulture)
{
ShouldUseConstructorParameters = _ => false
};
_csvWriter = new CsvWriter(_swCsv, opt);
}
catch (Exception ex)
{
Log.Error(ex, "Unable to open {OutFile}! Is it in use? Exiting!", outFile);
Console.WriteLine();
Environment.Exit(0);
}
var foo = _csvWriter.Context.AutoMap <EventRecord>();
foo.Map(t => t.RecordPosition).Ignore();
foo.Map(t => t.Size).Ignore();
foo.Map(t => t.Timestamp).Ignore();
foo.Map(t => t.RecordNumber).Index(0);
foo.Map(t => t.EventRecordId).Index(1);
foo.Map(t => t.TimeCreated).Index(2);
foo.Map(t => t.TimeCreated).Convert(t =>
$"{t.Value.TimeCreated.ToString(dt)}");
foo.Map(t => t.EventId).Index(3);
foo.Map(t => t.Level).Index(4);
foo.Map(t => t.Provider).Index(5);
foo.Map(t => t.Channel).Index(6);
foo.Map(t => t.ProcessId).Index(7);
foo.Map(t => t.ThreadId).Index(8);
foo.Map(t => t.Computer).Index(9);
foo.Map(t => t.UserId).Index(10);
foo.Map(t => t.MapDescription).Index(11);
foo.Map(t => t.UserName).Index(12);
foo.Map(t => t.RemoteHost).Index(13);
foo.Map(t => t.PayloadData1).Index(14);
foo.Map(t => t.PayloadData2).Index(15);
foo.Map(t => t.PayloadData3).Index(16);
foo.Map(t => t.PayloadData4).Index(17);
foo.Map(t => t.PayloadData5).Index(18);
foo.Map(t => t.PayloadData6).Index(19);
foo.Map(t => t.ExecutableInfo).Index(20);
foo.Map(t => t.HiddenRecord).Index(21);
foo.Map(t => t.SourceFile).Index(22);
foo.Map(t => t.Keywords).Index(23);
foo.Map(t => t.Payload).Index(24);
_csvWriter.Context.RegisterClassMap(foo);
_csvWriter.WriteHeader <EventRecord>();
_csvWriter.NextRecord();
}
if (Directory.Exists(maps) == false)
{
Log.Warning("Maps directory {Maps} does not exist! Event ID maps will not be loaded!!", maps);
}
else
{
Log.Debug("Loading maps from {Path}", Path.GetFullPath(maps));
var errors = EventLog.LoadMaps(Path.GetFullPath(maps));
if (errors)
{
return;
}
Log.Information("Maps loaded: {Count:N0}", EventLog.EventLogMaps.Count);
}
_includeIds = new HashSet <int>();
_excludeIds = new HashSet <int>();
if (exc.IsNullOrEmpty() == false)
{
var excSegs = exc.Split(',');
foreach (var incSeg in excSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_excludeIds.Add(goodId);
}
}
}
if (inc.IsNullOrEmpty() == false)
{
_excludeIds.Clear();
var incSegs = inc.Split(',');
foreach (var incSeg in incSegs)
{
if (int.TryParse(incSeg, out var goodId))
{
_includeIds.Add(goodId);
}
}
}
if (vss)
{
string driveLetter;
if (f.IsEmpty() == false)
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(f))
.Substring(0, 1);
}
else
{
driveLetter = Path.GetPathRoot(Path.GetFullPath(d))
.Substring(0, 1);
}
Helper.MountVss(driveLetter, VssDir);
Console.WriteLine();
}
EventLog.TimeDiscrepancyThreshold = tdt;
if (f.IsNullOrEmpty() == false)
{
if (File.Exists(f) == false)
{
Log.Warning("\t{F} does not exist! Exiting", f);
Console.WriteLine();
return;
}
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
dedupe = false;
ProcessFile(Path.GetFullPath(f), dedupe, fj, met);
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
var root = Path.GetPathRoot(Path.GetFullPath(f));
var stem = Path.GetFullPath(f).Replace(root, "");
foreach (var vssDir in vssDirs)
{
var newPath = Path.Combine(vssDir, stem);
if (File.Exists(newPath))
{
ProcessFile(newPath, dedupe, fj, met);
}
}
}
}
else
{
if (Directory.Exists(d) == false)
{
Log.Warning("\t{D} does not exist! Exiting", d);
Console.WriteLine();
return;
}
Log.Information("Looking for event log files in {D}", d);
Console.WriteLine();
#if !NET6_0
var directoryEnumerationFilters = new DirectoryEnumerationFilters
{
InclusionFilter = fsei => fsei.Extension.ToUpperInvariant() == ".EVTX",
RecursionFilter = entryInfo => !entryInfo.IsMountPoint && !entryInfo.IsSymbolicLink,
ErrorFilter = (errorCode, errorMessage, pathProcessed) => true
};
var dirEnumOptions =
DirectoryEnumerationOptions.Files | DirectoryEnumerationOptions.Recursive |
DirectoryEnumerationOptions.SkipReparsePoints | DirectoryEnumerationOptions.ContinueOnException |
DirectoryEnumerationOptions.BasicSearch;
var files2 =
Directory.EnumerateFileSystemEntries(Path.GetFullPath(d), dirEnumOptions, directoryEnumerationFilters);
#else
var enumerationOptions = new EnumerationOptions
{
IgnoreInaccessible = true,
MatchCasing = MatchCasing.CaseInsensitive,
RecurseSubdirectories = true,
AttributesToSkip = 0
};
var files2 =
Directory.EnumerateFileSystemEntries(d, "*.evtx", enumerationOptions);
#endif
if (_swXml == null && _swJson == null && _swCsv == null)
{
//no need for maps
Log.Debug("Clearing map collection since no output specified");
EventLog.EventLogMaps.Clear();
}
foreach (var file in files2)
{
ProcessFile(file, dedupe, fj, met);
}
if (vss)
{
var vssDirs = Directory.GetDirectories(VssDir);
Console.WriteLine();
foreach (var vssDir in vssDirs)
{
var root = Path.GetPathRoot(Path.GetFullPath(d));
var stem = Path.GetFullPath(d).Replace(root, "");
var target = Path.Combine(vssDir, stem);
Console.WriteLine();
Log.Information("Searching {Vss} for event logs...", $"VSS{target.Replace($"{VssDir}\\", "")}");
var vssFiles = Helper.GetFilesFromPath(target, "*.evtx", true);
foreach (var file in vssFiles)
{
ProcessFile(file, dedupe, fj, met);
}
}
}
}
try
{
_swCsv?.Flush();
_swCsv?.Close();
_swJson?.Flush();
_swJson?.Close();
_swXml?.Flush();
_swXml?.Close();
}
catch (Exception e)
{
Log.Error(e, "Error when flushing output files to disk! Error message: {Message}", e.Message);
}
sw.Stop();
Console.WriteLine();
if (_fileCount == 1)
{
Log.Information("Processed {FileCount:N0} file in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
else
{
Log.Information("Processed {FileCount:N0} files in {TotalSeconds:N4} seconds", _fileCount, sw.Elapsed.TotalSeconds);
}
Console.WriteLine();
if (_errorFiles.Count > 0)
{
Console.WriteLine();
Log.Information("Files with errors");
foreach (var errorFile in _errorFiles)
{
Log.Information("{Key} error count: {Value:N0}", errorFile.Key, errorFile.Value);
}
Console.WriteLine();
}
if (vss)
{
if (Directory.Exists(VssDir))
{
foreach (var directory in Directory.GetDirectories(VssDir))
{
Directory.Delete(directory);
}
#if !NET6_0
Directory.Delete(VssDir, true, true);
#else
Directory.Delete(VssDir, true);
#endif
}
}
}