static void DisplayParseResults(
string[] words, int[] tags, int[] results, ParseTarget target)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Tags:");
for (int i = 0; i < words.Length; i++)
{
dynamic tag = tags[i];
if (target == ParseTarget.Address)
{
tag = (AddressTag)tag;
}
else
{
tag = (NameTag)tag;
}
Console.WriteLine($" {words[i]}: {tag}");
}
Console.ForegroundColor = ConsoleColor.Cyan;
Console.WriteLine("Parsed:");
for (int i = 0; i < words.Length; i++)
{
dynamic label = results[i];
if (target == ParseTarget.Address)
{
label = (AddressLabel)label;
}
else
{
label = (NameLabel)label;
}
Console.WriteLine($" {words[i]}: {label}");
}
}
static void ParseFromConsole(HiddenMarkovModel hmm, ParseTarget target)
{
ConsoleColor defaultColor = Console.ForegroundColor;
ITagger tagger;
char[] splitChars = null;
if (target == ParseTarget.Address)
{
tagger = new AddressTagger();
splitChars = new char[1] {
'-'
};
}
else
{
tagger = new NameTagger();
}
while (true)
{
Console.WriteLine($"Enter the {target}:");
Console.ForegroundColor = ConsoleColor.Green;
string input = Console.ReadLine();
if (string.IsNullOrWhiteSpace(input))
{
continue;
}
string[] words = Preprocessing.GetFormattedWords(input, splitChars);
int[] tags = tagger.TagInput(words);
int[] result = hmm.Decide(tags);
DisplayParseResults(words, tags, result, target);
Console.ForegroundColor = defaultColor;
}
}
private ModEntityData <T> CreateDependant <T>(EntityCreator <T> creator, ParseTarget parseTarget) where T : Entity
{
ModEntityData <T> entities = null;
foreach (var modDirectoryHelper in StellarisDirectoryHelper.CreateCombinedList(stellarisDirectoryHelper, modDirectoryHelpers))
{
entities = creator.ProcessDirectoryHelper(entities, modDirectoryHelper, techsAndDependencies);
}
return(entities);
}
static void Test(HiddenMarkovModel hmm, ParseTarget target)
{
string filePath = GetFilePath(".csv", "Please enter path to test data file:");
Tuple <int[][], int[][]> xAndY = GetTransformedData(filePath, target);
Console.WriteLine("Testing model...");
ModelTester tester = new ModelTester(hmm);
HMMTestResult results = tester.Test(xAndY);
Console.WriteLine("Results:");
Console.WriteLine($" Total Accuracy: {results.TotalAccuracy:0.###}");
Console.WriteLine($" Average Accuracy: {results.AverageAccuracy:0.###}");
}
public static IEnumerable <Entity> Get(this ObjectsDependantOnTechs deps, ParseTarget parseTarget)
{
switch (parseTarget)
{
case ParseTarget.Technologies: throw new InvalidOperationException("No techs in dependants");
case ParseTarget.Buildings: return(deps.Buildings.AllEntities);
case ParseTarget.ShipComponents: return(deps.ShipComponentsSets.AllEntities);
case ParseTarget.Decisions: return(deps.Decisions.AllEntities);
default: throw new InvalidOperationException("Unknown type: " + parseTarget);
}
}
public static string ImagesDirectory(this ParseTarget target)
{
switch (target)
{
case ParseTarget.Technologies: return("technologies");
case ParseTarget.Buildings: return("buildings");
case ParseTarget.ShipComponents: return("ship_parts");
case ParseTarget.Decisions: return("decisions");
}
return(null);
}
static void Train(ParseTarget target)
{
string filePath = GetFilePath(".csv", "Please enter path to training data file:");
Tuple <int[][], int[][]> xAndY = GetTransformedData(filePath, target);
Console.WriteLine("Training model...");
HiddenMarkovModel hmm = null;
if (target == ParseTarget.Name)
{
ModelTrainer <NameTag, NameLabel> trainer = new ModelTrainer <NameTag, NameLabel>();
hmm = trainer.TrainModel(xAndY.Item1, xAndY.Item2);
}
if (target == ParseTarget.Address)
{
ModelTrainer <AddressTag, AddressLabel> trainer = new ModelTrainer <AddressTag, AddressLabel>();
hmm = trainer.TrainModel(xAndY.Item1, xAndY.Item2);
}
Console.WriteLine("Model trained.");
Console.WriteLine("Do you want to (T)est this model or (S)ave it right away?");
ConsoleKey key = Console.ReadKey().Key;
if (key == ConsoleKey.T)
{
Test(hmm, target);
Console.WriteLine("Save this model (Y/N)?");
ConsoleKey saveKey = Console.ReadKey().Key;
if (saveKey == ConsoleKey.Y)
{
SaveModel(hmm);
}
}
if (key == ConsoleKey.S)
{
SaveModel(hmm);
}
}
private ModEntityData <T> ProcessDependant <T>(EntityCreator <T> creator, ParseTarget parseTarget) where T : Entity
{
ModEntityData <T> entities = CreateDependant(creator, parseTarget);
entities?.ApplyToChain((ents, links) => {
var invalidEntities = ents.Where(x => !x.Value.Prerequisites.Any()).Select(x => x.Value).ToList();
foreach (var invalidEntity in invalidEntities)
{
Log.Logger.Warning("Removing {entityId} from {file} dependant entities as we were unable to locate its specified pre-requisite techs", invalidEntity.Id,
invalidEntity.FilePath);
ents.Remove(invalidEntity.Id);
var invalidLinks = links.Where(x => x.To.Id == invalidEntity.Id).ToList();
links.RemoveAll(invalidLinks);
}
});
if (entities != null)
{
foreach (var entity in entities.AllEntities)
{
foreach (var prerequisite in entity.Prerequisites)
{
if (prerequisite.ExtraDesc != null)
{
entity.ExtraDesc = prerequisite.ExtraDesc;
}
}
}
Log.Logger.Debug("Processed {entityCount} {parseTarget} with {linkCount} links", entities.EntityCount, parseTarget, entities.LinkCount);
}
else
{
Log.Logger.Warning("{parseTarget} had no items in any of the sources");
}
return(entities);
}
static Tuple <int[][], int[][]> GetTransformedData(string filePath, ParseTarget target)
{
Tuple <int[][], int[][]> result = null;
Console.WriteLine("Loading and transforming records...");
if (target == ParseTarget.Address)
{
RecordsLoader <AddressTrainingSample> loader = new RecordsLoader <AddressTrainingSample>(filePath);
RecordsTransformer <AddressTrainingSample, AddressLabel, AddressTagger> transformer =
new RecordsTransformer <AddressTrainingSample, AddressLabel, AddressTagger>();
result = transformer.GetXAndY(loader.Load(), '-');
}
if (target == ParseTarget.Name)
{
RecordsLoader <NameTrainingSample> loader = new RecordsLoader <NameTrainingSample>(filePath);
RecordsTransformer <NameTrainingSample, NameLabel, NameTagger> transformer =
new RecordsTransformer <NameTrainingSample, NameLabel, NameTagger>();
result = transformer.GetXAndY(loader.Load());
}
return(result);
}
public TestData(ParseTarget target)
{
Arguments = new string[] {};
Target = target;
}
public TestData(ParseTarget target, params string[] args)
{
Arguments = args;
Target = target;
}
static void Main(string[] args)
{
ConsoleColor defaultColor = Console.ForegroundColor;
ShowTitle(defaultColor);
while (true)
{
Console.WriteLine("What will this model be parsing?");
Console.WriteLine(" A: Addresses");
Console.WriteLine(" N: Names");
ParseTarget target = ParseTarget.Name;
ConsoleKey targetKey = Console.ReadKey().Key;
if (targetKey == ConsoleKey.A)
{
target = ParseTarget.Address;
}
if (targetKey == ConsoleKey.N)
{
target = ParseTarget.Name;
}
Console.WriteLine($"Parse Target: {target}");
ConsoleKey key = PromptForMainOption();
try
{
if (key == ConsoleKey.N)
{
Train(target);
}
else if (key == ConsoleKey.E)
{
string modelPath =
GetFilePath(REQUIRED_MODEL_EXTENSION, "Please enter path to your model file:");
HiddenMarkovModel hmm = Serializer.Load <HiddenMarkovModel>(modelPath);
Test(hmm, target);
}
else if (key == ConsoleKey.P)
{
string modelPath =
GetFilePath(REQUIRED_MODEL_EXTENSION, "Please enter path to your model file:");
HiddenMarkovModel hmm = Serializer.Load <HiddenMarkovModel>(modelPath);
ParseFromConsole(hmm, target);
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine();
Console.WriteLine($"{key} is not one of the options.");
Console.ForegroundColor = defaultColor;
}
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("An error occurred!");
Console.WriteLine($"{ex.GetType().Name}: {ex.Message}");
Trace.WriteLine(ex.StackTrace);
Console.ForegroundColor = defaultColor;
}
}
}
