public void AddGrammar(IGram gram)
{
Assert.IsTrue(gram.GetN() == N);
NGram ngram = (NGram)gram;
foreach (string key in ngram.Grammar.Keys)
{
if (Grammar.ContainsKey(key) == false)
{
Grammar[key] = new UniGram();
}
Grammar[key].AddGrammar(ngram.Grammar[key]);
}
}
public void AddGrammar(IGram gram)
{
Assert.IsTrue(gram.GetN() == 1);
UniGram unigram = (UniGram)gram;
foreach (KeyValuePair <string, float> keyValue in unigram.Grammar)
{
if (Grammar.ContainsKey(keyValue.Key) == false)
{
Grammar[keyValue.Key] = keyValue.Value;
}
else
{
Grammar[keyValue.Key] += keyValue.Value;
}
}
}
public void AddGrammar(IGram gram)
{
HierarchicalNGram grammar = gram as HierarchicalNGram;
if (grammar != null)
{
Assert.AreEqual(N, grammar.N);
for (int grammarSize = 0; grammarSize < N; ++grammarSize)
{
Grammars[grammarSize].AddGrammar(grammar.Grammars[grammarSize]);
}
}
else
{
int n = gram.GetN();
Assert.IsTrue(n <= N);
Grammars[n - 1].AddGrammar(gram);
}
}
public void Execute()
{
string keyDirectory = Path.Combine(basePath, $"{extension}_{gram.GetN()}");
if (Directory.Exists(keyDirectory) == false)
{
Directory.CreateDirectory(keyDirectory);
}
StreamWriter writer = File.CreateText($"{keyDirectory}.txt");
writer.WriteLine("Sequence_Probability,Perplexity,Linearity_JSON_Positions,Leniency");
ICompiledGram compiled = gram.Compile();
ICompiledGram simpleCompiled = simplifiedGram?.Compile();
for (int i = 0; i < numSimulations; ++i)
{
UtilityRandom.SetSeed(new DateTime().Millisecond);
Tuple <List <string>, List <string> > tuple;
if (gram as NGram == null)
{
tuple = GetColumnsBestGuess(compiled, simpleCompiled);
}
else
{
tuple = GetColumnsSemiGuaranteed(compiled, simpleCompiled);
}
List <string> columns = tuple.Item1;
List <string> simplified = tuple.Item2;
string[] columnsArray = columns.ToArray();
List <int> positions = LevelAnalyzer.Positions(columnsArray);
JsonArray jsonPositions = new JsonArray();
foreach (int pos in positions)
{
jsonPositions.Add(pos);
}
double sequenceProbability = compiled.SequenceProbability(columnsArray);
writer.Write($"{sequenceProbability},");
if (sequenceProbability == 0)
{
writer.Write($"0,");
}
else
{
writer.Write($"{1d/sequenceProbability},");
}
writer.Write($"{jsonPositions},");
writer.Write($"{LevelAnalyzer.Leniency(simplified.ToArray())}\n");
StreamWriter levelWriter = File.CreateText(Path.Combine(keyDirectory, $"{i}.txt"));
levelWriter.Write(string.Join("\n", columnsArray));
levelWriter.Flush();
levelWriter.Close();
if (i % 200 == 0)
{
writer.Flush();
}
}
writer.Flush();
writer.Close();
}