public void TestGetValues()
{
UniGram unigram = new UniGram();
Assert.AreEqual(
new Dictionary <string, float>(),
unigram.Compile().GetValues(null));
unigram.AddData(null, "a");
Assert.AreEqual(
new Dictionary <string, float>()
{
{ "a", 1f }
},
unigram.Compile().GetValues(null));
unigram.AddData(null, "a");
unigram.AddData(null, "a");
unigram.AddData(null, "b");
Assert.AreEqual(
new Dictionary <string, float>()
{
{ "a", 0.75f }, { "b", 0.25f }
},
unigram.Compile().GetValues(null));
}
public void TestHasNextStep()
{
UniGram unigram = new UniGram();
Assert.IsFalse(unigram.Compile().HasNextStep(null));
unigram.AddData(null, "a");
Assert.IsTrue(unigram.Compile().HasNextStep(null));
unigram.AddData(null, "a");
Assert.IsTrue(unigram.Compile().HasNextStep(null));
unigram.AddData(null, "b");
Assert.IsTrue(unigram.Compile().HasNextStep(null));
}
public void TestConstructor()
{
UniGram unigram = new UniGram();
Assert.DoesNotThrow(() => { unigram.Compile(); });
unigram.AddData(null, "a");
Assert.DoesNotThrow(() => { unigram.Compile(); });
unigram.AddData(null, "b");
Assert.DoesNotThrow(() => { unigram.Compile(); });
unigram.AddData(null, "b");
unigram.AddData(null, "b");
unigram.AddData(null, "b");
unigram.AddData(null, "a");
Assert.DoesNotThrow(() => { unigram.Compile(); });
}
public void TestPerplexity()
{
UniGram unigram = new UniGram();
Assert.AreEqual(
double.PositiveInfinity,
unigram.Compile().Perplexity(new string[] { "a", "b" }));
unigram.AddData(null, "a");
Assert.AreEqual(
1,
unigram.Compile().Perplexity(new string[] { "a", "a" }));
unigram.AddData(null, "b");
Assert.AreEqual(
1 / 0.5,
unigram.Compile().Perplexity(new string[] { "a" }));
Assert.AreEqual(
1 / 0.25,
unigram.Compile().Perplexity(new string[] { "b", "b" }));
unigram.AddData(null, "b");
unigram.AddData(null, "b");
Assert.AreEqual(
1 / 0.75,
unigram.Compile().Perplexity(new string[] { "b" }));
Assert.AreEqual(
1 / (0.75 * 0.25 * 0.75),
unigram.Compile().Perplexity(new string[] { "b", "a", "b" }));
}
public void TestSequenceProbability()
{
UniGram unigram = new UniGram();
Assert.AreEqual(
0,
unigram.Compile().SequenceProbability(new string[] { "a", "b" }));
unigram.AddData(null, "a");
Assert.AreEqual(
1,
unigram.Compile().SequenceProbability(new string[] { "a", "a" }));
unigram.AddData(null, "b");
Assert.AreEqual(
0.5,
unigram.Compile().SequenceProbability(new string[] { "a" }));
Assert.AreEqual(
0.25,
unigram.Compile().SequenceProbability(new string[] { "b", "b" }));
unigram.AddData(null, "b");
unigram.AddData(null, "b");
Assert.AreEqual(
0.75,
unigram.Compile().SequenceProbability(new string[] { "b" }));
Assert.AreEqual(
0.75 * 0.25 * 0.75,
unigram.Compile().SequenceProbability(new string[] { "b", "a", "b" }));
}
public void TestGetGuesses()
{
UniGram unigram = new UniGram();
Assert.AreEqual(0, unigram.Compile().GetGuesses(null).Length);
unigram.AddData(null, "a");
Assert.AreEqual(1, unigram.Compile().GetGuesses(null).Length);
unigram.AddData(null, "a");
Assert.AreEqual(1, unigram.Compile().GetGuesses(null).Length);
unigram.AddData(null, "b");
Assert.AreEqual(2, unigram.Compile().GetGuesses(null).Length);
unigram.AddData(null, "c");
string[] guesses = unigram.Compile().GetGuesses(null);
Assert.AreEqual(3, guesses.Length);
Assert.IsTrue(guesses.Contains("a"));
Assert.IsTrue(guesses.Contains("b"));
Assert.IsTrue(guesses.Contains("c"));
}
public void TestSequenceProbability()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.9f);
Assert.AreEqual(
0,
gram.Compile().SequenceProbability(new string[] { "a", "b", "c" }));
double denominator = 0.9 + 0.81 + 0.729;
double triWeight = 0.9 / denominator;
double biWeight = 0.81 / denominator;
double uniweight = 0.729 / denominator;
UniGram gram1 = new UniGram();
gram1.AddData(null, "a");
gram1.AddData(null, "a");
gram1.AddData(null, "b");
NGram gram2 = new NGram(2);
gram2.AddData(new string[] { "a" }, "a");
gram2.AddData(new string[] { "a" }, "b");
NGram gram3 = new NGram(3);
gram3.AddData(new string[] { "a", "a" }, "b");
gram.AddData(new string[] { "a", "a" }, "b");
ICompiledGram c1 = gram1.Compile();
ICompiledGram c2 = gram2.Compile();
ICompiledGram c3 = gram3.Compile();
string[] input = new string[] { "a", "a", "b" };
double expected =
uniweight * c1.SequenceProbability(input) +
biWeight * c2.SequenceProbability(input) +
triWeight * c3.SequenceProbability(input);
double actual = gram.Compile().SequenceProbability(new string[] { "a", "a", "b" });
Assert.IsTrue(
Mathf.Approximately((float)expected, (float)actual),
$"Expected {expected} but received {actual}.");
}