public void TestAddData()
{
HierarchicalNGram a = new HierarchicalNGram(3, 0.6f);
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
NGram n3 = a.Grammars[2] as NGram;
a.AddData(new string[] { "a", "b" }, "c");
Assert.AreEqual(3, u1.Grammar.Keys.Count);
Assert.AreEqual(2, n2.Grammar.Keys.Count);
Assert.AreEqual(1, n3.Grammar.Keys.Count);
Assert.AreEqual(1f, u1.Grammar["a"]);
Assert.AreEqual(1f, u1.Grammar["b"]);
Assert.AreEqual(1f, u1.Grammar["c"]);
Assert.AreEqual(1f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(1f, n2.Grammar["b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["a,b"].Grammar["c"]);
a.AddData(new string[] { "a", "b" }, "c");
a.AddData(new string[] { "c", "b" }, "c");
Assert.AreEqual(3, u1.Grammar.Keys.Count);
Assert.AreEqual(3, n2.Grammar.Keys.Count);
Assert.AreEqual(2, n3.Grammar.Keys.Count);
Assert.AreEqual(2f, u1.Grammar["a"]);
Assert.AreEqual(3f, u1.Grammar["b"]);
Assert.AreEqual(4f, u1.Grammar["c"]);
Assert.AreEqual(2f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(3f, n2.Grammar["b"].Grammar["c"]);
Assert.AreEqual(1f, n2.Grammar["c"].Grammar["b"]);
Assert.AreEqual(2f, n3.Grammar["a,b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["c,b"].Grammar["c"]);
// a, b, c, d
// ab, bc, cb, bd
// abc, cbc, abd
a.AddData(new string[] { "a", "b" }, "d");
Assert.AreEqual(4, u1.Grammar.Keys.Count);
Assert.AreEqual(3, n2.Grammar.Keys.Count);
Assert.AreEqual(2, n3.Grammar.Keys.Count);
Assert.AreEqual(3f, u1.Grammar["a"]);
Assert.AreEqual(4f, u1.Grammar["b"]);
Assert.AreEqual(4f, u1.Grammar["c"]);
Assert.AreEqual(1f, u1.Grammar["d"]);
Assert.AreEqual(3f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(3f, n2.Grammar["b"].Grammar["c"]);
Assert.AreEqual(1f, n2.Grammar["b"].Grammar["d"]);
Assert.AreEqual(1f, n2.Grammar["c"].Grammar["b"]);
Assert.AreEqual(2f, n3.Grammar["a,b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["a,b"].Grammar["d"]);
Assert.AreEqual(1f, n3.Grammar["c,b"].Grammar["c"]);
}
public void TestAddNGram()
{
HierarchicalNGram a = new HierarchicalNGram(3, 0.6f);
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
NGram n3 = a.Grammars[2] as NGram;
NGram ngram = new NGram(2);
ngram.AddData(new string[] { "a" }, "b");
a.AddGrammar(ngram);
Assert.AreEqual(0, u1.Grammar.Keys.Count);
Assert.AreEqual(1, n2.Grammar.Keys.Count);
Assert.AreEqual(0, n3.Grammar.Keys.Count);
Assert.AreEqual(1f, n2.Grammar["a"].Grammar["b"]);
ngram = new NGram(3);
ngram.AddData(new string[] { "a", "b" }, "c");
ngram.AddData(new string[] { "a", "b" }, "c");
ngram.AddData(new string[] { "a", "b" }, "d");
ngram.AddData(new string[] { "a", "c" }, "d");
a.AddGrammar(ngram);
Assert.AreEqual(0, u1.Grammar.Keys.Count);
Assert.AreEqual(1, n2.Grammar.Keys.Count);
Assert.AreEqual(2, n3.Grammar.Keys.Count);
Assert.AreEqual(1f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(2f, n3.Grammar["a,b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["a,b"].Grammar["d"]);
Assert.AreEqual(1f, n3.Grammar["a,c"].Grammar["d"]);
}
public void TestGetGuesses()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.6f);
gram.AddData(new string[] { "b", "a" }, "c");
gram.AddData(new string[] { "b", "c" }, "c");
gram.AddData(new string[] { "b", "a" }, "a");
gram.AddData(new string[] { "a", "a" }, "d");
ICompiledGram compiledGram = gram.Compile();
string[] guesses = compiledGram.GetGuesses(new string[] { "b", "a" });
Assert.IsTrue(guesses.Contains("a"));
Assert.IsTrue(guesses.Contains("c"));
Assert.IsTrue(guesses.Contains("d"));
guesses = compiledGram.GetGuesses(new string[] { "b", "c" });
Assert.IsTrue(guesses.Contains("a"));
Assert.IsTrue(guesses.Contains("c"));
Assert.IsTrue(guesses.Contains("d"));
guesses = compiledGram.GetGuesses(new string[] { "a", "a" });
Assert.IsTrue(guesses.Contains("a"));
Assert.IsTrue(guesses.Contains("c"));
Assert.IsTrue(guesses.Contains("d"));
}
public void TestWeightBuilding()
{
CompiledHierarchicalNGram compiled;
HierarchicalNGram gram;
// test two detailed
gram = new HierarchicalNGram(2, 0.6f);
compiled = gram.Compile() as CompiledHierarchicalNGram;
Assert.AreEqual(0.375f, compiled.Weights[0]);
Assert.AreEqual(0.625f, compiled.Weights[1]);
Assert.AreEqual(1f, compiled.Weights[0] + compiled.Weights[1]);
// test three detailed
gram = new HierarchicalNGram(3, 0.6f);
compiled = gram.Compile() as CompiledHierarchicalNGram;
Assert.IsTrue(
Mathf.Approximately(0.183673469f, compiled.Weights[0]),
$"Expected ~0.18 and received ${compiled.Weights[0]}");
Assert.IsTrue(
Mathf.Approximately(0.3061224f, compiled.Weights[1]),
$"Expected ~0.306f and received ${compiled.Weights[1]}");
Assert.IsTrue(
Mathf.Approximately(0.5102041f, compiled.Weights[2]),
$"Expected ~0.510f and received {compiled.Weights[2]}");
Assert.AreEqual(1f, compiled.Weights[0] + compiled.Weights[1] + compiled.Weights[2]);
// test 4 to 100 more broadly
for (int i = 4; i < 100; ++i)
{
gram = new HierarchicalNGram(i, 0.6f);
compiled = gram.Compile() as CompiledHierarchicalNGram;
Assert.IsNotNull(compiled);
float previousWeight = 0;
float total = 0;
foreach (float weight in compiled.Weights)
{
total += weight;
Assert.IsFalse(weight <= 0, $"{i}: has a negative weight {weight}");
Assert.IsTrue(previousWeight < weight);
previousWeight = weight;
}
Assert.IsTrue(
Mathf.Approximately(1f, total),
$"Total weight is {total} which should atleast be very close to 1.");
}
}
public void TestAddUnigram()
{
HierarchicalNGram a = new HierarchicalNGram(2, 0.6f);
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
UniGram unigram = new UniGram();
unigram.AddData(null, "a");
a.AddGrammar(unigram);
Assert.AreEqual(1, u1.Grammar.Keys.Count);
Assert.AreEqual(0, n2.Grammar.Keys.Count);
Assert.AreEqual(1, u1.Grammar["a"]);
}
public void TestHasNextStep()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.6f);
Assert.IsFalse(gram.Compile().HasNextStep(new string[] { "a", "b" }));
gram.AddData(new string[] { "a", "b" }, "c");
Assert.IsTrue(gram.Compile().HasNextStep(new string[] { "a", "b" }));
Assert.IsTrue(gram.Compile().HasNextStep(new string[] { "a", "c" }));
gram.AddData(new string[] { "a", "d" }, "c");
Assert.IsTrue(gram.Compile().HasNextStep(new string[] { "a", "b" }));
Assert.IsTrue(gram.Compile().HasNextStep(new string[] { "a", "d" }));
Assert.IsTrue(gram.Compile().HasNextStep(new string[] { "a", "z" }));
}
public void TestConstructor()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.6f);
Assert.DoesNotThrow(() => { gram.Compile(); });
gram.AddData(new string[] { "b", "a" }, "a");
Assert.DoesNotThrow(() => { gram.Compile(); });
gram.AddData(new string[] { "b", "a" }, "z");
Assert.DoesNotThrow(() => { gram.Compile(); });
gram.AddData(new string[] { "b", "f" }, "2");
gram.AddData(new string[] { "c", "g" }, "a");
gram.AddData(new string[] { "d", "h" }, "e");
gram.AddData(new string[] { "e", "h" }, "c");
Assert.DoesNotThrow(() => { gram.Compile(); });
}
public void TestSequenceProbability()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.9f);
Assert.AreEqual(
0,
gram.Compile().SequenceProbability(new string[] { "a", "b", "c" }));
double denominator = 0.9 + 0.81 + 0.729;
double triWeight = 0.9 / denominator;
double biWeight = 0.81 / denominator;
double uniweight = 0.729 / denominator;
UniGram gram1 = new UniGram();
gram1.AddData(null, "a");
gram1.AddData(null, "a");
gram1.AddData(null, "b");
NGram gram2 = new NGram(2);
gram2.AddData(new string[] { "a" }, "a");
gram2.AddData(new string[] { "a" }, "b");
NGram gram3 = new NGram(3);
gram3.AddData(new string[] { "a", "a" }, "b");
gram.AddData(new string[] { "a", "a" }, "b");
ICompiledGram c1 = gram1.Compile();
ICompiledGram c2 = gram2.Compile();
ICompiledGram c3 = gram3.Compile();
string[] input = new string[] { "a", "a", "b" };
double expected =
uniweight * c1.SequenceProbability(input) +
biWeight * c2.SequenceProbability(input) +
triWeight * c3.SequenceProbability(input);
double actual = gram.Compile().SequenceProbability(new string[] { "a", "a", "b" });
Assert.IsTrue(
Mathf.Approximately((float)expected, (float)actual),
$"Expected {expected} but received {actual}.");
}
public void TestGet()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.6f);
gram.AddData(new string[] { "b", "a" }, "a");
gram.AddData(new string[] { "b", "a" }, "c");
gram.AddData(new string[] { "b", "c" }, "c");
gram.AddData(new string[] { "a", "a" }, "d");
ICompiledGram compiledGram = gram.Compile();
Assert.IsTrue(FoundValue(compiledGram, "c", new string[] { "b", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "d", new string[] { "b", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "a", new string[] { "b", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "c", new string[] { "b", "c" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "d", new string[] { "b", "c" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "a", new string[] { "b", "c" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "c", new string[] { "a", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "d", new string[] { "a", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "a", new string[] { "a", "a" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "c", new string[] { "z", "d" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "d", new string[] { "z", "d" }, 1000));
Assert.IsTrue(FoundValue(compiledGram, "a", new string[] { "z", "d" }, 1000));
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiledGram.Get(null);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiledGram.Get(new string[] { "z" });
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiledGram.Get(new string[] { "z", "a", "d" });
});
}
public void TestAddHierarchicalNGram()
{
HierarchicalNGram a = new HierarchicalNGram(3, 0.6f);
a.AddData(new string[] { "a", "b" }, "c");
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
NGram n3 = a.Grammars[2] as NGram;
HierarchicalNGram b = new HierarchicalNGram(3, 0.6f);
b.AddData(new string[] { "a", "b" }, "c");
b.AddData(new string[] { "a", "b" }, "c");
b.AddData(new string[] { "c", "b" }, "c");
b.AddData(new string[] { "b", "b" }, "d");
a.AddGrammar(b);
// a, b, c, d
Assert.AreEqual(4, u1.Grammar.Keys.Count);
Assert.AreEqual(3f, u1.Grammar["a"]);
Assert.AreEqual(6f, u1.Grammar["b"]);
Assert.AreEqual(5f, u1.Grammar["c"]);
Assert.AreEqual(1f, u1.Grammar["d"]);
// ab, bc, cb, bc, bb, bd
Assert.AreEqual(3, n2.Grammar.Count);
Assert.AreEqual(3f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(4f, n2.Grammar["b"].Grammar["c"]);
Assert.AreEqual(1f, n2.Grammar["c"].Grammar["b"]);
Assert.AreEqual(1f, n2.Grammar["b"].Grammar["b"]);
Assert.AreEqual(1f, n2.Grammar["b"].Grammar["d"]);
// abc, cbd, bbd
Assert.AreEqual(3f, n3.Grammar.Count);
Assert.AreEqual(3f, n3.Grammar["a,b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["c,b"].Grammar["c"]);
Assert.AreEqual(1f, n3.Grammar["b,b"].Grammar["d"]);
}
public void TestConstruction()
{
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
IGram test = new HierarchicalNGram(0, 0.6f);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
IGram test = new HierarchicalNGram(1, 0.6f);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
new HierarchicalNGram(1, 0f);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
new HierarchicalNGram(1, 1f);
});
for (int i = 2; i < 15; ++i)
{
Assert.DoesNotThrow(() => { IGram test = new HierarchicalNGram(i, 0.6f); });
}
HierarchicalNGram a = new HierarchicalNGram(3, 0.6f);
Assert.AreEqual(3, a.Grammars.Length);
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
NGram n3 = a.Grammars[2] as NGram;
Assert.NotNull(u1);
Assert.NotNull(n2);
Assert.NotNull(n3);
}
public void TestUpdateMemory()
{
HierarchicalNGram a = new HierarchicalNGram(3, 0.6f);
UniGram u1 = a.Grammars[0] as UniGram;
NGram n2 = a.Grammars[1] as NGram;
NGram n3 = a.Grammars[2] as NGram;
a.AddData(new string[] { "a", "b" }, "c");
a.UpdateMemory(0.9f);
Assert.AreEqual(3, u1.Grammar.Keys.Count);
Assert.AreEqual(2, n2.Grammar.Keys.Count);
Assert.AreEqual(1, n3.Grammar.Keys.Count);
Assert.AreEqual(0.9f, u1.Grammar["a"]);
Assert.AreEqual(0.9f, u1.Grammar["b"]);
Assert.AreEqual(0.9f, u1.Grammar["c"]);
Assert.AreEqual(0.9f, n2.Grammar["a"].Grammar["b"]);
Assert.AreEqual(0.9f, n2.Grammar["b"].Grammar["c"]);
Assert.AreEqual(0.9f, n3.Grammar["a,b"].Grammar["c"]);
a.UpdateMemory(0.9f);
Assert.AreEqual(3, u1.Grammar.Keys.Count);
Assert.AreEqual(2, n2.Grammar.Keys.Count);
Assert.AreEqual(1, n3.Grammar.Keys.Count);
Assert.IsTrue(Mathf.Approximately(0.81f, u1.Grammar["a"]));
Assert.IsTrue(Mathf.Approximately(0.81f, u1.Grammar["b"]));
Assert.IsTrue(Mathf.Approximately(0.81f, u1.Grammar["c"]));
Assert.IsTrue(Mathf.Approximately(0.81f, n2.Grammar["a"].Grammar["b"]));
Assert.IsTrue(Mathf.Approximately(0.81f, n2.Grammar["b"].Grammar["c"]));
Assert.IsTrue(Mathf.Approximately(0.81f, n3.Grammar["a,b"].Grammar["c"]));
}
public void TestGetValues()
{
HierarchicalNGram uncompiled = new HierarchicalNGram(2, 0.6f);
ICompiledGram compiled = uncompiled.Compile();
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(null);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(new string[] { "a", "b" });
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(new string[] { "a", "b", "c" });
});
Assert.AreEqual(0, compiled.GetValues(new string[] { "a" }).Keys.Count);
// Test with one entry a->c
uncompiled.AddData(new string[] { "a" }, "c");
compiled = uncompiled.Compile();
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(null);
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(new string[] { "a", "b" });
});
Assert.Throws <UnityEngine.Assertions.AssertionException>(() =>
{
compiled.GetValues(new string[] { "a", "b", "c" });
});
float uniWeight = (0.36f / (0.6f + 0.36f));
float biWeight = (0.6f / (0.6f + 0.36f));
Dictionary <string, float> values = compiled.GetValues(new string[] { "z" });
Assert.AreEqual(2, values.Keys.Count);
Assert.AreEqual(0.5f, values["a"]);
Assert.AreEqual(0.5f, values["c"]);
values = compiled.GetValues(new string[] { "a" });
Assert.AreEqual(2, values.Keys.Count);
Assert.AreEqual(0.5 * uniWeight, values["a"]);
Assert.AreEqual(biWeight + 0.5 * uniWeight, values["c"]);
// test with three entries a->c, b->c & b->d
uncompiled.AddData(new string[] { "b" }, "c");
uncompiled.AddData(new string[] { "b" }, "d");
compiled = uncompiled.Compile();
// in this case we haven't seen the prior "z" so we only have the
// unigram to work with
values = compiled.GetValues(new string[] { "z" });
Assert.AreEqual(4, values.Keys.Count);
Assert.IsTrue(values.ContainsKey("a")); // 1
Assert.IsTrue(values.ContainsKey("b")); // 2
Assert.IsTrue(values.ContainsKey("c")); // 2
Assert.IsTrue(values.ContainsKey("d")); // 1
Assert.AreEqual(1 / 6f, values["a"]);
Assert.AreEqual(2 / 6f, values["b"]);
Assert.AreEqual(2 / 6f, values["c"]);
Assert.AreEqual(1 / 6f, values["d"]);
// we have the prior a, so we are working with it and the unigram
values = compiled.GetValues(new string[] { "a" });
Assert.AreEqual(4, values.Keys.Count);
Assert.IsTrue(values.ContainsKey("a"));
Assert.IsTrue(values.ContainsKey("b"));
Assert.IsTrue(values.ContainsKey("c"));
Assert.IsTrue(values.ContainsKey("d"));
Assert.AreEqual(1 / 6f * uniWeight, values["a"]); // only unigram
Assert.AreEqual(2 / 6f * uniWeight, values["b"]); // only unigram
Assert.AreEqual(biWeight + 2 / 6f * uniWeight, values["c"]); // uni-gram and bi-gram
Assert.AreEqual(1 / 6f * uniWeight, values["d"]); // only unigram
// we have the prior b, so we are working with it and the unigram
values = compiled.GetValues(new string[] { "b" });
Assert.AreEqual(4, values.Keys.Count);
Assert.IsTrue(values.ContainsKey("a"));
Assert.IsTrue(values.ContainsKey("b"));
Assert.IsTrue(values.ContainsKey("c"));
Assert.IsTrue(values.ContainsKey("d"));
Assert.AreEqual(1 / 6f * uniWeight, values["a"]); // only unigram
Assert.AreEqual(2 / 6f * uniWeight, values["b"]); // only unigram
Assert.AreEqual(0.5f * biWeight + 2 / 6f * uniWeight, values["c"]); // uni-gram and bi-gram
Assert.AreEqual(0.5f * biWeight + 1 / 6f * uniWeight, values["d"]); // only unigram
}
public void TestClone()
{
HierarchicalNGram gram = new HierarchicalNGram(3, 0.6f);
gram.AddData(new string[] { "b", "a" }, "a");
gram.AddData(new string[] { "b", "a" }, "z");
gram.AddData(new string[] { "b", "f" }, "2");
gram.AddData(new string[] { "c", "g" }, "a");
gram.AddData(new string[] { "d", "h" }, "e");
gram.AddData(new string[] { "e", "h" }, "c");
CompiledHierarchicalNGram compiledGram = gram.Compile() as CompiledHierarchicalNGram;
Assert.IsNotNull(compiledGram);
CompiledHierarchicalNGram clone = compiledGram.Clone() as CompiledHierarchicalNGram;
Assert.IsNotNull(clone);
Assert.AreEqual(compiledGram.GetN(), clone.GetN());
// uni-gram
TestValues(
compiledGram.CompiledGrammars[0] as CompiledUniGram,
clone.CompiledGrammars[0] as CompiledUniGram,
null);
// bi-gram
TestValues(
compiledGram.CompiledGrammars[1] as CompiledNGram,
clone.CompiledGrammars[1] as CompiledNGram,
new string[] { "a" });
TestValues(
compiledGram.CompiledGrammars[1] as CompiledNGram,
clone.CompiledGrammars[1] as CompiledNGram,
new string[] { "f" });
TestValues(
compiledGram.CompiledGrammars[1] as CompiledNGram,
clone.CompiledGrammars[1] as CompiledNGram,
new string[] { "g" });
TestValues(
compiledGram.CompiledGrammars[1] as CompiledNGram,
clone.CompiledGrammars[1] as CompiledNGram,
new string[] { "h" });
// tri-gram
TestValues(
compiledGram.CompiledGrammars[2] as CompiledNGram,
clone.CompiledGrammars[2] as CompiledNGram,
new string[] { "b", "a" });
TestValues(
compiledGram.CompiledGrammars[2] as CompiledNGram,
clone.CompiledGrammars[2] as CompiledNGram,
new string[] { "b", "f" });
TestValues(
compiledGram.CompiledGrammars[2] as CompiledNGram,
clone.CompiledGrammars[2] as CompiledNGram,
new string[] { "c", "g" });
TestValues(
compiledGram.CompiledGrammars[2] as CompiledNGram,
clone.CompiledGrammars[2] as CompiledNGram,
new string[] { "d", "h" });
TestValues(
compiledGram.CompiledGrammars[2] as CompiledNGram,
clone.CompiledGrammars[2] as CompiledNGram,
new string[] { "e", "h" });
}