public void TestMissingStart01() {
var productions = new List<Production> {
CFGParser.Production(@"<X_0> -> <X_0> <X_0>"),
CFGParser.Production(@"<X_0> -> 'a'"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
}
public static void Do() {
// S -> aSa | bSb | ε
var productions = new List<Production> {
// construct productions by passing arguments...
new Production(
lhs: Nonterminal.Of("S"),
rhs: new Sentence { Terminal.Of("a"), Nonterminal.Of("S"), Terminal.Of("a") },
weight: 20
),
// or from a string...
CFGParser.Production(@"<S> -> 'b' <S> 'b' [10]"),
CFGParser.Production(@"<S> -> ε [5]"),
};
var cfg = new Grammar(productions, Nonterminal.Of("S"));
// var cnf = cfg.ToCNF();
//var probs = cfg.EstimateProbabilities(1000000);
//foreach (var entry in probs) {
// var key = entry.Key;
// var value = entry.Value;
// if (key.Length <= 4) {
// Console.WriteLine("{0}: {1}", key, value);
// }
//}
// Print out the new CNF grammar
// Console.WriteLine(cnf);
var ep = new EarleyParser(cfg);
// var cp = new CykParser(cnf);
// Does this grammar accept the string "aabb"?
Console.WriteLine("aabb: {0}", ep.ParseGetProbability(Sentence.FromLetters("aabb")));
// How about "abba"?
Console.WriteLine("abba: {0}", ep.ParseGetProbability(Sentence.FromLetters("abba")));
Console.WriteLine(ep.ParseGetForest(Sentence.FromLetters("abba")));
for (int i = 0; i < 5; i++) {
Console.WriteLine(cfg.ProduceRandom().AsTerminals());
}
var sentences = cfg.ProduceToDepth(3);
foreach (var sentence in sentences) {
Console.WriteLine(sentence.Value.AsTerminals());
}
var gg = new GrammarGenerator(1);
var terminals = new List<Terminal> { Terminal.Of("a"), Terminal.Of("b") };
var randGram = gg.NextCFG(
numNonterminals: 4,
numProductions: 10,
maxProductionLength: 4,
terminals: terminals
);
Console.WriteLine(randGram);
}
public void TestParsing02() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → <X_1>"),
CFGParser.Production("<X_1> → 'x1' 'x2'"),
CFGParser.Production("<X_1> → ε")
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x1 x2"));
ExecuteTest(g, sentences);
}
public void TestFreshNames() {
var productions = new List<Production> {
CFGParser.Production(@"<S> -> 'a'"),
CFGParser.Production(@"<X_0> -> 'b'"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
Assert.IsTrue(h.Accepts(Sentence.FromLetters("a")));
Assert.IsFalse(h.Accepts(Sentence.FromLetters("b")));
}
private static void ExecuteTest(Grammar g, List<Sentence> sentences) {
CNFGrammar h = g.ToCNF();
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
foreach (var sentence in sentences) {
var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
}
}
public void TestParsing03() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<S> → <X> 'b'"),
CFGParser.Production("<S> → 'a'"),
CFGParser.Production("<X> → <S>"),
CFGParser.Production("<X> → 'x'")
}, Nonterminal.Of("S"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("a b b"));
ExecuteTest(g, sentences);
}
public void TestToCNF01() {
var productions = new List<Production> {
CFGParser.Production(@"<X_0> -> 'x3' <X_0> [23]"),
CFGParser.Production(@"<X_0> -> ε [85]"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("X_0"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(85.0 / 108.0, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear((23.0 / 108.0) * (85.0 / 108.0), h.Cyk(Sentence.FromWords("x3")));
Helpers.AssertNear((23.0 / 108.0) * (23.0 / 108.0) * (85.0 / 108.0), h.Cyk(Sentence.FromWords("x3 x3")));
}
public void TestCNFNoNull02() {
var productions = new HashSet<Production> {
CFGParser.Production("<A> -> <A>"),
CFGParser.Production("<A> -> 'a'"),
CFGParser.Production("<A> -> 'b'"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("A"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(0, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear(0.5, h.Cyk(Sentence.FromLetters("a")));
Helpers.AssertNear(0.5, h.Cyk(Sentence.FromLetters("b")));
}
public void TestUnitProductions01() {
var productions = new List<Production> {
CFGParser.Production(@"<S> -> <A> [3]"),
CFGParser.Production(@"<S> -> <B> [1]"),
CFGParser.Production(@"<A> -> 'a'"),
CFGParser.Production(@"<B> -> 'b'"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(0.0, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear(0.75, h.Cyk(Sentence.FromLetters("a")));
Helpers.AssertNear(0.25, h.Cyk(Sentence.FromLetters("b")));
}
public void TestCFGToCNFBadProb01() {
// S -> aSa | bSb | ε
var productions = new List<Production> {
CFGParser.Production(@"<S> -> 'a' <S> 'a' [1]"),
CFGParser.Production(@"<S> -> 'b' <S> 'b' [3]"),
CFGParser.Production(@"<S> -> ε [4]"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(0.5, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear((1.0 / 8) * 0.5, h.Cyk(Sentence.FromLetters("aa")));
Helpers.AssertNear((3.0 / 8) * 0.5, h.Cyk(Sentence.FromLetters("bb")));
Helpers.AssertNear((1.0 / 8) * (3.0 / 8) * 0.5, h.Cyk(Sentence.FromLetters("abba")));
}
public void TestUnitProductions02() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_5> → 'x0' [22.103242747999378]"),
CFGParser.Production("<X_5> → <X_4> <X_2> 'x3' 'x3' <X_5> 'x4' <X_5> [13.799233343824387]"),
CFGParser.Production("<X_7> → 'x1' <X_0> 'x3' <X_9> 'x1' 'x0' <X_9> <X_5> [95.444187448566865]"),
CFGParser.Production("<X_7> → ε [92.294680857702474]"),
CFGParser.Production("<X_2> → <X_9> <X_4> [67.541615066370753]"),
CFGParser.Production("<X_9> → <X_0> <X_7> 'x3' <X_4> 'x2' 'x1' 'x3' [36.206671634319555]"),
CFGParser.Production("<X_0> → <X_4> [41.7635276861319]"),
CFGParser.Production("<X_4> → <X_0> [85.572940545423393]"),
CFGParser.Production("<X_4> → <X_9> 'x2' 'x1' <X_9> 'x2' 'x0' [19.93626149694261]"),
CFGParser.Production("<X_0> → 'x3' [13.794410955530783]"),
CFGParser.Production("<X_9> → ε [86.73295911109679]")
}, Nonterminal.Of("X_0"));
var h = g.ToCNF();
}
public void TestParsing01() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → <X_2> <X_6> [84.663251941866818]"),
CFGParser.Production("<X_5> → ε [58.744849453561407]"),
CFGParser.Production("<X_6> → 'x0' [14.931547298064245]"),
CFGParser.Production("<X_1> → <X_4> 'x3' 'x2' [8.0317742447516771]"),
CFGParser.Production("<X_2> → <X_6> [31.352648361750251]"),
CFGParser.Production("<X_5> → 'x2' [70.697925527439423]"),
CFGParser.Production("<X_4> → ε [95.484905709738328]"),
CFGParser.Production("<X_2> → <X_2> [1]"),
CFGParser.Production("<X_2> → <X_1> <X_5> <X_5> <X_1> 'x2' <X_5> <X_0> [76.400903250743127]")
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x0 x0"));
ExecuteTest(g, sentences);
}
public void TestParsing21() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<S> → ε [45.90244483803513]"),
CFGParser.Production("<S> → <S> <S> [83.928846908234448]"),
}, Nonterminal.Of("S"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords(""));
ExecuteTest(g, sentences);
}
public void TestParsing20() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<S> → <A> <A>"),
CFGParser.Production("<A> → 'a' <A>"),
CFGParser.Production("<A> → ε")
}, Nonterminal.Of("S"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("a"));
ExecuteTest(g, sentences);
}
public void TestParsing19() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → <X_2> 'x2' 'x0' [89.380999392075935]"),
CFGParser.Production("<X_2> → <X_0> [54.4160114142187]"),
CFGParser.Production("<X_0> → 'x1' [73.603592962307658]"),
CFGParser.Production("<X_2> → 'x2' 'x2' 'x1' <X_2> [72.673047343116735]")
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x1"));
ExecuteTest(g, sentences);
}
public void TestParsing18() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_9> → 'x3' <X_4> <X_9> [69.71867415901211]"),
CFGParser.Production("<X_6> → 'x4' [43.169519673180545]"),
CFGParser.Production("<X_0> → 'x0' 'x3' <X_6> <X_9> <X_9> [95.5660355475573]"),
CFGParser.Production("<X_5> → <X_9> 'x1' 'x0' 'x1' 'x3' <X_2> [35.638882444537657]"),
CFGParser.Production("<X_1> → 'x4' 'x3' 'x1' 'x1' <X_9> <X_8> [60.963767072169006]"),
CFGParser.Production("<X_9> → <X_6> [96.869668710916145]"),
CFGParser.Production("<X_8> → 'x1' <X_0> 'x0' <X_2> <X_2> [10.412202848779131]"),
CFGParser.Production("<X_4> → ε [89.394112460498746]"),
CFGParser.Production("<X_4> → <X_8> 'x2' <X_5> 'x1' [41.46934854261081]"),
CFGParser.Production("<X_2> → ε [28.04076097674703]"),
CFGParser.Production("<X_8> → ε [55.798571558109757]"),
CFGParser.Production("<X_0> → 'x2' 'x2' 'x3' <X_6> [48.407048357374521]"),
CFGParser.Production("<X_0> → <X_1> 'x3' 'x2' [82.3935272774629]"),
CFGParser.Production("<X_1> → <X_8> <X_1> <X_2> [68.051246746932733]")
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x3 x2"));
ExecuteTest(g, sentences);
}
public void TestToCNF03() {
var productions = new HashSet<Production> {
CFGParser.Production("<A> -> <B> <C>"),
CFGParser.Production("<B> -> <C>"),
CFGParser.Production("<B> -> 'b'"),
CFGParser.Production("<B> -> ε"),
CFGParser.Production("<C> -> <B>"),
CFGParser.Production("<C> -> 'c'"),
CFGParser.Production("<C> -> ε"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("A"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(0.25, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear(0.25, h.Cyk(Sentence.FromLetters("b")));
Helpers.AssertNear(0.25, h.Cyk(Sentence.FromLetters("c")));
Helpers.AssertNear(0.140625, h.Cyk(Sentence.FromLetters("bc")));
Helpers.AssertNear(0.046875, h.Cyk(Sentence.FromLetters("cc")));
Helpers.AssertNear(0.046875, h.Cyk(Sentence.FromLetters("bb")));
Helpers.AssertNear(0.015625, h.Cyk(Sentence.FromLetters("cb")));
}
public void TestParsing12() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<A> → <B>"),
CFGParser.Production("<B> → <A>"),
CFGParser.Production("<B> → 'x'"),
CFGParser.Production("<B> → 'z'"),
}, Nonterminal.Of("A"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x"));
ExecuteTest(g, sentences);
}
internal CFGtoCNF(Grammar grammar) {
_grammar = grammar;
}
public void TestToCNF05() {
Grammar g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → 'x4' <X_4> [16.517998587115667]"),
CFGParser.Production("<X_4> → 'x3' [49.290950734303777]"),
CFGParser.Production("<X_0> → 'x4' 'x1' [23.628313965456705]")
}, Nonterminal.Of("X_0"));
CNFGrammar h = g.ToCNF();
var first = 16.517998587115667;
var third = 23.628313965456705;
var sum = first + third;
Helpers.AssertNear(third / sum, h.Cyk(Sentence.FromWords("x4 x1")));
Helpers.AssertNear(first / sum , h.Cyk(Sentence.FromWords("x4 x3")));
}
public void TestToCNF02() {
var productions = new List<Production> {
CFGParser.Production(@"<S> -> 'a' <B> <B>"),
CFGParser.Production(@"<B> -> 'b'"),
CFGParser.Production(@"<B> -> ε"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
var pa = h.Cyk(Sentence.FromLetters("a"));
var pab = h.Cyk(Sentence.FromLetters("ab"));
var pabb = h.Cyk(Sentence.FromLetters("abb"));
Assert.IsTrue(pa > 0.0);
Assert.IsTrue(pab > 0.0);
Assert.IsTrue(pabb > 0.0);
Helpers.AssertNear(1.0, pa + pab + pabb);
}
public void TestToCNF06() {
Grammar g = new Grammar(new List<Production>{
CFGParser.Production("<S> → ε"),
CFGParser.Production("<S> → 'x' <A>")
}, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
Helpers.AssertNear(1.0, h.Cyk(Sentence.FromWords("")));
}
public void TestParsing22() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → <X_1> 'x2' [6.6639601037204077]"),
CFGParser.Production("<X_1> → 'x0' 'x2' 'x2' [90.202239638754278]"),
CFGParser.Production("<X_0> → <X_0> [74.838852799422511]"),
CFGParser.Production("<X_1> → 'x1' [41.753055941664172]")
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x1 x2"));
ExecuteTest(g, sentences);
}
public void TestParsing15() {
// S -> aSa | bSb | ε
var g = new Grammar(new List<Production> {
CFGParser.Production(@"<S> -> 'a' <S> 'a'"),
CFGParser.Production(@"<S> -> 'b' <S> 'b'"),
CFGParser.Production(@"<S> -> ε"),
}, Nonterminal.Of("S"));
var sentences = new List<Sentence> {
Sentence.FromLetters("ab"),
Sentence.FromLetters("abc"),
Sentence.FromLetters("aaa"),
Sentence.FromLetters("abbba"),
Sentence.FromLetters(""),
Sentence.FromLetters("aa"),
Sentence.FromLetters("bb"),
Sentence.FromLetters("abba"),
Sentence.FromLetters("baab"),
Sentence.FromLetters("aaaa"),
Sentence.FromLetters("bbbb"),
Sentence.FromLetters("aaabbabbabbaaa"),
};
ExecuteTest(g, sentences);
}
public void TestToCNF04() {
var productions = new HashSet<Production> {
CFGParser.Production("<A> -> <A> <B>"),
CFGParser.Production("<A> -> ε"),
CFGParser.Production("<B> -> 'b'"),
CFGParser.Production("<B> -> ε"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("A"));
CNFGrammar h = g.ToCNF();
var third = 1.0 / 3.0;
Helpers.AssertNear(0.5 + third * 0.5, h.Cyk(Sentence.FromLetters("")));
Helpers.AssertNear(third * 2.0 / 3.0, h.Cyk(Sentence.FromLetters("b")));
}
public void TestParsing16() {
var g = new Grammar(new List<Production> {
CFGParser.Production("<X_0> -> <X_0> <X_1>"),
CFGParser.Production("<X_2> -> 'b'"),
CFGParser.Production("<X_0> -> 'b'"),
CFGParser.Production("<X_0> -> 'a'"),
CFGParser.Production("<X_2> -> 'a'"),
}, Nonterminal.Of("X_0"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("x"));
sentences.Add(Sentence.FromLetters("abc"));
sentences.Add(Sentence.FromLetters("ab"));
sentences.Add(Sentence.FromLetters("ba"));
sentences.Add(Sentence.FromLetters("aabb"));
sentences.Add(Sentence.FromLetters("abba"));
sentences.Add(Sentence.FromLetters("a"));
sentences.Add(Sentence.FromLetters("b"));
ExecuteTest(g, sentences);
}
static void Main(string[] args) {
//var rand = new Random(0);
//Experimental.TestSolver(rand);
// RandomTests.RandomJacobianTest();
var t = new TestCFGToCNF();
var tp = new TestCFGToCNFEmptyProb();
var tr = new RegressionTests();
var testp = new TestParsing();
// testp.TestParsing21();
// testp.TestWeirdSppf06();
// testp.TestWeirdSppf07();
// Console.Read();
//var g = new Grammar(new List<Production>{
// CFGParser.Production("<S> → 'x' <X>"),
// CFGParser.Production("<S> → <X> 'x'"),
// CFGParser.Production("<S> → 'x' 'x'"),
// CFGParser.Production("<X> → 'x'"),
//}, Nonterminal.Of("S"));
//var g = new Grammar(new List<Production>{
// CFGParser.Production("<S> → <S> <S> <S>"),
// CFGParser.Production("<S> → 'x'"),
// CFGParser.Production("<S> → ε"),
//}, Nonterminal.Of("S"));
//var g = new Grammar(new List<Production>{
// CFGParser.Production("<S> → <S> <S>"),
// CFGParser.Production("<S> → 'b'"),
// CFGParser.Production("<S> → ε"),
//}, Nonterminal.Of("S"));
var g = new Grammar(new List<Production>{
CFGParser.Production("<S> → <S> '+' <S>"),
// CFGParser.Production("<S> → <S> '*' <S>"),
// CFGParser.Production("<S> → [0-9]+"),
CFGParser.Production("<S> → '0'"),
// CFGParser.Production("<S> → '2'"),
}, Nonterminal.Of("S"));
//var ests = g.EstimateProbabilities(10000);
//foreach (var est in ests) {
// Console.WriteLine("{0}: {1}", est.Key, est.Value);
//}
// 0 + 123 * 72
var ep = new EarleyParser(g);
var sppf = ep.ParseGetForest(Sentence.FromWords("0 + 0 + 0"));
// var sppf = ep.ParseGetForest(Sentence.FromWords("x x"));
// var sppf = ep.ParseGetForest(Sentence.FromWords("b"));
//Console.WriteLine();
Console.WriteLine(sppf);
// var dot = ForestHelpers.ToDot(sppf);
var rawdot = sppf.GetRawDot();
DotRunner.Run(rawdot, "rawGraph");
var dot = sppf.ToDot();
DotRunner.Run(dot, "addition");
// var dotShared = ForestHelpers.ToDot(sppf, true);
//var dotShared = sppf.ToDot(true);
//DotRunner.Run(dotShared, "additionShared");
//var pp = new PrettyPrinter();
//sppf.Accept(pp);
//Console.WriteLine(pp.Result);
//// Console.WriteLine(sppf.ToStringHelper("", new HashSet<Sppf>()));
//Console.WriteLine();
// Readme.Do();
//var p = CFGParser.Production("<S> -> 'a' [5]");
//Console.WriteLine(p);
//Console.Read();
//return;
//var rt = new CFGLibTest.RandomTests();
//var sw = Stopwatch.StartNew();
//// rt.RandomParsingTest(50000, 4, 3, 5, 4, 6, 1);
//// rt.RandomParsingTest(500, 10, 5, 30, 8, 6);
//rt.RandomParsingTest(1, 10, 5, 50, 8, 6);
//sw.Stop();
//Console.WriteLine("Elapsed: {0}s", sw.Elapsed.TotalMilliseconds / 1000.0);
Console.WriteLine("Finished!");
Console.Read();
}
public void TestToCNF07() {
Grammar g = new Grammar(new List<Production>{
CFGParser.Production("<X_0> → 'x1' 'x4' <X_2> <X_6> [48.024797111295534]"),
CFGParser.Production("<X_1> → <X_0> 'x0' [28.859845304796398]"),
CFGParser.Production("<X_4> → 'x4' 'x4' 'x4' <X_3> <X_6> [32.396577754708275]"),
CFGParser.Production("<X_4> → ε [46.519217869974312]"),
CFGParser.Production("<X_0> → 'x4' 'x0' 'x3' 'x3' [90.918005973993814]"),
CFGParser.Production("<X_6> → <X_4> <X_0> 'x3' <X_3> [31.319837867431264]"),
CFGParser.Production("<X_2> → <X_2> [72.917730323932]"),
CFGParser.Production("<X_3> → <X_1> [36.901735786302822]"),
CFGParser.Production("<X_6> → ε [40.092343899464396]"),
CFGParser.Production("<X_2> → 'x1' [16.801839537826293]"),
CFGParser.Production("<X_0> → ε [136.892430380868]")
}, Nonterminal.Of("X_0"));
CNFGrammar h = g.ToCNF();
// Helpers.AssertNear(1.0, h.Cyk(Sentence.FromWords("")));
}
public static ISet<Production> CloneGrammar(Grammar grammar) {
return CloneProductions(grammar.Productions);
}
public void TestParsing09() {
var g = new Grammar(new List<Production>{
CFGParser.Production("<S> → <A> 'b'"),
CFGParser.Production("<S> → <AB>"),
CFGParser.Production("<S> → <ABPrime>"),
CFGParser.Production("<A> → 'a'"),
CFGParser.Production("<AB> → 'a' 'b'"),
CFGParser.Production("<ABPrime> → 'a' 'b'"),
}, Nonterminal.Of("S"));
var sentences = new List<Sentence>();
sentences.Add(Sentence.FromWords("a b"));
ExecuteTest(g, sentences);
}
