private static void TraversePlay() { var g = new Grammar(new List <Production> { CFGParser.Production("<A> → <B>"), CFGParser.Production("<B> → <A>"), CFGParser.Production("<B> → 'x'"), }, Nonterminal.Of("A")); g = IdentityActions.Annotate(g); var earley2 = new EarleyParser2(g); var sentence = Sentence.FromWords("x"); var sppf2 = earley2.ParseGetForest(sentence); DotRunner.Run(DotBuilder.GetRawDot(sppf2), "infinite"); var t2 = new Traversal(sppf2, g); var r2 = t2.Traverse(); //foreach (var option in r2) { // var s2 = (Sentence)option.Payload; // if (!sentence.SequenceEqual(s2)) { // throw new Exception(); // } //} }
private static void ParserGenerator() { var g = Ebnf.GrammarLayout(); var earley = new EarleyParser2(g); var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic_ebnf); // var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Bnf); // var sentence2 = Sentence.FromLetters("<S> ::= <S> '+' <S>\r\n<S> ::= '1'\r\n"); // if (!sentence1.Equals(sentence2)) { } // int index = sentence1.Zip(sentence2, (c1, c2) => c1 == c2).TakeWhile(b => b).Count() + 1; var sppf = earley.ParseGetForest(sentence1); if (sppf == null) { throw new Exception(); } DotRunner.Run(DotBuilder.GetRawDot(sppf), "arithmetic_ebnf"); //var traversal = new Traversal(sppf, sentence1, g); //var result = traversal.Traverse(); //if (result.Count() != 1) { // throw new Exception(); //} //var generatedGrammar = new Grammar((IEnumerable<Production>)result.First().Payload, Nonterminal.Of("S")); //Console.WriteLine(generatedGrammar); }
private static void DevelopAnnotations() { var sw = Stopwatch.StartNew(); var input = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic_annotated); var inputWithoutLayout = Ebnf.RemoveLayout(input, out _); var grammar = Ebnf.GrammarSyntax(); var parser = new EarleyParser2(grammar); var sppf = parser.ParseGetForest(inputWithoutLayout); if (sppf == null) { throw new Exception(); } var ms1 = sw.Elapsed.TotalMilliseconds; Console.WriteLine("Parse: {0:0.#}ms", ms1); DotRunner.Run(DotBuilder.GetRawDot(sppf), "annotations"); var traversal = new Traversal(sppf, grammar); var resCollection = traversal.Traverse(); if (resCollection.Count() > 1) { throw new Exception("ambiguous"); } var res = resCollection.Single(); Console.WriteLine(res); }
private static double MinTime(int times, EarleyParser2 ep, BaseGrammar grammar, Sentence input) { double fastest = double.MaxValue; var sw = new Stopwatch(); for (var i = 0; i < times; i++) { sw.Restart(); // var sppf = ep.ParseGetRawSppf(input); var sppf = ep.ParseGetForest(input); //var trav = new Traversal(sppf, input, gp); //var resultList = trav.Traverse(); //if (resultList.Count() != 1) { // throw new Exception(); //} //var result = (long)resultList.First().Payload; //if (result != expectedResult) { // throw new Exception(); //} var elapsed = sw.Elapsed.TotalMilliseconds; if (elapsed < fastest) { fastest = elapsed; } } return(fastest); }
private static void EbnfPlay() { var input = Sentence.FromLetters(Grammars.Properties.Resources.Ebnf_bench); Sentence inputNoLayout = Ebnf.RemoveLayout(input, out var layoutSppf); //DotRunner.Run(DotBuilder.GetRawDot(layoutSppf), "arithmetic_ebnf_layout"); Console.WriteLine(inputNoLayout.AsTerminals()); var layoutGrammar = Ebnf.GrammarSyntax(); var earley = new EarleyParser2(layoutGrammar); var sppf = earley.ParseGetForest(inputNoLayout); if (sppf == null) { throw new Exception(); } //DotRunner.Run(DotBuilder.GetRawDot(sppf), "arithmetic_ebnf"); //var traversal = new Traversal(sppf, input, layoutGrammar); //var result = traversal.Traverse(); //if (result.Count() != 1) { // throw new Exception(); //} //var inputNoLayout = new Sentence((List<Terminal>)result.First().Payload); //return inputNoLayout; //Console.WriteLine(inputNoLayout); }
private static void ExecuteTest(Grammar g, string input) { var earley1 = new EarleyParser(g); var earley2 = new EarleyParser2(g); var sentence = Sentence.FromWords(input); var p1 = earley1.ParseGetProbability(sentence); var p2 = earley2.ParseGetProbability(sentence); Helpers.AssertNear(p1, p2); }
private static void ExecuteTest(Grammar g, string input) { var ag = IdentityActions.Annotate(g); var earley1 = new EarleyParser(ag); var earley2 = new EarleyParser2(ag); var sentence = Sentence.FromWords(input); var sppf1 = earley1.ParseGetForest(sentence); var sppf2 = earley2.ParseGetForest(sentence); CheckTraversal(ag, sentence, sppf1); CheckTraversal(ag, sentence, sppf2); }
private static void EbnfParse(Sentence sentence) { var noLayoutSentence = Ebnf.RemoveLayout(sentence, out var layoutSppf); var g = Ebnf.GrammarSyntax(); //var earley = new EarleyParser(g); var earley2 = new EarleyParser2(g); //var sppf1 = earley.ParseGetForest(noLayoutSentence); //Assert.IsNotNull(sppf1); var sppf2 = earley2.ParseGetForest(noLayoutSentence); Assert.IsNotNull(sppf2); }
private static void ExecuteTest(Grammar g, List <Sentence> sentences) { CNFGrammar h = g.ToCNF(); var earley = new EarleyParser(g); var cyk = new CykParser(h); var earley2 = new EarleyParser2(g); foreach (var sentence in sentences) { var p1 = cyk.ParseGetProbability(sentence); var p2 = earley.ParseGetProbability(sentence); var p3 = earley2.ParseGetProbability(sentence); Helpers.AssertNear(p1, p2); Helpers.AssertNear(p2, p3); } }
private static void BnfParse(Sentence sentence) { var g = Bnf.Grammar(); //var h = g.ToCNF(); // too much memory var earley = new EarleyParser(g); // var cyk = new CykParser(h); var earley2 = new EarleyParser2(g); // var p1 = cyk.ParseGetProbability(sentence); var p2 = earley.ParseGetProbability(sentence); var p3 = earley2.ParseGetProbability(sentence); // Helpers.AssertNear(p1, p2); Helpers.AssertNear(p2, p3); Assert.IsTrue(p2 > 0.0); }
// from http://dx.doi.org/10.1016/j.entcs.2008.03.044 private static void PaperExamples() { var ex3 = new Grammar(new List <Production> { CFGParser.Production("<S> → <A> <T>"), CFGParser.Production("<S> → 'a' <T>"), CFGParser.Production("<A> → 'a'"), CFGParser.Production("<A> → <B> <A>"), CFGParser.Production("<B> → ε"), CFGParser.Production("<T> → 'b' 'b' 'b'"), }, Nonterminal.Of("S")); var input = Sentence.FromLetters("abbb"); var sppf1 = new EarleyParser(ex3).ParseGetForest(input); var sppf2 = new EarleyParser2(ex3).ParseGetForest(input); DotRunner.Run(DotBuilder.GetRawDot(sppf1), "example3_old"); DotRunner.Run(DotBuilder.GetRawDot(sppf2), "example3_new"); }
private static void BnfPlay() { var bnf = Bnf.Grammar(); var earley = new EarleyParser2(bnf); var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic); // var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Bnf); // var sentence2 = Sentence.FromLetters("<S> ::= <S> '+' <S>\r\n<S> ::= '1'\r\n"); // if (!sentence1.Equals(sentence2)) { } // int index = sentence1.Zip(sentence2, (c1, c2) => c1 == c2).TakeWhile(b => b).Count() + 1; var sppf = earley.ParseGetForest(sentence1); if (sppf == null) { throw new Exception(); } DotRunner.Run(DotBuilder.GetRawDot(sppf), "bnfPlay"); }
private static void EbnfBench() { var tup = EbnfBenchLayout(); var inputNoLayout = tup.Item1; var ms1 = tup.Item2; var sw = Stopwatch.StartNew(); var grammar = Ebnf.GrammarSyntax(); var earley = new EarleyParser2(grammar); var sppf = earley.ParseGetForest(inputNoLayout); var ms2 = sw.Elapsed.TotalMilliseconds; if (sppf == null) { throw new Exception(); } Console.WriteLine("Parse: {0:0.#}ms", ms2); Console.WriteLine("Total: {0:0.#}ms", ms1 + ms2); }
private static void Benchmark() { Console.WriteLine("Benching..."); var inputs = new List <Tuple <Sentence, long, int> >(); // for (var i = 80; i < 105; i++) { // 13336ms // for (var i = 1; i < 500; i += 1) { // for (var i = 170; i < 195; i++) { // 10755ms after gather in sppf // for (var i = 170; i < 195; i++) { // 9203ms after hash change // for (var i = 170; i < 195; i++) { // 2703ms after doing gather earlier // for (var i = 170; i < 195; i++) { // for (var i = 751; i < 752; i++) { // for (var i = 95; i < 130; i++) { // new; 15385 for (var i = 160; i < 200; i++) { // for (var i = 300; i < 350; i++) { // new; inputs.Add(Tuple.Create(Sentence.FromWords(AdditionInput(i)), (long)i, i)); } var gp = AdditionGrammar(argList => (long)argList[0].Payload + (long)argList[2].Payload); var ep = new EarleyParser2(gp); // var totalSw = Stopwatch.StartNew(); double totalMs = 0; foreach (var inputPair in inputs) { var input = inputPair.Item1; var expectedResult = inputPair.Item2; var i = inputPair.Item3; var time = MinTime(3, ep, gp, input); totalMs += time; Console.WriteLine("{0}, {1}", i, time); } Console.WriteLine("Done in {0}ms (prev 3686ms)", (int)totalMs); //foreach (var kvp in EarleyParser2._stats.Data) { // Console.WriteLine("{0}, {1}", kvp.Key, kvp.Value); //} }
private static void VisitorPlay() { var gp = AdditionGrammar(argList => string.Format("({0} + {1})", argList[0].Payload, argList[2].Payload)); //var actions2 = new Dictionary<Production, IParserAction> { // [p1] = new ParserAction<long>((argList) => (long)(argList[0].Payload) + (long)(argList[2].Payload)), //}; //var termAction2 = new ParserAction<long>(x => Convert.ToInt64(x[0].Payload)); //foreach (var num in nums) { // actions2[num] = termAction2; //} var ep = new EarleyParser2(gp); var inputString = AdditionInput(5); var input = Sentence.FromWords(inputString); var sppf = ep.ParseGetForest(input); var rawdot = DotBuilder.GetRawDot(sppf); DotRunner.Run(rawdot, "newSppf"); Console.WriteLine(sppf.ToString()); Console.WriteLine(); Console.WriteLine("Starting Traversal..."); var trav = new Traversal(sppf, gp); var resultList = trav.Traverse(); Console.WriteLine("-----------------"); foreach (var result in resultList) { Console.WriteLine(result.Payload); } //Console.WriteLine("-----------------"); //foreach (var result in new Traversal(sppf, input, gp).Traverse()) { // Console.WriteLine(result.Payload); //} }
public static Sentence RemoveLayout(Sentence input, out SppfNode sppf) { var layoutGrammar = Ebnf.GrammarLayout(); var earley = new EarleyParser2(layoutGrammar); sppf = earley.ParseGetForest(input); if (sppf == null) { throw new Exception(); } var traversal = new Traversal(sppf, layoutGrammar); var result = traversal.Traverse(); if (result.Count() != 1) { throw new Exception(); } var inputNoLayout = new Sentence((List <Terminal>)result.First().Payload); return(inputNoLayout); }
private static void DebugGrammar() { BaseGrammar g = new Grammar(new List <Production> { CFGParser.Production("<S> → ε"), }, Nonterminal.Of("S")); var sentence = Sentence.FromWords("1 + 1 + 1"); var grammar = AdditionGrammar(argList => string.Format("({0} + {1})", argList[0].Payload, argList[2].Payload)); g = grammar; var earley = new EarleyParser(g); var earley2 = new EarleyParser2(g); //DotRunner.Run(earley.ParseGetForest(sentence).GetRawDot(), "testEarleyOld"); //DotRunner.Run(earley2.ParseGetForest(sentence).GetRawDot(), "testEarleyNew"); DotRunner.Run(DotBuilder.GetRawDot(earley.ParseGetForest(sentence)), "testEarleyOld"); DotRunner.Run(DotBuilder.GetRawDot(earley2.ParseGetForest(sentence)), "testEarleyNew"); // DotRunner.Run(DotBuilder.GetFlattenedDot(earley2.ParseGetForest(sentence)), "testEarleyFlat"); // var prob0 = earley.ParseGetProbability(sentence); var prob = earley2.ParseGetProbability(sentence); }
private bool ProcessOneGrammar() { var(g, terminals) = NextGrammar(); var h = g.ToCNF(); // Console.WriteLine(g.Productions.Count()); var preparedSentences = new List <Sentence>(); for (int length = 0; length <= _maxInputLength; length++) { var combinations = CFGLibTest.Helpers.CombinationsWithRepetition(terminals, length); foreach (var target in combinations) { var sentence = new Sentence(target); preparedSentences.Add(sentence); } } AddRandomSentences(preparedSentences, terminals); var uniquifySentences = new Dictionary <string, Sentence>(); foreach (var sentence in preparedSentences) { uniquifySentences[sentence.AsTerminals()] = sentence; } preparedSentences = uniquifySentences.Values.ToList(); // Console.WriteLine("Parsing sentences..."); EarleyParser earley1; EarleyParser2 earley2; CykParser cyk; try { earley1 = new EarleyParser(g); earley2 = new EarleyParser2(g); cyk = new CykParser(h); } catch (Exception e) { Report(g, e); return(true); } foreach (var sentence in preparedSentences) { try { var sppf1 = earley1.ParseGetForest(sentence); var sppf2 = earley2.ParseGetForest(sentence); if (sppf1 == null && sppf2 != null) { throw new Exception(); } if (sppf2 == null && sppf1 != null) { throw new Exception(); } var p1 = earley1.ProbabilityOfSppf(sppf1); var p2 = earley2.ProbabilityOfSppf(sppf2); var p3 = cyk.ParseGetProbability(sentence); if (!Helpers.IsNear(p1, p2)) { throw new Exception(); } if (!Helpers.IsNear(p1, p3)) { throw new Exception(); } try { TestTraversal.CheckTraversal(g, sentence, sppf1); } catch (TraversalLoopException) { } try { TestTraversal.CheckTraversal(g, sentence, sppf2); } catch (TraversalLoopException) { } } catch (Exception e) { Report(g, e, sentence); return(true); // throw new RandomTestException(e, g, sentence); } } return(false); }