private static void TraversePlay()
{
var g = new Grammar(new List <Production> {
CFGParser.Production("<A> → <B>"),
CFGParser.Production("<B> → <A>"),
CFGParser.Production("<B> → 'x'"),
}, Nonterminal.Of("A"));
g = IdentityActions.Annotate(g);
var earley2 = new EarleyParser2(g);
var sentence = Sentence.FromWords("x");
var sppf2 = earley2.ParseGetForest(sentence);
DotRunner.Run(DotBuilder.GetRawDot(sppf2), "infinite");
var t2 = new Traversal(sppf2, g);
var r2 = t2.Traverse();
//foreach (var option in r2) {
// var s2 = (Sentence)option.Payload;
// if (!sentence.SequenceEqual(s2)) {
// throw new Exception();
// }
//}
}
private static void ParserGenerator()
{
var g = Ebnf.GrammarLayout();
var earley = new EarleyParser2(g);
var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic_ebnf);
// var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Bnf);
// var sentence2 = Sentence.FromLetters("<S> ::= <S> '+' <S>\r\n<S> ::= '1'\r\n");
// if (!sentence1.Equals(sentence2)) { }
// int index = sentence1.Zip(sentence2, (c1, c2) => c1 == c2).TakeWhile(b => b).Count() + 1;
var sppf = earley.ParseGetForest(sentence1);
if (sppf == null)
{
throw new Exception();
}
DotRunner.Run(DotBuilder.GetRawDot(sppf), "arithmetic_ebnf");
//var traversal = new Traversal(sppf, sentence1, g);
//var result = traversal.Traverse();
//if (result.Count() != 1) {
// throw new Exception();
//}
//var generatedGrammar = new Grammar((IEnumerable<Production>)result.First().Payload, Nonterminal.Of("S"));
//Console.WriteLine(generatedGrammar);
}
private static void DevelopAnnotations()
{
var sw = Stopwatch.StartNew();
var input = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic_annotated);
var inputWithoutLayout = Ebnf.RemoveLayout(input, out _);
var grammar = Ebnf.GrammarSyntax();
var parser = new EarleyParser2(grammar);
var sppf = parser.ParseGetForest(inputWithoutLayout);
if (sppf == null)
{
throw new Exception();
}
var ms1 = sw.Elapsed.TotalMilliseconds;
Console.WriteLine("Parse: {0:0.#}ms", ms1);
DotRunner.Run(DotBuilder.GetRawDot(sppf), "annotations");
var traversal = new Traversal(sppf, grammar);
var resCollection = traversal.Traverse();
if (resCollection.Count() > 1)
{
throw new Exception("ambiguous");
}
var res = resCollection.Single();
Console.WriteLine(res);
}
private static double MinTime(int times, EarleyParser2 ep, BaseGrammar grammar, Sentence input)
{
double fastest = double.MaxValue;
var sw = new Stopwatch();
for (var i = 0; i < times; i++)
{
sw.Restart();
// var sppf = ep.ParseGetRawSppf(input);
var sppf = ep.ParseGetForest(input);
//var trav = new Traversal(sppf, input, gp);
//var resultList = trav.Traverse();
//if (resultList.Count() != 1) {
// throw new Exception();
//}
//var result = (long)resultList.First().Payload;
//if (result != expectedResult) {
// throw new Exception();
//}
var elapsed = sw.Elapsed.TotalMilliseconds;
if (elapsed < fastest)
{
fastest = elapsed;
}
}
return(fastest);
}
private static void EbnfPlay()
{
var input = Sentence.FromLetters(Grammars.Properties.Resources.Ebnf_bench);
Sentence inputNoLayout = Ebnf.RemoveLayout(input, out var layoutSppf);
//DotRunner.Run(DotBuilder.GetRawDot(layoutSppf), "arithmetic_ebnf_layout");
Console.WriteLine(inputNoLayout.AsTerminals());
var layoutGrammar = Ebnf.GrammarSyntax();
var earley = new EarleyParser2(layoutGrammar);
var sppf = earley.ParseGetForest(inputNoLayout);
if (sppf == null)
{
throw new Exception();
}
//DotRunner.Run(DotBuilder.GetRawDot(sppf), "arithmetic_ebnf");
//var traversal = new Traversal(sppf, input, layoutGrammar);
//var result = traversal.Traverse();
//if (result.Count() != 1) {
// throw new Exception();
//}
//var inputNoLayout = new Sentence((List<Terminal>)result.First().Payload);
//return inputNoLayout;
//Console.WriteLine(inputNoLayout);
}
private static void ExecuteTest(Grammar g, string input)
{
var earley1 = new EarleyParser(g);
var earley2 = new EarleyParser2(g);
var sentence = Sentence.FromWords(input);
var p1 = earley1.ParseGetProbability(sentence);
var p2 = earley2.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
}
private static void ExecuteTest(Grammar g, string input)
{
var ag = IdentityActions.Annotate(g);
var earley1 = new EarleyParser(ag);
var earley2 = new EarleyParser2(ag);
var sentence = Sentence.FromWords(input);
var sppf1 = earley1.ParseGetForest(sentence);
var sppf2 = earley2.ParseGetForest(sentence);
CheckTraversal(ag, sentence, sppf1);
CheckTraversal(ag, sentence, sppf2);
}
private static void EbnfParse(Sentence sentence)
{
var noLayoutSentence = Ebnf.RemoveLayout(sentence, out var layoutSppf);
var g = Ebnf.GrammarSyntax();
//var earley = new EarleyParser(g);
var earley2 = new EarleyParser2(g);
//var sppf1 = earley.ParseGetForest(noLayoutSentence);
//Assert.IsNotNull(sppf1);
var sppf2 = earley2.ParseGetForest(noLayoutSentence);
Assert.IsNotNull(sppf2);
}
private static void ExecuteTest(Grammar g, List <Sentence> sentences)
{
CNFGrammar h = g.ToCNF();
var earley = new EarleyParser(g);
var cyk = new CykParser(h);
var earley2 = new EarleyParser2(g);
foreach (var sentence in sentences)
{
var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
var p3 = earley2.ParseGetProbability(sentence);
Helpers.AssertNear(p1, p2);
Helpers.AssertNear(p2, p3);
}
}
private static void BnfParse(Sentence sentence)
{
var g = Bnf.Grammar();
//var h = g.ToCNF(); // too much memory
var earley = new EarleyParser(g);
// var cyk = new CykParser(h);
var earley2 = new EarleyParser2(g);
// var p1 = cyk.ParseGetProbability(sentence);
var p2 = earley.ParseGetProbability(sentence);
var p3 = earley2.ParseGetProbability(sentence);
// Helpers.AssertNear(p1, p2);
Helpers.AssertNear(p2, p3);
Assert.IsTrue(p2 > 0.0);
}
// from http://dx.doi.org/10.1016/j.entcs.2008.03.044
private static void PaperExamples()
{
var ex3 = new Grammar(new List <Production> {
CFGParser.Production("<S> → <A> <T>"),
CFGParser.Production("<S> → 'a' <T>"),
CFGParser.Production("<A> → 'a'"),
CFGParser.Production("<A> → <B> <A>"),
CFGParser.Production("<B> → ε"),
CFGParser.Production("<T> → 'b' 'b' 'b'"),
}, Nonterminal.Of("S"));
var input = Sentence.FromLetters("abbb");
var sppf1 = new EarleyParser(ex3).ParseGetForest(input);
var sppf2 = new EarleyParser2(ex3).ParseGetForest(input);
DotRunner.Run(DotBuilder.GetRawDot(sppf1), "example3_old");
DotRunner.Run(DotBuilder.GetRawDot(sppf2), "example3_new");
}
private static void BnfPlay()
{
var bnf = Bnf.Grammar();
var earley = new EarleyParser2(bnf);
var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Arithmetic);
// var sentence1 = Sentence.FromLetters(Grammars.Properties.Resources.Bnf);
// var sentence2 = Sentence.FromLetters("<S> ::= <S> '+' <S>\r\n<S> ::= '1'\r\n");
// if (!sentence1.Equals(sentence2)) { }
// int index = sentence1.Zip(sentence2, (c1, c2) => c1 == c2).TakeWhile(b => b).Count() + 1;
var sppf = earley.ParseGetForest(sentence1);
if (sppf == null)
{
throw new Exception();
}
DotRunner.Run(DotBuilder.GetRawDot(sppf), "bnfPlay");
}
private static void EbnfBench()
{
var tup = EbnfBenchLayout();
var inputNoLayout = tup.Item1;
var ms1 = tup.Item2;
var sw = Stopwatch.StartNew();
var grammar = Ebnf.GrammarSyntax();
var earley = new EarleyParser2(grammar);
var sppf = earley.ParseGetForest(inputNoLayout);
var ms2 = sw.Elapsed.TotalMilliseconds;
if (sppf == null)
{
throw new Exception();
}
Console.WriteLine("Parse: {0:0.#}ms", ms2);
Console.WriteLine("Total: {0:0.#}ms", ms1 + ms2);
}
private static void Benchmark()
{
Console.WriteLine("Benching...");
var inputs = new List <Tuple <Sentence, long, int> >();
// for (var i = 80; i < 105; i++) { // 13336ms
// for (var i = 1; i < 500; i += 1) {
// for (var i = 170; i < 195; i++) { // 10755ms after gather in sppf
// for (var i = 170; i < 195; i++) { // 9203ms after hash change
// for (var i = 170; i < 195; i++) { // 2703ms after doing gather earlier
// for (var i = 170; i < 195; i++) {
// for (var i = 751; i < 752; i++) {
// for (var i = 95; i < 130; i++) { // new; 15385
for (var i = 160; i < 200; i++)
{
// for (var i = 300; i < 350; i++) { // new;
inputs.Add(Tuple.Create(Sentence.FromWords(AdditionInput(i)), (long)i, i));
}
var gp = AdditionGrammar(argList => (long)argList[0].Payload + (long)argList[2].Payload);
var ep = new EarleyParser2(gp);
// var totalSw = Stopwatch.StartNew();
double totalMs = 0;
foreach (var inputPair in inputs)
{
var input = inputPair.Item1;
var expectedResult = inputPair.Item2;
var i = inputPair.Item3;
var time = MinTime(3, ep, gp, input);
totalMs += time;
Console.WriteLine("{0}, {1}", i, time);
}
Console.WriteLine("Done in {0}ms (prev 3686ms)", (int)totalMs);
//foreach (var kvp in EarleyParser2._stats.Data) {
// Console.WriteLine("{0}, {1}", kvp.Key, kvp.Value);
//}
}
private static void VisitorPlay()
{
var gp = AdditionGrammar(argList => string.Format("({0} + {1})", argList[0].Payload, argList[2].Payload));
//var actions2 = new Dictionary<Production, IParserAction> {
// [p1] = new ParserAction<long>((argList) => (long)(argList[0].Payload) + (long)(argList[2].Payload)),
//};
//var termAction2 = new ParserAction<long>(x => Convert.ToInt64(x[0].Payload));
//foreach (var num in nums) {
// actions2[num] = termAction2;
//}
var ep = new EarleyParser2(gp);
var inputString = AdditionInput(5);
var input = Sentence.FromWords(inputString);
var sppf = ep.ParseGetForest(input);
var rawdot = DotBuilder.GetRawDot(sppf);
DotRunner.Run(rawdot, "newSppf");
Console.WriteLine(sppf.ToString());
Console.WriteLine();
Console.WriteLine("Starting Traversal...");
var trav = new Traversal(sppf, gp);
var resultList = trav.Traverse();
Console.WriteLine("-----------------");
foreach (var result in resultList)
{
Console.WriteLine(result.Payload);
}
//Console.WriteLine("-----------------");
//foreach (var result in new Traversal(sppf, input, gp).Traverse()) {
// Console.WriteLine(result.Payload);
//}
}
public static Sentence RemoveLayout(Sentence input, out SppfNode sppf)
{
var layoutGrammar = Ebnf.GrammarLayout();
var earley = new EarleyParser2(layoutGrammar);
sppf = earley.ParseGetForest(input);
if (sppf == null)
{
throw new Exception();
}
var traversal = new Traversal(sppf, layoutGrammar);
var result = traversal.Traverse();
if (result.Count() != 1)
{
throw new Exception();
}
var inputNoLayout = new Sentence((List <Terminal>)result.First().Payload);
return(inputNoLayout);
}
private static void DebugGrammar()
{
BaseGrammar g = new Grammar(new List <Production> {
CFGParser.Production("<S> → ε"),
}, Nonterminal.Of("S"));
var sentence = Sentence.FromWords("1 + 1 + 1");
var grammar = AdditionGrammar(argList => string.Format("({0} + {1})", argList[0].Payload, argList[2].Payload));
g = grammar;
var earley = new EarleyParser(g);
var earley2 = new EarleyParser2(g);
//DotRunner.Run(earley.ParseGetForest(sentence).GetRawDot(), "testEarleyOld");
//DotRunner.Run(earley2.ParseGetForest(sentence).GetRawDot(), "testEarleyNew");
DotRunner.Run(DotBuilder.GetRawDot(earley.ParseGetForest(sentence)), "testEarleyOld");
DotRunner.Run(DotBuilder.GetRawDot(earley2.ParseGetForest(sentence)), "testEarleyNew");
// DotRunner.Run(DotBuilder.GetFlattenedDot(earley2.ParseGetForest(sentence)), "testEarleyFlat");
// var prob0 = earley.ParseGetProbability(sentence);
var prob = earley2.ParseGetProbability(sentence);
}
private bool ProcessOneGrammar()
{
var(g, terminals) = NextGrammar();
var h = g.ToCNF();
// Console.WriteLine(g.Productions.Count());
var preparedSentences = new List <Sentence>();
for (int length = 0; length <= _maxInputLength; length++)
{
var combinations = CFGLibTest.Helpers.CombinationsWithRepetition(terminals, length);
foreach (var target in combinations)
{
var sentence = new Sentence(target);
preparedSentences.Add(sentence);
}
}
AddRandomSentences(preparedSentences, terminals);
var uniquifySentences = new Dictionary <string, Sentence>();
foreach (var sentence in preparedSentences)
{
uniquifySentences[sentence.AsTerminals()] = sentence;
}
preparedSentences = uniquifySentences.Values.ToList();
// Console.WriteLine("Parsing sentences...");
EarleyParser earley1;
EarleyParser2 earley2;
CykParser cyk;
try {
earley1 = new EarleyParser(g);
earley2 = new EarleyParser2(g);
cyk = new CykParser(h);
} catch (Exception e) {
Report(g, e);
return(true);
}
foreach (var sentence in preparedSentences)
{
try {
var sppf1 = earley1.ParseGetForest(sentence);
var sppf2 = earley2.ParseGetForest(sentence);
if (sppf1 == null && sppf2 != null)
{
throw new Exception();
}
if (sppf2 == null && sppf1 != null)
{
throw new Exception();
}
var p1 = earley1.ProbabilityOfSppf(sppf1);
var p2 = earley2.ProbabilityOfSppf(sppf2);
var p3 = cyk.ParseGetProbability(sentence);
if (!Helpers.IsNear(p1, p2))
{
throw new Exception();
}
if (!Helpers.IsNear(p1, p3))
{
throw new Exception();
}
try {
TestTraversal.CheckTraversal(g, sentence, sppf1);
} catch (TraversalLoopException) { }
try {
TestTraversal.CheckTraversal(g, sentence, sppf2);
} catch (TraversalLoopException) { }
} catch (Exception e) {
Report(g, e, sentence);
return(true);
// throw new RandomTestException(e, g, sentence);
}
}
return(false);
}
