public void ParseRunnerWhenNoLexemesMatchCharacterShouldCreateNewLexeme() { #if false const string input = "aaaa"; ProductionExpression A = "A", S = "S"; A.Rule = (Expr)'a' + 'a'; var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar(); var a = new GrammarLexerRule("a", aGrammar); S.Rule = (a + S) | a; var grammar = new GrammarExpression(S, new[] { S }).ToGrammar(); var parseEngine = new ParseEngine(grammar); var parseRunner = new ParseRunner(parseEngine, input); var chart = GetParseEngineChart(parseEngine); for (var i = 0; i < 3; i++) { Assert.IsTrue(parseRunner.Read()); } Assert.AreEqual(2, chart.Count); #endif }
public void ParseEngineLexemeShouldMatchLongestAcceptableTokenWhenGivenAmbiguity() { var lexemeList = new List <ParseEngineLexeme>(); ProductionExpression There = "there"; There.Rule = (Expr)'t' + 'h' + 'e' + 'r' + 'e'; var thereGrammar = new GrammarExpression(There, new[] { There }) .ToGrammar(); var thereLexerRule = new GrammarLexerRule(new TokenType(There.ProductionModel.LeftHandSide.NonTerminal.Value), thereGrammar); ProductionExpression Therefore = "therefore"; Therefore.Rule = (Expr)'t' + 'h' + 'e' + 'r' + 'e' + 'f' + 'o' + 'r' + 'e'; var thereforeGrammar = new GrammarExpression(Therefore, new[] { Therefore }) .ToGrammar(); var thereforeLexerRule = new GrammarLexerRule(new TokenType(Therefore.ProductionModel.LeftHandSide.NonTerminal.Value), thereforeGrammar); var input = "therefore"; var thereLexeme = new ParseEngineLexeme(thereLexerRule, input.AsCapture(), 0); lexemeList.Add(thereLexeme); var thereforeLexeme = new ParseEngineLexeme(thereforeLexerRule, input.AsCapture(), 0); lexemeList.Add(thereforeLexeme); var i = 0; for (; i < input.Length; i++) { var passedLexemes = lexemeList .Where(l => l.Scan()) .ToList(); // all existing lexemes have failed // fall back onto the lexemes that existed before // we read this character if (passedLexemes.Count() == 0) { break; } lexemeList = passedLexemes; } Assert.AreEqual(i, input.Length); Assert.AreEqual(1, lexemeList.Count); var remainingLexeme = lexemeList[0]; Assert.IsNotNull(remainingLexeme); Assert.IsTrue(remainingLexeme.IsAccepted()); }
public void ParseEngineLexemeShouldConsumeCharacterSequence() { ProductionExpression sequence = "sequence"; sequence.Rule = (Expr)'a' + 'b' + 'c' + '1' + '2' + '3'; var grammar = new GrammarExpression(sequence, new[] { sequence }).ToGrammar(); var lexerRule = new GrammarLexerRule(new TokenType("sequence"), grammar); var lexeme = new ParseEngineLexeme(lexerRule); var input = "abc123"; for (int i = 0; i < input.Length; i++) { Assert.IsTrue(lexeme.Scan(input[i])); } Assert.IsTrue(lexeme.IsAccepted()); }
public void ParseRunnerShouldEmitTokenWhenCharacterMatchesNextProduction() { const string input = "aabb"; ProductionExpression A = "A"; A.Rule = 'a' + A | 'a'; var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar(); var a = new GrammarLexerRule("a", aGrammar); ProductionExpression B = "B"; B.Rule = 'b' + B | 'b'; var bGrammar = new GrammarExpression(B, new[] { B }).ToGrammar(); var b = new GrammarLexerRule("b", bGrammar); ProductionExpression S = "S"; S.Rule = (Expr) a + b; var grammar = new GrammarExpression(S, new[] { S }).ToGrammar(); var parseEngine = new ParseEngine(grammar); var chart = GetParseEngineChart(parseEngine); var parseRunner = new ParseRunner(parseEngine, input); for (int i = 0; i < input.Length; i++) { Assert.IsTrue(parseRunner.Read()); if (i < 2) { Assert.AreEqual(1, chart.Count); } else if (i < 3) { Assert.AreEqual(2, chart.Count); } else { Assert.AreEqual(3, chart.Count); } } }
public void ParseRunnerShouldUseExistingMatchingLexemesToPerformMatch() { #if false const string input = "aaaa"; ProductionExpression A = "A"; A.Rule = (Expr)'a' + 'a'; var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar(); var a = new GrammarLexerRule("a", aGrammar); ProductionExpression S = "S"; S.Rule = (a + S) | a; var grammar = new GrammarExpression(S, new[] { S }).ToGrammar(); var parseEngine = new ParseEngine(grammar); var parseRunner = new ParseRunner(parseEngine, input); var chart = GetParseEngineChart(parseEngine); Assert.IsTrue(parseRunner.Read()); Assert.AreEqual(1, chart.Count); Assert.IsTrue(parseRunner.Read()); Assert.AreEqual(1, chart.Count); #endif }
public void ParseEngineLexemeShouldConsumeWhitespace() { ProductionExpression S = "S", W = "W"; S.Rule = W | W + S; W.Rule = new WhitespaceTerminal(); var grammar = new GrammarExpression(S, new[] { S, W }).ToGrammar(); var lexerRule = new GrammarLexerRule( "whitespace", grammar); var lexeme = new ParseEngineLexeme(lexerRule); var input = "\t\r\n\v\f "; for (int i = 0; i < input.Length; i++) { Assert.IsTrue(lexeme.Scan(input[i]), $"Unable to recognize input[{i}]"); } Assert.IsTrue(lexeme.IsAccepted()); }
public ParseRunnerTests() { _whitespaceRule = CreateWhitespaceRule(); _wordRule = CreateWordRule(); }
static ParseRunnerTests() { _whitespaceRule = CreateWhitespaceRule(); _wordRule = CreateWordRule(); _repeatingWordGrammar = CreateRepeatingWordGrammar(); }