public void ParseRunnerWhenNoLexemesMatchCharacterShouldCreateNewLexeme()
{
#if false
const string input = "aaaa";
ProductionExpression A = "A", S = "S";
A.Rule = (Expr)'a' + 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
S.Rule = (a + S) | a;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
for (var i = 0; i < 3; i++)
{
Assert.IsTrue(parseRunner.Read());
}
Assert.AreEqual(2, chart.Count);
#endif
}
public void ParseEngineLexemeShouldMatchLongestAcceptableTokenWhenGivenAmbiguity()
{
var lexemeList = new List <ParseEngineLexeme>();
ProductionExpression There = "there";
There.Rule = (Expr)'t' + 'h' + 'e' + 'r' + 'e';
var thereGrammar = new GrammarExpression(There, new[] { There })
.ToGrammar();
var thereLexerRule = new GrammarLexerRule(new TokenType(There.ProductionModel.LeftHandSide.NonTerminal.Value), thereGrammar);
ProductionExpression Therefore = "therefore";
Therefore.Rule = (Expr)'t' + 'h' + 'e' + 'r' + 'e' + 'f' + 'o' + 'r' + 'e';
var thereforeGrammar = new GrammarExpression(Therefore, new[] { Therefore })
.ToGrammar();
var thereforeLexerRule = new GrammarLexerRule(new TokenType(Therefore.ProductionModel.LeftHandSide.NonTerminal.Value), thereforeGrammar);
var input = "therefore";
var thereLexeme = new ParseEngineLexeme(thereLexerRule, input.AsCapture(), 0);
lexemeList.Add(thereLexeme);
var thereforeLexeme = new ParseEngineLexeme(thereforeLexerRule, input.AsCapture(), 0);
lexemeList.Add(thereforeLexeme);
var i = 0;
for (; i < input.Length; i++)
{
var passedLexemes = lexemeList
.Where(l => l.Scan())
.ToList();
// all existing lexemes have failed
// fall back onto the lexemes that existed before
// we read this character
if (passedLexemes.Count() == 0)
{
break;
}
lexemeList = passedLexemes;
}
Assert.AreEqual(i, input.Length);
Assert.AreEqual(1, lexemeList.Count);
var remainingLexeme = lexemeList[0];
Assert.IsNotNull(remainingLexeme);
Assert.IsTrue(remainingLexeme.IsAccepted());
}
public void ParseEngineLexemeShouldConsumeCharacterSequence()
{
ProductionExpression sequence = "sequence";
sequence.Rule = (Expr)'a' + 'b' + 'c' + '1' + '2' + '3';
var grammar = new GrammarExpression(sequence, new[] { sequence }).ToGrammar();
var lexerRule = new GrammarLexerRule(new TokenType("sequence"), grammar);
var lexeme = new ParseEngineLexeme(lexerRule);
var input = "abc123";
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(lexeme.Scan(input[i]));
}
Assert.IsTrue(lexeme.IsAccepted());
}
public void ParseRunnerShouldEmitTokenWhenCharacterMatchesNextProduction()
{
const string input = "aabb";
ProductionExpression A = "A";
A.Rule =
'a' + A
| 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
ProductionExpression B = "B";
B.Rule =
'b' + B
| 'b';
var bGrammar = new GrammarExpression(B, new[] { B }).ToGrammar();
var b = new GrammarLexerRule("b", bGrammar);
ProductionExpression S = "S";
S.Rule = (Expr)
a + b;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var chart = GetParseEngineChart(parseEngine);
var parseRunner = new ParseRunner(parseEngine, input);
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(parseRunner.Read());
if (i < 2)
{
Assert.AreEqual(1, chart.Count);
}
else if (i < 3)
{
Assert.AreEqual(2, chart.Count);
}
else
{
Assert.AreEqual(3, chart.Count);
}
}
}
public void ParseRunnerShouldUseExistingMatchingLexemesToPerformMatch()
{
#if false
const string input = "aaaa";
ProductionExpression A = "A";
A.Rule = (Expr)'a' + 'a';
var aGrammar = new GrammarExpression(A, new[] { A }).ToGrammar();
var a = new GrammarLexerRule("a", aGrammar);
ProductionExpression S = "S";
S.Rule = (a + S) | a;
var grammar = new GrammarExpression(S, new[] { S }).ToGrammar();
var parseEngine = new ParseEngine(grammar);
var parseRunner = new ParseRunner(parseEngine, input);
var chart = GetParseEngineChart(parseEngine);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(1, chart.Count);
Assert.IsTrue(parseRunner.Read());
Assert.AreEqual(1, chart.Count);
#endif
}
public void ParseEngineLexemeShouldConsumeWhitespace()
{
ProductionExpression
S = "S",
W = "W";
S.Rule = W | W + S;
W.Rule = new WhitespaceTerminal();
var grammar = new GrammarExpression(S, new[] { S, W }).ToGrammar();
var lexerRule = new GrammarLexerRule(
"whitespace",
grammar);
var lexeme = new ParseEngineLexeme(lexerRule);
var input = "\t\r\n\v\f ";
for (int i = 0; i < input.Length; i++)
{
Assert.IsTrue(lexeme.Scan(input[i]), $"Unable to recognize input[{i}]");
}
Assert.IsTrue(lexeme.IsAccepted());
}
public ParseRunnerTests()
{
_whitespaceRule = CreateWhitespaceRule();
_wordRule = CreateWordRule();
}
static ParseRunnerTests()
{
_whitespaceRule = CreateWhitespaceRule();
_wordRule = CreateWordRule();
_repeatingWordGrammar = CreateRepeatingWordGrammar();
}
