private bool TryRecognizeSimpleLiteralExpression(
QualifiedName fullyQualifiedName,
IEbnfLexerRuleExpression ebnfLexerRule,
out LexerRule lexerRule)
{
lexerRule = null;
if (ebnfLexerRule is EbnfLexerRuleExpressionSimple)
{
var term = ebnfLexerRule.Term;
if (term is EbnfLexerRuleTermConcatenation)
{
return(false);
}
var factor = term.Factor;
if (factor is EbnfLexerRuleFactorLiteral literal)
{
lexerRule = new StringLiteralLexerRule(literal.Value, new TokenName(fullyQualifiedName.FullName));
return(true);
}
}
return(false);
}
public void GrammarModelToGrammarShouldCreateGrammar()
{
var grammarModel = new GrammarModel();
var S = new ProductionModel("S");
var A = new ProductionModel("A");
var B = new ProductionModel("B");
var a = new StringLiteralLexerRule("a");
var b = new StringLiteralLexerRule("b");
var space = new StringLiteralLexerRule(" ");
S.AddWithAnd(A.LeftHandSide);
S.AddWithAnd(B.LeftHandSide);
S.AddWithOr(B.LeftHandSide);
A.AddWithAnd(new LexerRuleModel(a));
B.AddWithAnd(new LexerRuleModel(b));
grammarModel.Productions.Add(S);
grammarModel.Productions.Add(A);
grammarModel.Productions.Add(B);
grammarModel.IgnoreRules.Add(new LexerRuleModel(space));
grammarModel.Start = S;
var grammar = grammarModel.ToGrammar();
Assert.AreEqual(4, grammar.Productions.Count);
Assert.AreEqual(1, grammar.Ignores.Count);
}
private bool TryRecognizeSimpleLiteralExpression(
FullyQualifiedName fullyQualifiedName,
EbnfLexerRuleExpression ebnfLexerRule,
out ILexerRule lexerRule)
{
lexerRule = null;
if (ebnfLexerRule.NodeType != EbnfNodeType.EbnfLexerRuleExpression)
{
return(false);
}
var term = ebnfLexerRule.Term;
if (term.NodeType != EbnfNodeType.EbnfLexerRuleTerm)
{
return(false);
}
var factor = term.Factor;
if (factor.NodeType != EbnfNodeType.EbnfLexerRuleFactorLiteral)
{
return(false);
}
var literal = factor as EbnfLexerRuleFactorLiteral;
lexerRule = new StringLiteralLexerRule(
literal.Value,
new TokenType(fullyQualifiedName.FullName));
return(true);
}
public void GrammarModelToGrammarShouldCreateGrammar()
{
var grammarModel = new GrammarModel();
var S = ProductionModel.From("S");
var A = ProductionModel.From("A");
var B = ProductionModel.From("B");
var a = new StringLiteralLexerRule("a");
var b = new StringLiteralLexerRule("b");
var space = new StringLiteralLexerRule(" ");
S.AddWithAnd(A.LeftHandSide);
S.AddWithAnd(B.LeftHandSide);
S.AddWithOr(B.LeftHandSide);
A.AddWithAnd(new LexerRuleModel(a));
B.AddWithAnd(new LexerRuleModel(b));
grammarModel.AddProduction(S);
grammarModel.AddProduction(A);
grammarModel.AddProduction(B);
var lexerRuleModel = new LexerRuleModel(space);
grammarModel.AddLexerRule(lexerRuleModel);
grammarModel.AddIgnoreSetting(new IgnoreSettingModel(new QualifiedName(lexerRuleModel.LexerRule.TokenName.Id)));
grammarModel.Start = S;
var grammar = grammarModel.ToGrammar();
Assert.AreEqual(4, grammar.Productions.Count);
Assert.AreEqual(1, grammar.Ignores.Count);
}
public void ProductionExpressionShouldSupportConcatenationOfLexerRuleAndString()
{
ProductionExpression
S = "S";
LexerRule
a = new StringLiteralLexerRule("a");
S.Rule = a + (Expr)"s";
Assert.IsNotNull(S.ProductionModel);
Assert.AreEqual(1, S.ProductionModel.Alterations.Count);
Assert.AreEqual(2, S.ProductionModel.Alterations[0].Symbols.Count);
}
public void ProductionExpressionShouldSupportAlterationOfStringAndLexerRule()
{
ProductionExpression
S = "S";
LexerRule
a = new StringLiteralLexerRule("a");
S.Rule = (Expr)"b" | a;
Assert.IsNotNull(S.ProductionModel);
Assert.AreEqual(2, S.ProductionModel.Alterations.Count);
Assert.AreEqual(1, S.ProductionModel.Alterations[0].Symbols.Count);
Assert.AreEqual(1, S.ProductionModel.Alterations[1].Symbols.Count);
}
public void ProductionExpressionShouldSupportConcatenationOfTwoBaseLexerRules()
{
ProductionExpression
S = "S";
LexerRule
a = new StringLiteralLexerRule("a"),
b = new StringLiteralLexerRule("b");
S.Rule = (Expr)a + b;
Assert.IsNotNull(S.ProductionModel);
Assert.AreEqual(1, S.ProductionModel.Alterations.Count);
Assert.AreEqual(2, S.ProductionModel.Alterations[0].Symbols.Count);
}
IEnumerable <ProductionModel> Factor(EbnfFactor factor, ProductionModel currentProduction)
{
switch (factor.NodeType)
{
case EbnfNodeType.EbnfFactorGrouping:
var grouping = factor as EbnfFactorGrouping;
foreach (var production in Grouping(grouping, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorOptional:
var optional = factor as EbnfFactorOptional;
foreach (var production in Optional(optional, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorRepetition:
var repetition = factor as EbnfFactorRepetition;
foreach (var production in Repetition(repetition, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorIdentifier:
var identifier = factor as EbnfFactorIdentifier;
var nonTerminal = GetNonTerminalFromQualifiedIdentifier(identifier.QualifiedIdentifier);
currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal));
break;
case EbnfNodeType.EbnfFactorLiteral:
var literal = factor as EbnfFactorLiteral;
var stringLiteralRule = new StringLiteralLexerRule(literal.Value);
currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule));
break;
case EbnfNodeType.EbnfFactorRegex:
var regex = factor as EbnfFactorRegex;
var nfa = _thompsonConstructionAlgorithm.Transform(regex.Regex);
var dfa = _subsetConstructionAlgorithm.Transform(nfa);
var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString());
currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule));
break;
}
}
public void StringLiteralLexemeResetShouldResetLexemeValues()
{
var abc123LexerRule = new StringLiteralLexerRule("abc123");
//var lexeme = new StringLiteralLexeme(abc123LexerRule, 0);
var lexeme = abc123LexerRule.CreateLexeme(0);
const string input = "abc123";
foreach (var character in input)
{
var result = lexeme.Scan(character);
if (!result)
{
Assert.Fail($"Did not recognize character {character}");
}
}
}
private IEnumerable <ProductionModel> Factor(IEbnfFactor factor, ProductionModel currentProduction)
{
switch (factor)
{
case EbnfFactorGrouping grouping:
foreach (var production in Grouping(grouping, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorOptional optional:
foreach (var production in Optional(optional, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorRepetition repetition:
foreach (var production in Repetition(repetition, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorIdentifier identifier:
var nonTerminal = NonTerminal.From(GetQualifiedName(identifier.QualifiedIdentifier));
currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal));
break;
case EbnfFactorLiteral literal:
var stringLiteralRule = new StringLiteralLexerRule(literal.Value);
currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule));
break;
case EbnfFactorRegex regex:
var nfa = this.regexToNfa.Transform(regex.Regex);
var dfa = this.nfaToDfa.Transform(nfa);
var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString());
currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule));
break;
}
}
public void ParseRunnerGivenIgnoreCharactersWhenOverlapWithTerminalShouldChooseTerminal()
{
var input = "word \t\r\n word";
var endOfLine = new StringLiteralLexerRule(
Environment.NewLine,
new TokenType("EOL"));
ProductionExpression S = "S";
S.Rule = (Expr)_wordRule + endOfLine + _wordRule;
var grammar = new GrammarExpression(
S,
new[] { S },
new[] { _whitespaceRule })
.ToGrammar();
var parseEngine = new ParseEngine(grammar);
RunParse(parseEngine, input);
}
public void StringLiteralLexemeResetShouldResetLexemeValues()
{
var abc123LexerRule = new StringLiteralLexerRule("abc123");
var zyx654LexerRule = new StringLiteralLexerRule("zyx654");
var lexeme = new StringLiteralLexeme(abc123LexerRule, 0);
const string input = "abc123";
for (var i = 0; i < input.Length; i++)
{
var result = lexeme.Scan(input[i]);
if (!result)
{
Assert.Fail($"Did not recognize character {input[i]}");
}
}
lexeme.Reset(zyx654LexerRule, 50);
Assert.AreEqual(string.Empty, lexeme.Value);
Assert.AreEqual(50, lexeme.Position);
Assert.AreEqual(zyx654LexerRule.LexerRuleType, lexeme.LexerRule.LexerRuleType);
Assert.AreEqual(zyx654LexerRule.TokenType, lexeme.TokenType);
}
public void StringLiteralLexemeResetShouldResetLexemeValues()
{
var abc123LexerRule = new StringLiteralLexerRule("abc123");
var zyx654LexerRule = new StringLiteralLexerRule("zyx654");
const string input = "abc123";
var segment = input.AsCapture();
var lexeme = new StringLiteralLexeme(abc123LexerRule, segment, 0);
for (var i = 0; i < input.Length; i++)
{
var result = lexeme.Scan();
if (!result)
{
Assert.Fail($"Did not recognize character '{input[i]}' at position {i}");
}
}
lexeme.Reset(zyx654LexerRule, 0);
Assert.AreEqual(string.Empty, lexeme.Capture.ToString());
Assert.AreEqual(0, lexeme.Position);
Assert.AreEqual(zyx654LexerRule.LexerRuleType, lexeme.LexerRule.LexerRuleType);
Assert.AreEqual(zyx654LexerRule.TokenType, lexeme.TokenType);
}