private static void AssertLexerRuleMatches(DfaLexerRule lexerRule, string input)
{
var lexeme = lexerRule.CreateLexeme(0);
for (var i = 0; i < input.Length; i++)
{
Assert.IsTrue(lexeme.Scan(input[i]), $"character '{input[i]}' not recognized at position {i}.");
}
Assert.IsTrue(lexeme.IsAccepted(), $"input {input} not accepted.");
}
public void RegexToDfaShouldConvertCharacterRegexToDfa()
{
var pattern = "a";
var dfa = CreateDfaFromRegexPattern(pattern);
Assert.IsNotNull(dfa);
var lexerRule = new DfaLexerRule(dfa, "a");
var lexeme = new DfaLexemeFactory().Create(lexerRule, 0);
Assert.IsTrue(lexeme.Scan('a'));
}
private static BaseLexerRule Whitespace()
{
var whitespaceTerminal = new WhitespaceTerminal();
var startWhitespace = new DfaState();
var finalWhitespace = new DfaState(true);
startWhitespace.AddTransition(new DfaTransition(whitespaceTerminal, finalWhitespace));
finalWhitespace.AddTransition(new DfaTransition(whitespaceTerminal, finalWhitespace));
var whitespace = new DfaLexerRule(startWhitespace, TokenTypes.Whitespace);
return(whitespace);
}
private static LexerRule CreateWhitespaceLexerRule()
{
var whitespaceTerminal = WhitespaceTerminal.Instance;
var startWhitespace = DfaState.Inner();
var finalWhitespace = DfaState.Final();
startWhitespace.AddTransition(whitespaceTerminal, finalWhitespace);
finalWhitespace.AddTransition(whitespaceTerminal, finalWhitespace);
var whitespace = new DfaLexerRule(startWhitespace, TokenClasses.Whitespace);
return(whitespace);
}
public void RegexToDfaShouldConvertOptionalCharacterClassToDfa()
{
var pattern = @"[-+]?[0-9]";
var dfa = CreateDfaFromRegexPattern(pattern);
Assert.IsNotNull(dfa);
Assert.AreEqual(3, dfa.Transitions.Count);
var lexerRule = new DfaLexerRule(dfa, pattern);
AssertLexerRuleMatches(lexerRule, "+0");
AssertLexerRuleMatches(lexerRule, "-1");
AssertLexerRuleMatches(lexerRule, "9");
}
public void RegexToDfaShouldConvertSlasSToWhitespaceCharacterClass()
{
var pattern = @"\s";
var dfa = CreateDfaFromRegexPattern(pattern);
Assert.IsNotNull(dfa);
Assert.AreEqual(1, dfa.Transitions.Count);
Assert.AreEqual(0, dfa.Transitions[0].Target.Transitions.Count);
var lexerRule = new DfaLexerRule(dfa, pattern);
AssertLexerRuleMatches(lexerRule, " ");
AssertLexerRuleMatches(lexerRule, "\t");
AssertLexerRuleMatches(lexerRule, "\f");
}
public void RegexToDfaShouldConvertCharacterRegexToDfa()
{
var pattern = "a";
var regex = new RegexParser().Parse(pattern);
var nfa = new ThompsonConstructionAlgorithm().Transform(regex);
var dfa = new SubsetConstructionAlgorithm().Transform(nfa);
Assert.IsNotNull(dfa);
var lexerRule = new DfaLexerRule(dfa, "a");
var lexeme = new DfaLexemeFactory().Create(lexerRule, 0);
Assert.IsTrue(lexeme.Scan('a'));
}
IEnumerable <ProductionModel> Factor(EbnfFactor factor, ProductionModel currentProduction)
{
switch (factor.NodeType)
{
case EbnfNodeType.EbnfFactorGrouping:
var grouping = factor as EbnfFactorGrouping;
foreach (var production in Grouping(grouping, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorOptional:
var optional = factor as EbnfFactorOptional;
foreach (var production in Optional(optional, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorRepetition:
var repetition = factor as EbnfFactorRepetition;
foreach (var production in Repetition(repetition, currentProduction))
{
yield return(production);
}
break;
case EbnfNodeType.EbnfFactorIdentifier:
var identifier = factor as EbnfFactorIdentifier;
var nonTerminal = GetNonTerminalFromQualifiedIdentifier(identifier.QualifiedIdentifier);
currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal));
break;
case EbnfNodeType.EbnfFactorLiteral:
var literal = factor as EbnfFactorLiteral;
var stringLiteralRule = new StringLiteralLexerRule(literal.Value);
currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule));
break;
case EbnfNodeType.EbnfFactorRegex:
var regex = factor as EbnfFactorRegex;
var nfa = _thompsonConstructionAlgorithm.Transform(regex.Regex);
var dfa = _subsetConstructionAlgorithm.Transform(nfa);
var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString());
currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule));
break;
}
}
public void DfaLexemeGivenCharacerLexemeNumberShouldFail()
{
var numberInput = "0";
var dfa = new DfaState();
var final = new DfaState(true);
dfa.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final));
final.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final));
var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("lowerCase"));
var letterLexeme = new DfaLexeme(dfaLexerRule, 0);
Assert.IsFalse(letterLexeme.Scan(numberInput[0]));
Assert.AreEqual(string.Empty, letterLexeme.Value);
}
public void RegexToDfaShouldConvertOptionalCharacterClassToDfa()
{
var pattern = @"[-+]?[0-9]";
var regex = new RegexParser().Parse(pattern);
var nfa = new ThompsonConstructionAlgorithm().Transform(regex);
var dfa = new SubsetConstructionAlgorithm().Transform(nfa);
Assert.IsNotNull(dfa);
Assert.AreEqual(3, dfa.Transitions.Count);
var lexerRule = new DfaLexerRule(dfa, pattern);
AssertLexerRuleMatches(lexerRule, "+0");
AssertLexerRuleMatches(lexerRule, "-1");
AssertLexerRuleMatches(lexerRule, "9");
}
private static LexerRule CreateRuleNameLexerRule()
{
// /[a-zA-Z][a-zA-Z0-9-_]*/
var ruleNameState = DfaState.Inner();
var zeroOrMoreLetterOrDigit = DfaState.Final();
ruleNameState.AddTransition(AsciiLetterTerminal.Instance, zeroOrMoreLetterOrDigit);
zeroOrMoreLetterOrDigit.AddTransition(
new CharacterClassTerminal(
AsciiLetterTerminal.Instance,
DigitTerminal.Instance,
new SetTerminal("-_")),
zeroOrMoreLetterOrDigit);
var ruleName = new DfaLexerRule(ruleNameState, "rule-name");
return(ruleName);
}
public void DfaLexemeShouldMatchOneOrMoreRandomWhitespaceCharacters()
{
var randomWhitespace = "\t\f\v \r\n";
var dfa = new DfaState();
var final = new DfaState(true);
dfa.AddTransition(new DfaTransition(new WhitespaceTerminal(), final));
final.AddTransition(new DfaTransition(new WhitespaceTerminal(), final));
var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("whitespace"));
var whitespaceLexeme = new DfaLexeme(dfaLexerRule, 0);
for (int i = 0; i < randomWhitespace.Length; i++)
{
Assert.IsTrue(whitespaceLexeme.Scan(randomWhitespace[i]));
}
}
public void DfaLexemeShouldMatchOneOrMoreRandomWhitespaceCharacters()
{
var randomWhitespace = "\t\f\v \r\n";
var dfa = DfaState.Inner();
var final = DfaState.Final();
dfa.AddTransition(WhitespaceTerminal.Instance, final);
final.AddTransition(WhitespaceTerminal.Instance, final);
var dfaLexerRule = new DfaLexerRule(dfa, new TokenName("whitespace"));
var whitespaceLexeme = dfaLexerRule.CreateLexeme(0);
foreach (var character in randomWhitespace)
{
Assert.IsTrue(whitespaceLexeme.Scan(character));
}
}
private IEnumerable <ProductionModel> Factor(IEbnfFactor factor, ProductionModel currentProduction)
{
switch (factor)
{
case EbnfFactorGrouping grouping:
foreach (var production in Grouping(grouping, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorOptional optional:
foreach (var production in Optional(optional, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorRepetition repetition:
foreach (var production in Repetition(repetition, currentProduction))
{
yield return(production);
}
break;
case EbnfFactorIdentifier identifier:
var nonTerminal = NonTerminal.From(GetQualifiedName(identifier.QualifiedIdentifier));
currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal));
break;
case EbnfFactorLiteral literal:
var stringLiteralRule = new StringLiteralLexerRule(literal.Value);
currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule));
break;
case EbnfFactorRegex regex:
var nfa = this.regexToNfa.Transform(regex.Regex);
var dfa = this.nfaToDfa.Transform(nfa);
var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString());
currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule));
break;
}
}
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition()
{
var states = new DfaState[2]
{
DfaState.Inner(),
DfaState.Final()
};
states[0].AddTransition(WhitespaceTerminal.Instance, states[1]);
states[1].AddTransition(WhitespaceTerminal.Instance, states[1]);
var dfaLexerRule = new DfaLexerRule(states[0], new TokenName(@"\s+"));
Assert.IsTrue(dfaLexerRule.CanApply(' '));
Assert.IsTrue(dfaLexerRule.CanApply('\t'));
Assert.IsTrue(dfaLexerRule.CanApply('\r'));
Assert.IsFalse(dfaLexerRule.CanApply('a'));
}
public void DfaLexemeShouldMatchMixedCaseWord()
{
var wordInput = "t90vAriabl3";
var dfa = new DfaState();
var final = new DfaState(true);
dfa.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final));
dfa.AddTransition(new DfaTransition(new RangeTerminal('A', 'Z'), final));
final.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final));
final.AddTransition(new DfaTransition(new RangeTerminal('A', 'Z'), final));
final.AddTransition(new DfaTransition(new DigitTerminal(), final));
var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("Identifier"));
var indentifierLexeme = new DfaLexeme(dfaLexerRule, 0);
for (int i = 0; i < wordInput.Length; i++)
{
Assert.IsTrue(indentifierLexeme.Scan(wordInput[i]));
}
}
public void DfaLexemeShouldMatchMixedCaseWord()
{
var wordInput = "t90vAriabl3";
var dfa = DfaState.Inner();
var final = DfaState.Final();
dfa.AddTransition(new RangeTerminal('a', 'z'), final);
dfa.AddTransition(new RangeTerminal('A', 'Z'), final);
final.AddTransition(new RangeTerminal('a', 'z'), final);
final.AddTransition(new RangeTerminal('A', 'Z'), final);
final.AddTransition(DigitTerminal.Instance, final);
var dfaLexerRule = new DfaLexerRule(dfa, new TokenName("Identifier"));
var indentifierLexeme = dfaLexerRule.CreateLexeme(0);
foreach (var character in wordInput)
{
Assert.IsTrue(indentifierLexeme.Scan(character));
}
}
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition()
{
var states = new DfaState[2];
for (var i = 0; i < states.Length; i++)
{
states[i] = new DfaState(i == states.Length - 1);
}
var whitespaceToFinal = new DfaTransition(new WhitespaceTerminal(), states[1]);
states[0].AddTransition(whitespaceToFinal);
states[1].AddTransition(whitespaceToFinal);
var dfaLexerRule = new DfaLexerRule(states[0], new TokenType(@"\s+"));
Assert.IsTrue(dfaLexerRule.CanApply(' '));
Assert.IsTrue(dfaLexerRule.CanApply('\t'));
Assert.IsTrue(dfaLexerRule.CanApply('\r'));
Assert.IsFalse(dfaLexerRule.CanApply('a'));
}
private static ILexerRule CreateRuleNameLexerRule()
{
var ruleNameState = new DfaState();
var zeroOrMoreLetterOrDigit = new DfaState(true);
ruleNameState.AddTransition(
new DfaTransition(
new CharacterClassTerminal(
new RangeTerminal('a', 'z'),
new RangeTerminal('A', 'Z')),
zeroOrMoreLetterOrDigit));
zeroOrMoreLetterOrDigit.AddTransition(
new DfaTransition(
new CharacterClassTerminal(
new RangeTerminal('a', 'z'),
new RangeTerminal('A', 'Z'),
new DigitTerminal(),
new SetTerminal('-', '_')),
zeroOrMoreLetterOrDigit));
var ruleName = new DfaLexerRule(ruleNameState, new TokenType("rule-name"));
return(ruleName);
}
private static BaseLexerRule Identifier()
{
// /[a-zA-Z][a-zA-Z0-9-_]*/
var identifierState = new DfaState();
var zeroOrMoreLetterOrDigit = new DfaState(true);
identifierState.AddTransition(
new DfaTransition(
new CharacterClassTerminal(
new RangeTerminal('a', 'z'),
new RangeTerminal('A', 'Z')),
zeroOrMoreLetterOrDigit));
zeroOrMoreLetterOrDigit.AddTransition(
new DfaTransition(
new CharacterClassTerminal(
new RangeTerminal('a', 'z'),
new RangeTerminal('A', 'Z'),
new DigitTerminal(),
new SetTerminal('-', '_')),
zeroOrMoreLetterOrDigit));
var identifier = new DfaLexerRule(identifierState, TokenTypes.Identifier);
return(identifier);
}