private static void AssertLexerRuleMatches(DfaLexerRule lexerRule, string input) { var lexeme = lexerRule.CreateLexeme(0); for (var i = 0; i < input.Length; i++) { Assert.IsTrue(lexeme.Scan(input[i]), $"character '{input[i]}' not recognized at position {i}."); } Assert.IsTrue(lexeme.IsAccepted(), $"input {input} not accepted."); }
public void RegexToDfaShouldConvertCharacterRegexToDfa() { var pattern = "a"; var dfa = CreateDfaFromRegexPattern(pattern); Assert.IsNotNull(dfa); var lexerRule = new DfaLexerRule(dfa, "a"); var lexeme = new DfaLexemeFactory().Create(lexerRule, 0); Assert.IsTrue(lexeme.Scan('a')); }
private static BaseLexerRule Whitespace() { var whitespaceTerminal = new WhitespaceTerminal(); var startWhitespace = new DfaState(); var finalWhitespace = new DfaState(true); startWhitespace.AddTransition(new DfaTransition(whitespaceTerminal, finalWhitespace)); finalWhitespace.AddTransition(new DfaTransition(whitespaceTerminal, finalWhitespace)); var whitespace = new DfaLexerRule(startWhitespace, TokenTypes.Whitespace); return(whitespace); }
private static LexerRule CreateWhitespaceLexerRule() { var whitespaceTerminal = WhitespaceTerminal.Instance; var startWhitespace = DfaState.Inner(); var finalWhitespace = DfaState.Final(); startWhitespace.AddTransition(whitespaceTerminal, finalWhitespace); finalWhitespace.AddTransition(whitespaceTerminal, finalWhitespace); var whitespace = new DfaLexerRule(startWhitespace, TokenClasses.Whitespace); return(whitespace); }
public void RegexToDfaShouldConvertOptionalCharacterClassToDfa() { var pattern = @"[-+]?[0-9]"; var dfa = CreateDfaFromRegexPattern(pattern); Assert.IsNotNull(dfa); Assert.AreEqual(3, dfa.Transitions.Count); var lexerRule = new DfaLexerRule(dfa, pattern); AssertLexerRuleMatches(lexerRule, "+0"); AssertLexerRuleMatches(lexerRule, "-1"); AssertLexerRuleMatches(lexerRule, "9"); }
public void RegexToDfaShouldConvertSlasSToWhitespaceCharacterClass() { var pattern = @"\s"; var dfa = CreateDfaFromRegexPattern(pattern); Assert.IsNotNull(dfa); Assert.AreEqual(1, dfa.Transitions.Count); Assert.AreEqual(0, dfa.Transitions[0].Target.Transitions.Count); var lexerRule = new DfaLexerRule(dfa, pattern); AssertLexerRuleMatches(lexerRule, " "); AssertLexerRuleMatches(lexerRule, "\t"); AssertLexerRuleMatches(lexerRule, "\f"); }
public void RegexToDfaShouldConvertCharacterRegexToDfa() { var pattern = "a"; var regex = new RegexParser().Parse(pattern); var nfa = new ThompsonConstructionAlgorithm().Transform(regex); var dfa = new SubsetConstructionAlgorithm().Transform(nfa); Assert.IsNotNull(dfa); var lexerRule = new DfaLexerRule(dfa, "a"); var lexeme = new DfaLexemeFactory().Create(lexerRule, 0); Assert.IsTrue(lexeme.Scan('a')); }
IEnumerable <ProductionModel> Factor(EbnfFactor factor, ProductionModel currentProduction) { switch (factor.NodeType) { case EbnfNodeType.EbnfFactorGrouping: var grouping = factor as EbnfFactorGrouping; foreach (var production in Grouping(grouping, currentProduction)) { yield return(production); } break; case EbnfNodeType.EbnfFactorOptional: var optional = factor as EbnfFactorOptional; foreach (var production in Optional(optional, currentProduction)) { yield return(production); } break; case EbnfNodeType.EbnfFactorRepetition: var repetition = factor as EbnfFactorRepetition; foreach (var production in Repetition(repetition, currentProduction)) { yield return(production); } break; case EbnfNodeType.EbnfFactorIdentifier: var identifier = factor as EbnfFactorIdentifier; var nonTerminal = GetNonTerminalFromQualifiedIdentifier(identifier.QualifiedIdentifier); currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal)); break; case EbnfNodeType.EbnfFactorLiteral: var literal = factor as EbnfFactorLiteral; var stringLiteralRule = new StringLiteralLexerRule(literal.Value); currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule)); break; case EbnfNodeType.EbnfFactorRegex: var regex = factor as EbnfFactorRegex; var nfa = _thompsonConstructionAlgorithm.Transform(regex.Regex); var dfa = _subsetConstructionAlgorithm.Transform(nfa); var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString()); currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule)); break; } }
public void DfaLexemeGivenCharacerLexemeNumberShouldFail() { var numberInput = "0"; var dfa = new DfaState(); var final = new DfaState(true); dfa.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final)); final.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final)); var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("lowerCase")); var letterLexeme = new DfaLexeme(dfaLexerRule, 0); Assert.IsFalse(letterLexeme.Scan(numberInput[0])); Assert.AreEqual(string.Empty, letterLexeme.Value); }
public void RegexToDfaShouldConvertOptionalCharacterClassToDfa() { var pattern = @"[-+]?[0-9]"; var regex = new RegexParser().Parse(pattern); var nfa = new ThompsonConstructionAlgorithm().Transform(regex); var dfa = new SubsetConstructionAlgorithm().Transform(nfa); Assert.IsNotNull(dfa); Assert.AreEqual(3, dfa.Transitions.Count); var lexerRule = new DfaLexerRule(dfa, pattern); AssertLexerRuleMatches(lexerRule, "+0"); AssertLexerRuleMatches(lexerRule, "-1"); AssertLexerRuleMatches(lexerRule, "9"); }
private static LexerRule CreateRuleNameLexerRule() { // /[a-zA-Z][a-zA-Z0-9-_]*/ var ruleNameState = DfaState.Inner(); var zeroOrMoreLetterOrDigit = DfaState.Final(); ruleNameState.AddTransition(AsciiLetterTerminal.Instance, zeroOrMoreLetterOrDigit); zeroOrMoreLetterOrDigit.AddTransition( new CharacterClassTerminal( AsciiLetterTerminal.Instance, DigitTerminal.Instance, new SetTerminal("-_")), zeroOrMoreLetterOrDigit); var ruleName = new DfaLexerRule(ruleNameState, "rule-name"); return(ruleName); }
public void DfaLexemeShouldMatchOneOrMoreRandomWhitespaceCharacters() { var randomWhitespace = "\t\f\v \r\n"; var dfa = new DfaState(); var final = new DfaState(true); dfa.AddTransition(new DfaTransition(new WhitespaceTerminal(), final)); final.AddTransition(new DfaTransition(new WhitespaceTerminal(), final)); var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("whitespace")); var whitespaceLexeme = new DfaLexeme(dfaLexerRule, 0); for (int i = 0; i < randomWhitespace.Length; i++) { Assert.IsTrue(whitespaceLexeme.Scan(randomWhitespace[i])); } }
public void DfaLexemeShouldMatchOneOrMoreRandomWhitespaceCharacters() { var randomWhitespace = "\t\f\v \r\n"; var dfa = DfaState.Inner(); var final = DfaState.Final(); dfa.AddTransition(WhitespaceTerminal.Instance, final); final.AddTransition(WhitespaceTerminal.Instance, final); var dfaLexerRule = new DfaLexerRule(dfa, new TokenName("whitespace")); var whitespaceLexeme = dfaLexerRule.CreateLexeme(0); foreach (var character in randomWhitespace) { Assert.IsTrue(whitespaceLexeme.Scan(character)); } }
private IEnumerable <ProductionModel> Factor(IEbnfFactor factor, ProductionModel currentProduction) { switch (factor) { case EbnfFactorGrouping grouping: foreach (var production in Grouping(grouping, currentProduction)) { yield return(production); } break; case EbnfFactorOptional optional: foreach (var production in Optional(optional, currentProduction)) { yield return(production); } break; case EbnfFactorRepetition repetition: foreach (var production in Repetition(repetition, currentProduction)) { yield return(production); } break; case EbnfFactorIdentifier identifier: var nonTerminal = NonTerminal.From(GetQualifiedName(identifier.QualifiedIdentifier)); currentProduction.AddWithAnd(new NonTerminalModel(nonTerminal)); break; case EbnfFactorLiteral literal: var stringLiteralRule = new StringLiteralLexerRule(literal.Value); currentProduction.AddWithAnd(new LexerRuleModel(stringLiteralRule)); break; case EbnfFactorRegex regex: var nfa = this.regexToNfa.Transform(regex.Regex); var dfa = this.nfaToDfa.Transform(nfa); var dfaLexerRule = new DfaLexerRule(dfa, regex.Regex.ToString()); currentProduction.AddWithAnd(new LexerRuleModel(dfaLexerRule)); break; } }
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition() { var states = new DfaState[2] { DfaState.Inner(), DfaState.Final() }; states[0].AddTransition(WhitespaceTerminal.Instance, states[1]); states[1].AddTransition(WhitespaceTerminal.Instance, states[1]); var dfaLexerRule = new DfaLexerRule(states[0], new TokenName(@"\s+")); Assert.IsTrue(dfaLexerRule.CanApply(' ')); Assert.IsTrue(dfaLexerRule.CanApply('\t')); Assert.IsTrue(dfaLexerRule.CanApply('\r')); Assert.IsFalse(dfaLexerRule.CanApply('a')); }
public void DfaLexemeShouldMatchMixedCaseWord() { var wordInput = "t90vAriabl3"; var dfa = new DfaState(); var final = new DfaState(true); dfa.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final)); dfa.AddTransition(new DfaTransition(new RangeTerminal('A', 'Z'), final)); final.AddTransition(new DfaTransition(new RangeTerminal('a', 'z'), final)); final.AddTransition(new DfaTransition(new RangeTerminal('A', 'Z'), final)); final.AddTransition(new DfaTransition(new DigitTerminal(), final)); var dfaLexerRule = new DfaLexerRule(dfa, new TokenType("Identifier")); var indentifierLexeme = new DfaLexeme(dfaLexerRule, 0); for (int i = 0; i < wordInput.Length; i++) { Assert.IsTrue(indentifierLexeme.Scan(wordInput[i])); } }
public void DfaLexemeShouldMatchMixedCaseWord() { var wordInput = "t90vAriabl3"; var dfa = DfaState.Inner(); var final = DfaState.Final(); dfa.AddTransition(new RangeTerminal('a', 'z'), final); dfa.AddTransition(new RangeTerminal('A', 'Z'), final); final.AddTransition(new RangeTerminal('a', 'z'), final); final.AddTransition(new RangeTerminal('A', 'Z'), final); final.AddTransition(DigitTerminal.Instance, final); var dfaLexerRule = new DfaLexerRule(dfa, new TokenName("Identifier")); var indentifierLexeme = dfaLexerRule.CreateLexeme(0); foreach (var character in wordInput) { Assert.IsTrue(indentifierLexeme.Scan(character)); } }
public void DfaLexerRuleShouldApplyToCharacterIfFirstStateHasTransition() { var states = new DfaState[2]; for (var i = 0; i < states.Length; i++) { states[i] = new DfaState(i == states.Length - 1); } var whitespaceToFinal = new DfaTransition(new WhitespaceTerminal(), states[1]); states[0].AddTransition(whitespaceToFinal); states[1].AddTransition(whitespaceToFinal); var dfaLexerRule = new DfaLexerRule(states[0], new TokenType(@"\s+")); Assert.IsTrue(dfaLexerRule.CanApply(' ')); Assert.IsTrue(dfaLexerRule.CanApply('\t')); Assert.IsTrue(dfaLexerRule.CanApply('\r')); Assert.IsFalse(dfaLexerRule.CanApply('a')); }
private static ILexerRule CreateRuleNameLexerRule() { var ruleNameState = new DfaState(); var zeroOrMoreLetterOrDigit = new DfaState(true); ruleNameState.AddTransition( new DfaTransition( new CharacterClassTerminal( new RangeTerminal('a', 'z'), new RangeTerminal('A', 'Z')), zeroOrMoreLetterOrDigit)); zeroOrMoreLetterOrDigit.AddTransition( new DfaTransition( new CharacterClassTerminal( new RangeTerminal('a', 'z'), new RangeTerminal('A', 'Z'), new DigitTerminal(), new SetTerminal('-', '_')), zeroOrMoreLetterOrDigit)); var ruleName = new DfaLexerRule(ruleNameState, new TokenType("rule-name")); return(ruleName); }
private static BaseLexerRule Identifier() { // /[a-zA-Z][a-zA-Z0-9-_]*/ var identifierState = new DfaState(); var zeroOrMoreLetterOrDigit = new DfaState(true); identifierState.AddTransition( new DfaTransition( new CharacterClassTerminal( new RangeTerminal('a', 'z'), new RangeTerminal('A', 'Z')), zeroOrMoreLetterOrDigit)); zeroOrMoreLetterOrDigit.AddTransition( new DfaTransition( new CharacterClassTerminal( new RangeTerminal('a', 'z'), new RangeTerminal('A', 'Z'), new DigitTerminal(), new SetTerminal('-', '_')), zeroOrMoreLetterOrDigit)); var identifier = new DfaLexerRule(identifierState, TokenTypes.Identifier); return(identifier); }