private void RegisterRule(TokenRule rule) { if (rule != null) { _tokenRules[rule.Name] = rule; } }
// Define a new token rule, with a name and an optional rule public TokenRule AddTokenRule(TokenType type, string rule, bool canBeginLine = false, bool resetsLine = false, bool requiresFollowingWhitespace = false, TokenRule.FreeTextMode freeTextMode = TokenRule.FreeTextMode.DoNotMatch ) { var newTokenRule = new TokenRule(); newTokenRule.type = type; if (requiresFollowingWhitespace) { // Look for whitespace following this rule, but don't capture it as part of // the token rule += @"(?=\s+)"; } // Set up a regex if we have a rule for it if (rule != null) { newTokenRule.regex = new Regex(rule); } newTokenRule.resetsLine = resetsLine; newTokenRule.canBeginLine = canBeginLine; newTokenRule.freeTextMode = freeTextMode; // Store it in the list and return tokenRules.Add(newTokenRule); return(newTokenRule); }
public TokenRule AddTransition(TokenType type, string entersState = null, bool delimitsText = false) { var pattern = string.Format(@"\G{0}", patterns [type]); var rule = new TokenRule(type, new Regex(pattern), entersState, delimitsText); tokenRules.Add(rule); return(rule); }
protected static TokenRule CreateTokenRule(TToken token, [CallerMemberName] string name = null) { var tokenRule = new TokenRule(token) { Name = name }; CreatedTokenRules.Add(tokenRule); return(tokenRule); }
private void DefineToken(string name, Token token) { AssertIsNotLocked(); tokens.Add(name, token); Rule rule = new TokenRule(this, token); if (!ImplicitNames.Contains(name)) { rule = new NameRule(this, this, RuleName.Implicit).And(rule); } DefineRule(name, rule); }
public static void TestIntRule() { var rule = new TokenRule(Python3Parser.INT_REGEX); AssertEquals(rule.Matches(""), false); AssertEquals(rule.Matches("foo"), false); AssertEquals(rule.Matches("x0"), false); AssertEquals(rule.Matches("x0.0"), false); AssertEquals(rule.Matches("0x"), false); AssertEquals(rule.Matches("0.0x"), false); AssertEquals(rule.Matches("01"), false); AssertEquals(rule.Matches("123abc"), false); AssertEquals(rule.Matches("abc123"), false); TestBooleanStrings(rule, false); TestIntStrings(rule, true); TestFloatStrings(rule, false); }
public static void TestBooleanRule() { var rule = new TokenRule(Python3Parser.BOOL_REGEX); AssertEquals(rule.Matches(""), false); AssertEquals(rule.Matches("foo"), false); AssertEquals(rule.Matches("x0"), false); AssertEquals(rule.Matches("x0.0"), false); AssertEquals(rule.Matches("0x"), false); AssertEquals(rule.Matches("0.0x"), false); AssertEquals(rule.Matches("Trueabc"), false); AssertEquals(rule.Matches("abcTrue"), false); AssertEquals(rule.Matches("Falseabc"), false); AssertEquals(rule.Matches("abcFalse"), false); TestBooleanStrings(rule, true); TestIntStrings(rule, false); TestFloatStrings(rule, false); }
public Token NextToken() { while (!EOF && ignore(CurrentChar)) { collumnNumber++; if (CurrentChar == '\n') { lineNumber++; collumnNumber = 0; } Index++; } if (EOF) { return(Construct(Token.EOF)); } Match bestMatch = null; TokenRule bestFunc = null; foreach (var rule in rules) { Regex regex = rule.Key; var m = regex.Match(Text); if (m.Success) { if (bestMatch == null) { bestMatch = m; bestFunc = rule.Value; if (prioritizeMethod == PrioritizeMatchKind.First) { break; } } else { switch (prioritizeMethod) { case PrioritizeMatchKind.First: break; case PrioritizeMatchKind.Last: bestMatch = m; bestFunc = rule.Value; break; case PrioritizeMatchKind.Longest: if (m.Length > bestMatch.Length) { bestMatch = m; bestFunc = rule.Value; } break; case PrioritizeMatchKind.Shortest: if (m.Length < bestMatch.Length) { bestMatch = m; bestFunc = rule.Value; } break; } } } } if (bestMatch != null) { Index += bestMatch.Length; return(Construct(bestFunc, bestMatch)); } foreach (var symbolRule in specialSymbolRules) { var token = symbolRule(CurrentChar); if (token) { var chara = CurrentChar; Index += 1; return(Construct(token)); } } Index += 1; return(Construct(Token.UNKNOWN)); }
public void AddRegexRule(Regex regex, TokenRule rule) { rules.Add(regex, rule); }
Token Construct(TokenRule rule, Match match) { var token = rule(match); return(Construct(token, match.Length)); }
public ParserTokenRuleEntry(TokenRule tokenRule) { TokenRule = tokenRule; }
private static void TestFloatStrings(TokenRule rule, bool isFloat) { AssertEquals(rule.Matches("0."), isFloat); AssertEquals(rule.Matches(".0"), isFloat); AssertEquals(rule.Matches("1."), isFloat); AssertEquals(rule.Matches(".1"), isFloat); AssertEquals(rule.Matches("0.0"), isFloat); AssertEquals(rule.Matches("1.0"), isFloat); AssertEquals(rule.Matches("0.1"), isFloat); AssertEquals(rule.Matches("01.0"), isFloat); AssertEquals(rule.Matches("0.10"), isFloat); AssertEquals(rule.Matches("21.1"), isFloat); AssertEquals(rule.Matches("0e0"), isFloat); AssertEquals(rule.Matches("0e+3"), isFloat); AssertEquals(rule.Matches("0e-3"), isFloat); AssertEquals(rule.Matches("21.1e3"), isFloat); AssertEquals(rule.Matches("21.1e+3"), isFloat); AssertEquals(rule.Matches("21.1e-3"), isFloat); AssertEquals(rule.Matches("21.1e0"), isFloat); AssertEquals(rule.Matches("21.1e+0"), isFloat); AssertEquals(rule.Matches("21.1e-0"), isFloat); AssertEquals(rule.Matches("+0."), isFloat); AssertEquals(rule.Matches("+.0"), isFloat); AssertEquals(rule.Matches("+1."), isFloat); AssertEquals(rule.Matches("+.1"), isFloat); AssertEquals(rule.Matches("+0.0"), isFloat); AssertEquals(rule.Matches("+1.0"), isFloat); AssertEquals(rule.Matches("+0.1"), isFloat); AssertEquals(rule.Matches("+01.0"), isFloat); AssertEquals(rule.Matches("+0.10"), isFloat); AssertEquals(rule.Matches("+21.1"), isFloat); AssertEquals(rule.Matches("+0e0"), isFloat); AssertEquals(rule.Matches("+0e+3"), isFloat); AssertEquals(rule.Matches("+0e-3"), isFloat); AssertEquals(rule.Matches("+21.1e3"), isFloat); AssertEquals(rule.Matches("+21.1e+3"), isFloat); AssertEquals(rule.Matches("+21.1e-3"), isFloat); AssertEquals(rule.Matches("+21.1e0"), isFloat); AssertEquals(rule.Matches("+21.1e+0"), isFloat); AssertEquals(rule.Matches("+21.1e-0"), isFloat); AssertEquals(rule.Matches("-0."), isFloat); AssertEquals(rule.Matches("-.0"), isFloat); AssertEquals(rule.Matches("-1."), isFloat); AssertEquals(rule.Matches("-.1"), isFloat); AssertEquals(rule.Matches("-0.0"), isFloat); AssertEquals(rule.Matches("-1.0"), isFloat); AssertEquals(rule.Matches("-0.1"), isFloat); AssertEquals(rule.Matches("-01.0"), isFloat); AssertEquals(rule.Matches("-0.10"), isFloat); AssertEquals(rule.Matches("-21.1"), isFloat); AssertEquals(rule.Matches("-0e0"), isFloat); AssertEquals(rule.Matches("-0e+3"), isFloat); AssertEquals(rule.Matches("-0e-3"), isFloat); AssertEquals(rule.Matches("-21.1e3"), isFloat); AssertEquals(rule.Matches("-21.1e+3"), isFloat); AssertEquals(rule.Matches("-21.1e-3"), isFloat); AssertEquals(rule.Matches("-21.1e0"), isFloat); AssertEquals(rule.Matches("-21.1e+0"), isFloat); AssertEquals(rule.Matches("-21.1e-0"), isFloat); }
private static void TestBooleanStrings(TokenRule rule, bool isBool) { AssertEquals(rule.Matches("True"), isBool); AssertEquals(rule.Matches("False"), isBool); }
private static void TestIntStrings(TokenRule rule, bool isInt) { AssertEquals(rule.Matches("0"), isInt); AssertEquals(rule.Matches("000000000"), isInt); AssertEquals(rule.Matches("10"), isInt); AssertEquals(rule.Matches("11"), isInt); AssertEquals(rule.Matches("123451234512345"), isInt); AssertEquals(rule.Matches("+0"), isInt); AssertEquals(rule.Matches("+000000000"), isInt); AssertEquals(rule.Matches("+10"), isInt); AssertEquals(rule.Matches("+11"), isInt); AssertEquals(rule.Matches("+123451234512345"), isInt); AssertEquals(rule.Matches("-0"), isInt); AssertEquals(rule.Matches("-000000000"), isInt); AssertEquals(rule.Matches("-10"), isInt); AssertEquals(rule.Matches("-11"), isInt); AssertEquals(rule.Matches("-123451234512345"), isInt); }
private void RegisterRule(TokenRule rule) { if (rule != null) { m_TokenRules[rule.Name] = rule; } }
public TokenInterRuleEntry(TokenRule tokenRule) { TokenRule = tokenRule; }