private void RegisterRule(TokenRule rule)
{
if (rule != null)
{
_tokenRules[rule.Name] = rule;
}
}
// Define a new token rule, with a name and an optional rule
public TokenRule AddTokenRule(TokenType type, string rule,
bool canBeginLine = false,
bool resetsLine = false,
bool requiresFollowingWhitespace = false,
TokenRule.FreeTextMode freeTextMode = TokenRule.FreeTextMode.DoNotMatch
)
{
var newTokenRule = new TokenRule();
newTokenRule.type = type;
if (requiresFollowingWhitespace)
{
// Look for whitespace following this rule, but don't capture it as part of
// the token
rule += @"(?=\s+)";
}
// Set up a regex if we have a rule for it
if (rule != null)
{
newTokenRule.regex = new Regex(rule);
}
newTokenRule.resetsLine = resetsLine;
newTokenRule.canBeginLine = canBeginLine;
newTokenRule.freeTextMode = freeTextMode;
// Store it in the list and return
tokenRules.Add(newTokenRule);
return(newTokenRule);
}
public TokenRule AddTransition(TokenType type, string entersState = null, bool delimitsText = false)
{
var pattern = string.Format(@"\G{0}", patterns [type]);
var rule = new TokenRule(type, new Regex(pattern), entersState, delimitsText);
tokenRules.Add(rule);
return(rule);
}
protected static TokenRule CreateTokenRule(TToken token, [CallerMemberName] string name = null)
{
var tokenRule = new TokenRule(token)
{
Name = name
};
CreatedTokenRules.Add(tokenRule);
return(tokenRule);
}
private void DefineToken(string name, Token token)
{
AssertIsNotLocked();
tokens.Add(name, token);
Rule rule = new TokenRule(this, token);
if (!ImplicitNames.Contains(name))
{
rule = new NameRule(this, this, RuleName.Implicit).And(rule);
}
DefineRule(name, rule);
}
public static void TestIntRule()
{
var rule = new TokenRule(Python3Parser.INT_REGEX);
AssertEquals(rule.Matches(""), false);
AssertEquals(rule.Matches("foo"), false);
AssertEquals(rule.Matches("x0"), false);
AssertEquals(rule.Matches("x0.0"), false);
AssertEquals(rule.Matches("0x"), false);
AssertEquals(rule.Matches("0.0x"), false);
AssertEquals(rule.Matches("01"), false);
AssertEquals(rule.Matches("123abc"), false);
AssertEquals(rule.Matches("abc123"), false);
TestBooleanStrings(rule, false);
TestIntStrings(rule, true);
TestFloatStrings(rule, false);
}
public static void TestBooleanRule()
{
var rule = new TokenRule(Python3Parser.BOOL_REGEX);
AssertEquals(rule.Matches(""), false);
AssertEquals(rule.Matches("foo"), false);
AssertEquals(rule.Matches("x0"), false);
AssertEquals(rule.Matches("x0.0"), false);
AssertEquals(rule.Matches("0x"), false);
AssertEquals(rule.Matches("0.0x"), false);
AssertEquals(rule.Matches("Trueabc"), false);
AssertEquals(rule.Matches("abcTrue"), false);
AssertEquals(rule.Matches("Falseabc"), false);
AssertEquals(rule.Matches("abcFalse"), false);
TestBooleanStrings(rule, true);
TestIntStrings(rule, false);
TestFloatStrings(rule, false);
}
public Token NextToken()
{
while (!EOF && ignore(CurrentChar))
{
collumnNumber++;
if (CurrentChar == '\n')
{
lineNumber++;
collumnNumber = 0;
}
Index++;
}
if (EOF)
{
return(Construct(Token.EOF));
}
Match bestMatch = null;
TokenRule bestFunc = null;
foreach (var rule in rules)
{
Regex regex = rule.Key;
var m = regex.Match(Text);
if (m.Success)
{
if (bestMatch == null)
{
bestMatch = m;
bestFunc = rule.Value;
if (prioritizeMethod == PrioritizeMatchKind.First)
{
break;
}
}
else
{
switch (prioritizeMethod)
{
case PrioritizeMatchKind.First:
break;
case PrioritizeMatchKind.Last:
bestMatch = m;
bestFunc = rule.Value;
break;
case PrioritizeMatchKind.Longest:
if (m.Length > bestMatch.Length)
{
bestMatch = m;
bestFunc = rule.Value;
}
break;
case PrioritizeMatchKind.Shortest:
if (m.Length < bestMatch.Length)
{
bestMatch = m;
bestFunc = rule.Value;
}
break;
}
}
}
}
if (bestMatch != null)
{
Index += bestMatch.Length;
return(Construct(bestFunc, bestMatch));
}
foreach (var symbolRule in specialSymbolRules)
{
var token = symbolRule(CurrentChar);
if (token)
{
var chara = CurrentChar;
Index += 1;
return(Construct(token));
}
}
Index += 1;
return(Construct(Token.UNKNOWN));
}
public void AddRegexRule(Regex regex, TokenRule rule)
{
rules.Add(regex, rule);
}
Token Construct(TokenRule rule, Match match)
{
var token = rule(match);
return(Construct(token, match.Length));
}
public ParserTokenRuleEntry(TokenRule tokenRule)
{
TokenRule = tokenRule;
}
private static void TestFloatStrings(TokenRule rule, bool isFloat)
{
AssertEquals(rule.Matches("0."), isFloat);
AssertEquals(rule.Matches(".0"), isFloat);
AssertEquals(rule.Matches("1."), isFloat);
AssertEquals(rule.Matches(".1"), isFloat);
AssertEquals(rule.Matches("0.0"), isFloat);
AssertEquals(rule.Matches("1.0"), isFloat);
AssertEquals(rule.Matches("0.1"), isFloat);
AssertEquals(rule.Matches("01.0"), isFloat);
AssertEquals(rule.Matches("0.10"), isFloat);
AssertEquals(rule.Matches("21.1"), isFloat);
AssertEquals(rule.Matches("0e0"), isFloat);
AssertEquals(rule.Matches("0e+3"), isFloat);
AssertEquals(rule.Matches("0e-3"), isFloat);
AssertEquals(rule.Matches("21.1e3"), isFloat);
AssertEquals(rule.Matches("21.1e+3"), isFloat);
AssertEquals(rule.Matches("21.1e-3"), isFloat);
AssertEquals(rule.Matches("21.1e0"), isFloat);
AssertEquals(rule.Matches("21.1e+0"), isFloat);
AssertEquals(rule.Matches("21.1e-0"), isFloat);
AssertEquals(rule.Matches("+0."), isFloat);
AssertEquals(rule.Matches("+.0"), isFloat);
AssertEquals(rule.Matches("+1."), isFloat);
AssertEquals(rule.Matches("+.1"), isFloat);
AssertEquals(rule.Matches("+0.0"), isFloat);
AssertEquals(rule.Matches("+1.0"), isFloat);
AssertEquals(rule.Matches("+0.1"), isFloat);
AssertEquals(rule.Matches("+01.0"), isFloat);
AssertEquals(rule.Matches("+0.10"), isFloat);
AssertEquals(rule.Matches("+21.1"), isFloat);
AssertEquals(rule.Matches("+0e0"), isFloat);
AssertEquals(rule.Matches("+0e+3"), isFloat);
AssertEquals(rule.Matches("+0e-3"), isFloat);
AssertEquals(rule.Matches("+21.1e3"), isFloat);
AssertEquals(rule.Matches("+21.1e+3"), isFloat);
AssertEquals(rule.Matches("+21.1e-3"), isFloat);
AssertEquals(rule.Matches("+21.1e0"), isFloat);
AssertEquals(rule.Matches("+21.1e+0"), isFloat);
AssertEquals(rule.Matches("+21.1e-0"), isFloat);
AssertEquals(rule.Matches("-0."), isFloat);
AssertEquals(rule.Matches("-.0"), isFloat);
AssertEquals(rule.Matches("-1."), isFloat);
AssertEquals(rule.Matches("-.1"), isFloat);
AssertEquals(rule.Matches("-0.0"), isFloat);
AssertEquals(rule.Matches("-1.0"), isFloat);
AssertEquals(rule.Matches("-0.1"), isFloat);
AssertEquals(rule.Matches("-01.0"), isFloat);
AssertEquals(rule.Matches("-0.10"), isFloat);
AssertEquals(rule.Matches("-21.1"), isFloat);
AssertEquals(rule.Matches("-0e0"), isFloat);
AssertEquals(rule.Matches("-0e+3"), isFloat);
AssertEquals(rule.Matches("-0e-3"), isFloat);
AssertEquals(rule.Matches("-21.1e3"), isFloat);
AssertEquals(rule.Matches("-21.1e+3"), isFloat);
AssertEquals(rule.Matches("-21.1e-3"), isFloat);
AssertEquals(rule.Matches("-21.1e0"), isFloat);
AssertEquals(rule.Matches("-21.1e+0"), isFloat);
AssertEquals(rule.Matches("-21.1e-0"), isFloat);
}
private static void TestBooleanStrings(TokenRule rule, bool isBool)
{
AssertEquals(rule.Matches("True"), isBool);
AssertEquals(rule.Matches("False"), isBool);
}
private static void TestIntStrings(TokenRule rule, bool isInt)
{
AssertEquals(rule.Matches("0"), isInt);
AssertEquals(rule.Matches("000000000"), isInt);
AssertEquals(rule.Matches("10"), isInt);
AssertEquals(rule.Matches("11"), isInt);
AssertEquals(rule.Matches("123451234512345"), isInt);
AssertEquals(rule.Matches("+0"), isInt);
AssertEquals(rule.Matches("+000000000"), isInt);
AssertEquals(rule.Matches("+10"), isInt);
AssertEquals(rule.Matches("+11"), isInt);
AssertEquals(rule.Matches("+123451234512345"), isInt);
AssertEquals(rule.Matches("-0"), isInt);
AssertEquals(rule.Matches("-000000000"), isInt);
AssertEquals(rule.Matches("-10"), isInt);
AssertEquals(rule.Matches("-11"), isInt);
AssertEquals(rule.Matches("-123451234512345"), isInt);
}
private void RegisterRule(TokenRule rule)
{
if (rule != null)
{
m_TokenRules[rule.Name] = rule;
}
}
public TokenInterRuleEntry(TokenRule tokenRule)
{
TokenRule = tokenRule;
}
