//*
// * Returns the token pattern with the specified id. Only
// * token patterns handled by this matcher can be returned.
// *
// * @param id the token pattern id
// *
// * @return the token pattern found, or
// * null if not found
//
public TokenPattern GetPattern(int id)
{
TokenPattern pattern = default(TokenPattern);
for (int i = 0; i <= patterns.Count - 1; i++)
{
pattern = (TokenPattern)patterns[i];
if (pattern.Id == id)
{
return(pattern);
}
}
return(null);
}
//*
// * Creates a new token.
// *
// * @param pattern the token pattern
// * @param image the token image (i.e. characters)
// * @param line the line number of the first character
// * @param col the column number of the first character
//
public Token(TokenPattern pattern, string image, int line, int col)
{
this.m_pattern = pattern;
this.m_image = image;
this.m_startLine = line;
this.m_startColumn = col;
this.m_endLine = line;
this.m_endColumn = col + image.Length - 1;
int pos = 0;
while (image.IndexOf(Convert.ToChar(10), pos) >= 0)
{
pos = image.IndexOf(Convert.ToChar(10), pos) + 1;
this.m_endLine += 1;
m_endColumn = image.Length - pos;
}
}
//*
// * Returns a description of the token pattern with the
// * specified id.
// *
// * @param id the token pattern id
// *
// * @return the token pattern description, or
// * null if not present
//
public string GetPatternDescription(int id)
{
TokenPattern pattern = default(TokenPattern);
RegExpTokenMatcher re = default(RegExpTokenMatcher);
pattern = stringMatcher.GetPattern(id);
if (pattern != null)
{
return(pattern.ToShortString());
}
for (int i = 0; i <= regexpMatchers.Count - 1; i++)
{
re = (RegExpTokenMatcher)regexpMatchers[i];
if (re.GetPattern().Id == id)
{
return(re.GetPattern().ToShortString());
}
}
return(null);
}
//*
// * Adds a new token pattern to the tokenizer. The pattern will be
// * added last in the list, choosing a previous token pattern in
// * case two matches the same string.
// *
// * @param pattern the pattern to add
// *
// * @throws ParserCreationException if the pattern couldn't be
// * added to the tokenizer
//
public void AddPattern(TokenPattern pattern)
{
switch (pattern.Type)
{
case TokenPattern.PatternType.STRING:
stringMatcher.AddPattern(pattern);
break;
case TokenPattern.PatternType.REGEXP:
try
{
regexpMatchers.Add(new RegExpTokenMatcher(pattern, ignoreCase, input));
}
catch (RegExpException e)
{
throw new ParserCreationException(ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "regular expression contains error(s): " + e.Message);
}
break;
default:
throw new ParserCreationException(ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "pattern type " + pattern.Type + " is undefined");
}
}
///<summary>Initializes the tokenizer by creating all the token
///patterns.</summary>
///
///<exception cref='ParserCreationException'>if the tokenizer
///couldn't be initialized correctly</exception>
private void CreatePatterns()
{
TokenPattern pattern = default(TokenPattern);
CustomTokenPattern customPattern = default(CustomTokenPattern);
pattern = new TokenPattern((int)ExpressionConstants.ADD, "ADD", TokenPattern.PatternType.STRING, "+");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.SUB, "SUB", TokenPattern.PatternType.STRING, "-");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.MUL, "MUL", TokenPattern.PatternType.STRING, "*");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.DIV, "DIV", TokenPattern.PatternType.STRING, "/");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.POWER, "POWER", TokenPattern.PatternType.STRING, "^");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.MOD, "MOD", TokenPattern.PatternType.STRING, "%");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.LEFT_PAREN, "LEFT_PAREN", TokenPattern.PatternType.STRING, "(");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.RIGHT_PAREN, "RIGHT_PAREN", TokenPattern.PatternType.STRING, ")");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.LEFT_BRACE, "LEFT_BRACE", TokenPattern.PatternType.STRING, "[");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.RIGHT_BRACE, "RIGHT_BRACE", TokenPattern.PatternType.STRING, "]");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.EQ, "EQ", TokenPattern.PatternType.STRING, "=");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.LT, "LT", TokenPattern.PatternType.STRING, "<");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.GT, "GT", TokenPattern.PatternType.STRING, ">");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.LTE, "LTE", TokenPattern.PatternType.STRING, "<=");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.GTE, "GTE", TokenPattern.PatternType.STRING, ">=");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.NE, "NE", TokenPattern.PatternType.STRING, "<>");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.AND, "AND", TokenPattern.PatternType.STRING, "AND");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.OR, "OR", TokenPattern.PatternType.STRING, "OR");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.XOR, "XOR", TokenPattern.PatternType.STRING, "XOR");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.NOT, "NOT", TokenPattern.PatternType.STRING, "NOT");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.IN, "IN", TokenPattern.PatternType.STRING, "in");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.DOT, "DOT", TokenPattern.PatternType.STRING, ".");
AddPattern(pattern);
customPattern = new ArgumentSeparatorPattern();
customPattern.Initialize((int)ExpressionConstants.ARGUMENT_SEPARATOR, "ARGUMENT_SEPARATOR", TokenPattern.PatternType.STRING, ",", MyContext);
AddPattern(customPattern);
pattern = new TokenPattern((int)ExpressionConstants.ARRAY_BRACES, "ARRAY_BRACES", TokenPattern.PatternType.STRING, "[]");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.LEFT_SHIFT, "LEFT_SHIFT", TokenPattern.PatternType.STRING, "<<");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.RIGHT_SHIFT, "RIGHT_SHIFT", TokenPattern.PatternType.STRING, ">>");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.WHITESPACE, "WHITESPACE", TokenPattern.PatternType.REGEXP, "\\s+");
pattern.Ignore = true;
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.INTEGER, "INTEGER", TokenPattern.PatternType.REGEXP, "\\d+(u|l|ul|lu)?");
AddPattern(pattern);
customPattern = new RealPattern();
customPattern.Initialize((int)ExpressionConstants.REAL, "REAL", TokenPattern.PatternType.REGEXP, "\\d*\\.\\d+([e][+-]\\d{1,3})?f?", MyContext);
AddPattern(customPattern);
pattern = new TokenPattern((int)ExpressionConstants.STRING_LITERAL, "STRING_LITERAL", TokenPattern.PatternType.REGEXP, "\"([^\"\\r\\n\\\\]|\\\\u[0-9a-f]{4}|\\\\[\\\\\"'trn])*\"");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.CHAR_LITERAL, "CHAR_LITERAL", TokenPattern.PatternType.REGEXP, "'([^'\\r\\n\\\\]|\\\\u[0-9a-f]{4}|\\\\[\\\\\"'trn])'");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.TRUE, "TRUE", TokenPattern.PatternType.STRING, "True");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.FALSE, "FALSE", TokenPattern.PatternType.STRING, "False");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.IDENTIFIER, "IDENTIFIER", TokenPattern.PatternType.REGEXP, "[a-z_]\\w*");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.HEX_LITERAL, "HEX_LITERAL", TokenPattern.PatternType.REGEXP, "0x[0-9a-f]+(u|l|ul|lu)?");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.NULL_LITERAL, "NULL_LITERAL", TokenPattern.PatternType.STRING, "null");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.TIMESPAN, "TIMESPAN", TokenPattern.PatternType.REGEXP, "##(\\d+\\.)?\\d{2}:\\d{2}(:\\d{2}(\\.\\d{1,7})?)?#");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.DATETIME, "DATETIME", TokenPattern.PatternType.REGEXP, "#[^#]+#");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.IF, "IF", TokenPattern.PatternType.STRING, "if");
AddPattern(pattern);
pattern = new TokenPattern((int)ExpressionConstants.CAST, "CAST", TokenPattern.PatternType.STRING, "cast");
AddPattern(pattern);
}
//*
// * Checks if the token pattern matches the input stream. This
// * method will also reset all flags in this matcher.
// *
// * @param input the input stream to match
// *
// * @return true if a match was found, or
// * false otherwise
// *
// * @throws IOException if an I/O error occurred
//
public bool Match(LookAheadReader input)
{
this.Reset();
m_match = (TokenPattern)start.MatchFrom(input, 0, ignoreCase);
return(m_match != null);
}
//*
// * Adds a string token pattern to this matcher.
// *
// * @param pattern the pattern to add
//
public void AddPattern(TokenPattern pattern)
{
patterns.Add(pattern);
start.AddMatch(pattern.Pattern, ignoreCase, pattern);
}
//*
// * Resets the matcher state. This will clear the results of
// * the last match.
//
public void Reset()
{
m_match = null;
}
//*
// * Creates a new regular expression token matcher.
// *
// * @param pattern the pattern to match
// * @param ignoreCase the character case ignore flag
// * @param input the input stream to check
// *
// * @throws RegExpException if the regular expression couldn't
// * be created properly
//
public RegExpTokenMatcher(TokenPattern pattern, bool ignoreCase, LookAheadReader input)
{
this.pattern = pattern;
this.regExp = new RegExp(pattern.Pattern, ignoreCase);
this.matcher = regExp.Matcher(input);
}