//* // * Returns the token pattern with the specified id. Only // * token patterns handled by this matcher can be returned. // * // * @param id the token pattern id // * // * @return the token pattern found, or // * null if not found // public TokenPattern GetPattern(int id) { TokenPattern pattern = default(TokenPattern); for (int i = 0; i <= patterns.Count - 1; i++) { pattern = (TokenPattern)patterns[i]; if (pattern.Id == id) { return(pattern); } } return(null); }
//* // * Creates a new token. // * // * @param pattern the token pattern // * @param image the token image (i.e. characters) // * @param line the line number of the first character // * @param col the column number of the first character // public Token(TokenPattern pattern, string image, int line, int col) { this.m_pattern = pattern; this.m_image = image; this.m_startLine = line; this.m_startColumn = col; this.m_endLine = line; this.m_endColumn = col + image.Length - 1; int pos = 0; while (image.IndexOf(Convert.ToChar(10), pos) >= 0) { pos = image.IndexOf(Convert.ToChar(10), pos) + 1; this.m_endLine += 1; m_endColumn = image.Length - pos; } }
//* // * Returns a description of the token pattern with the // * specified id. // * // * @param id the token pattern id // * // * @return the token pattern description, or // * null if not present // public string GetPatternDescription(int id) { TokenPattern pattern = default(TokenPattern); RegExpTokenMatcher re = default(RegExpTokenMatcher); pattern = stringMatcher.GetPattern(id); if (pattern != null) { return(pattern.ToShortString()); } for (int i = 0; i <= regexpMatchers.Count - 1; i++) { re = (RegExpTokenMatcher)regexpMatchers[i]; if (re.GetPattern().Id == id) { return(re.GetPattern().ToShortString()); } } return(null); }
//* // * Adds a new token pattern to the tokenizer. The pattern will be // * added last in the list, choosing a previous token pattern in // * case two matches the same string. // * // * @param pattern the pattern to add // * // * @throws ParserCreationException if the pattern couldn't be // * added to the tokenizer // public void AddPattern(TokenPattern pattern) { switch (pattern.Type) { case TokenPattern.PatternType.STRING: stringMatcher.AddPattern(pattern); break; case TokenPattern.PatternType.REGEXP: try { regexpMatchers.Add(new RegExpTokenMatcher(pattern, ignoreCase, input)); } catch (RegExpException e) { throw new ParserCreationException(ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "regular expression contains error(s): " + e.Message); } break; default: throw new ParserCreationException(ParserCreationException.ErrorType.INVALID_TOKEN, pattern.Name, "pattern type " + pattern.Type + " is undefined"); } }
///<summary>Initializes the tokenizer by creating all the token ///patterns.</summary> /// ///<exception cref='ParserCreationException'>if the tokenizer ///couldn't be initialized correctly</exception> private void CreatePatterns() { TokenPattern pattern = default(TokenPattern); CustomTokenPattern customPattern = default(CustomTokenPattern); pattern = new TokenPattern((int)ExpressionConstants.ADD, "ADD", TokenPattern.PatternType.STRING, "+"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.SUB, "SUB", TokenPattern.PatternType.STRING, "-"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.MUL, "MUL", TokenPattern.PatternType.STRING, "*"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.DIV, "DIV", TokenPattern.PatternType.STRING, "/"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.POWER, "POWER", TokenPattern.PatternType.STRING, "^"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.MOD, "MOD", TokenPattern.PatternType.STRING, "%"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.LEFT_PAREN, "LEFT_PAREN", TokenPattern.PatternType.STRING, "("); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.RIGHT_PAREN, "RIGHT_PAREN", TokenPattern.PatternType.STRING, ")"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.LEFT_BRACE, "LEFT_BRACE", TokenPattern.PatternType.STRING, "["); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.RIGHT_BRACE, "RIGHT_BRACE", TokenPattern.PatternType.STRING, "]"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.EQ, "EQ", TokenPattern.PatternType.STRING, "="); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.LT, "LT", TokenPattern.PatternType.STRING, "<"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.GT, "GT", TokenPattern.PatternType.STRING, ">"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.LTE, "LTE", TokenPattern.PatternType.STRING, "<="); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.GTE, "GTE", TokenPattern.PatternType.STRING, ">="); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.NE, "NE", TokenPattern.PatternType.STRING, "<>"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.AND, "AND", TokenPattern.PatternType.STRING, "AND"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.OR, "OR", TokenPattern.PatternType.STRING, "OR"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.XOR, "XOR", TokenPattern.PatternType.STRING, "XOR"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.NOT, "NOT", TokenPattern.PatternType.STRING, "NOT"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.IN, "IN", TokenPattern.PatternType.STRING, "in"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.DOT, "DOT", TokenPattern.PatternType.STRING, "."); AddPattern(pattern); customPattern = new ArgumentSeparatorPattern(); customPattern.Initialize((int)ExpressionConstants.ARGUMENT_SEPARATOR, "ARGUMENT_SEPARATOR", TokenPattern.PatternType.STRING, ",", MyContext); AddPattern(customPattern); pattern = new TokenPattern((int)ExpressionConstants.ARRAY_BRACES, "ARRAY_BRACES", TokenPattern.PatternType.STRING, "[]"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.LEFT_SHIFT, "LEFT_SHIFT", TokenPattern.PatternType.STRING, "<<"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.RIGHT_SHIFT, "RIGHT_SHIFT", TokenPattern.PatternType.STRING, ">>"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.WHITESPACE, "WHITESPACE", TokenPattern.PatternType.REGEXP, "\\s+"); pattern.Ignore = true; AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.INTEGER, "INTEGER", TokenPattern.PatternType.REGEXP, "\\d+(u|l|ul|lu)?"); AddPattern(pattern); customPattern = new RealPattern(); customPattern.Initialize((int)ExpressionConstants.REAL, "REAL", TokenPattern.PatternType.REGEXP, "\\d*\\.\\d+([e][+-]\\d{1,3})?f?", MyContext); AddPattern(customPattern); pattern = new TokenPattern((int)ExpressionConstants.STRING_LITERAL, "STRING_LITERAL", TokenPattern.PatternType.REGEXP, "\"([^\"\\r\\n\\\\]|\\\\u[0-9a-f]{4}|\\\\[\\\\\"'trn])*\""); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.CHAR_LITERAL, "CHAR_LITERAL", TokenPattern.PatternType.REGEXP, "'([^'\\r\\n\\\\]|\\\\u[0-9a-f]{4}|\\\\[\\\\\"'trn])'"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.TRUE, "TRUE", TokenPattern.PatternType.STRING, "True"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.FALSE, "FALSE", TokenPattern.PatternType.STRING, "False"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.IDENTIFIER, "IDENTIFIER", TokenPattern.PatternType.REGEXP, "[a-z_]\\w*"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.HEX_LITERAL, "HEX_LITERAL", TokenPattern.PatternType.REGEXP, "0x[0-9a-f]+(u|l|ul|lu)?"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.NULL_LITERAL, "NULL_LITERAL", TokenPattern.PatternType.STRING, "null"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.TIMESPAN, "TIMESPAN", TokenPattern.PatternType.REGEXP, "##(\\d+\\.)?\\d{2}:\\d{2}(:\\d{2}(\\.\\d{1,7})?)?#"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.DATETIME, "DATETIME", TokenPattern.PatternType.REGEXP, "#[^#]+#"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.IF, "IF", TokenPattern.PatternType.STRING, "if"); AddPattern(pattern); pattern = new TokenPattern((int)ExpressionConstants.CAST, "CAST", TokenPattern.PatternType.STRING, "cast"); AddPattern(pattern); }
//* // * Checks if the token pattern matches the input stream. This // * method will also reset all flags in this matcher. // * // * @param input the input stream to match // * // * @return true if a match was found, or // * false otherwise // * // * @throws IOException if an I/O error occurred // public bool Match(LookAheadReader input) { this.Reset(); m_match = (TokenPattern)start.MatchFrom(input, 0, ignoreCase); return(m_match != null); }
//* // * Adds a string token pattern to this matcher. // * // * @param pattern the pattern to add // public void AddPattern(TokenPattern pattern) { patterns.Add(pattern); start.AddMatch(pattern.Pattern, ignoreCase, pattern); }
//* // * Resets the matcher state. This will clear the results of // * the last match. // public void Reset() { m_match = null; }
//* // * Creates a new regular expression token matcher. // * // * @param pattern the pattern to match // * @param ignoreCase the character case ignore flag // * @param input the input stream to check // * // * @throws RegExpException if the regular expression couldn't // * be created properly // public RegExpTokenMatcher(TokenPattern pattern, bool ignoreCase, LookAheadReader input) { this.pattern = pattern; this.regExp = new RegExp(pattern.Pattern, ignoreCase); this.matcher = regExp.Matcher(input); }