/// <summary> /// Reads the next token from the character /// sequence being tokenized. /// </summary> private void ReadToken() { if (m_wait) { m_wait = false; m_currentIndex = m_nextTokenIndex; return; } if (!m_inIdentifierChain) { m_identifierChain = null; m_identifierChainPredecessor = null; m_identifierChainPredecessorType = SqlTokenType.None; } while (m_currentIndex < m_charsLength && JavaCharacter.isWhitespace(m_chars[m_currentIndex])) { m_currentIndex++; } m_token = string.Empty; m_parameterName = string.Empty; m_parameterNamePrefix = ' '; m_tokenIndex = m_currentIndex; if (m_currentIndex >= m_charsLength) { // character sequence has been exhausted. m_tokenType = SqlTokenType.None; return; } char c = m_chars[m_currentIndex]; bool point = false; bool digit = false; bool exp = false; bool afterexp = false; bool end = false; char cfirst = (char)0; m_wasLastTokenDelimited = false; if (JavaCharacter.isJavaIdentifierStart(c)) { m_tokenType = SqlTokenType.Name; } else if (JavaCharacter.isDigit(c)) { m_tokenType = SqlTokenType.NumberLiteral; digit = true; } else { switch (c) { case '(': { m_token = Token.ValueFor.OPENBRACKET; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case ')': { m_token = Token.ValueFor.CLOSEBRACKET; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case ',': { m_token = Token.ValueFor.COMMA; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case '*': { m_token = Token.ValueFor.MULTIPLY; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case '=': { m_token = Token.ValueFor.EQUALS; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case ';': { m_token = Token.ValueFor.SEMICOLON; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case '+': { m_token = Token.ValueFor.PLUS; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case '%': { m_token = Token.ValueFor.PERCENT; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case '?': { if (!m_acceptParameterMarkers) { throw Tokenizer.UnexpectedToken(c); } m_token = Token.ValueFor.QUESTION; m_tokenType = SqlTokenType.Special; m_currentIndex++; return; } case ':': { if (!m_acceptNamedParameters) { throw UnexpectedToken(c); } if (!(++m_currentIndex < m_charsLength)) { throw Tokenizer.UnexpectedEndOfCommand(); } c = m_chars[m_currentIndex]; if (!JavaCharacter.isJavaIdentifierStart(c)) { throw Tokenizer.InvalidIdentifier(":" + c); } m_tokenType = SqlTokenType.NamedParameter; m_parameterNamePrefix = ':'; break; } case '@': { if (!m_acceptNamedParameters) { throw UnexpectedToken(c); } if (!(++m_currentIndex < m_charsLength)) { throw Tokenizer.UnexpectedEndOfCommand(); } c = m_chars[m_currentIndex]; if (!JavaCharacter.isJavaIdentifierStart(c)) { throw InvalidIdentifier(":" + c); } m_tokenType = SqlTokenType.NamedParameter; m_parameterNamePrefix = '@'; break; } case '"': { m_wasLastTokenDelimited = true; m_tokenType = SqlTokenType.DelimitedIdentifier; m_currentIndex++; m_token = GetAsQuotedString('"'); if (m_currentIndex == m_chars.Length) { if (m_inIdentifierChain) { m_identifierChain.Add(new Token(m_token, m_tokenType)); m_inIdentifierChain = false; m_tokenType = SqlTokenType.IdentifierChain; } return; } c = m_chars[m_currentIndex]; if (c == '.') { ReadIdentifierChain(); } else if (m_inIdentifierChain) { m_identifierChain.Add(new Token(m_token, m_tokenType)); m_inIdentifierChain = false; m_tokenType = SqlTokenType.IdentifierChain; } return; } case '\'': { m_tokenType = SqlTokenType.StringLiteral; m_currentIndex++; m_token = GetAsQuotedString('\''); return; } case '!': case '<': case '>': case '|': case '/': case '-': { cfirst = c; m_tokenType = SqlTokenType.Special; break; } case '.': { m_tokenType = SqlTokenType.DecimalLiteral; point = true; break; } default: { throw UnexpectedToken(c); } } } int start = m_currentIndex++; while (true) { if (m_currentIndex >= m_charsLength) { c = ' '; end = true; switch (m_tokenType) { case SqlTokenType.StringLiteral: case SqlTokenType.DelimitedIdentifier: { throw Tokenizer.UnexpectedEndOfCommand(); } } } else { c = m_chars[m_currentIndex]; } switch (m_tokenType) { case SqlTokenType.NamedParameter: case SqlTokenType.Name: { if (JavaCharacter.isJavaIdentifierPart(c)) { break; } m_token = m_chars.Substring(start, m_currentIndex - start).ToUpper(m_EnglishCulture); if (m_tokenType == SqlTokenType.NamedParameter) { m_parameterName = m_chars.Substring(start, (m_currentIndex - start)); return; } // only for Name, not for NamedParameter if (c == '.') { ReadIdentifierChain(); } else if (m_inIdentifierChain /*&& c!= '.'*/) { m_identifierChain.Add(new Token(m_token, m_tokenType)); m_inIdentifierChain = false; m_tokenType = SqlTokenType.IdentifierChain; } else if (c == '(') { // no-op // potentially a function call, SQL operator, etc. } else { // if in value list then it is a value int type = m_ValueTokens.get(m_token, -1); if (type != -1) { m_tokenType = (SqlTokenType)type; } } return; } case SqlTokenType.DelimitedIdentifier: case SqlTokenType.StringLiteral: { // should never get here break; } case SqlTokenType.Remark: { if (end) { // unfinished remark // TODO: log/trace error condition here m_tokenType = SqlTokenType.None; return; } else if (c == '*') { m_currentIndex++; if (m_currentIndex < m_charsLength && m_chars[m_currentIndex] == '/') { // TODO: eliminate recursion m_currentIndex++; ReadToken(); return; } } break; } case SqlTokenType.RemarkLine: { if (end) { m_tokenType = SqlTokenType.None; return; } else if (c == '\r' || c == '\n') { // TODO: eliminate recursion ReadToken(); return; } break; } case SqlTokenType.Special: { if (c == '/' && cfirst == '/') { m_tokenType = SqlTokenType.RemarkLine; break; } else if (c == '-' && cfirst == '-') { m_tokenType = SqlTokenType.RemarkLine; break; } else if (c == '*' && cfirst == '/') { m_tokenType = SqlTokenType.Remark; break; } else if (c == '>' || c == '=' || c == '|') { break; } m_token = m_chars.Substring(start, m_currentIndex - start); return; } case SqlTokenType.NumberLiteral: case SqlTokenType.FloatLiteral: case SqlTokenType.DecimalLiteral: { if (JavaCharacter.isDigit(c)) { digit = true; } else if (c == '.') { if (point) { throw Tokenizer.UnexpectedToken(c); } m_tokenType = SqlTokenType.DecimalLiteral; point = true; } else if (c == 'E' || c == 'e') { if (exp) { throw UnexpectedToken(c); } m_tokenType = SqlTokenType.FloatLiteral; // first character after exp may be + or - afterexp = true; point = true; exp = true; } else if (c == '-' && afterexp) { afterexp = false; } else if (c == '+' && afterexp) { afterexp = false; } else { //afterexp = false; if (!digit) { if (point && (start == (m_currentIndex - 1))) { m_token = Token.ValueFor.PERIOD; m_tokenType = SqlTokenType.Special; return; } throw UnexpectedToken(c); } m_token = m_chars.Substring(start, (m_currentIndex - start)); return; } break; } } m_currentIndex++; } }
/// <summary> /// Resets this tokenizer with the given character sequence. /// </summary> /// <param name="chars"> /// The new character sequence to be tokenized. /// </param> /// <exception cref="ArgumentNullException"> /// When <c>chars</c> is <c>null</c>. /// </exception> public void Reset(string chars) { if (chars == null) { throw new ArgumentNullException("chars"); } m_chars = chars; m_charsLength = chars.Length; m_currentIndex = 0; m_tokenIndex = 0; m_nextTokenIndex = 0; m_beginIndex = 0; m_tokenType = SqlTokenType.None; m_identifierChainPredecessorType = SqlTokenType.None; m_token = null; m_parameterName = string.Empty; m_parameterNamePrefix = ' '; m_identifierChainPredecessor = null; m_wait = false; m_wasLastTokenDelimited = false; m_inIdentifierChain = false; }
/// <summary> /// Reads an entire identifier chain. /// </summary> private void ReadIdentifierChain() { if (m_identifierChain == null) { m_identifierChain = new GenericTokenList(); } if (m_inIdentifierChain && m_enforceTwoPartIdentifierChain) { throw Tokenizer.IdentiferChainLengthExceeded(); } m_identifierChainPredecessor = m_token; m_identifierChainPredecessorType = m_tokenType; m_inIdentifierChain = true; m_currentIndex++; m_identifierChain.Add(new Token(m_token, m_tokenType)); // TODO: avoid recursion // Also, this has problems when there is whitespace // after the dot; same with NAME ReadToken(); }
/// <summary> /// Retrieves the SQL literal value of the given token value. /// </summary> /// <remarks> /// Note that, as a ref parameter, the tokenType may be /// re-assigned as part of computing the literal value. /// </remarks> /// <param name="tokenType"> /// Type of the token. /// </param> /// <param name="tokenValue"> /// The token value, as a character sequence /// </param> /// <returns> /// the token literal value, as an object representing the SQL literal /// numeric, temporal, boolean or character value. /// </returns> public static object ToLiteralValue(ref SqlTokenType tokenType, string tokenValue) { switch (tokenType) { case SqlTokenType.Null: { return null; } case SqlTokenType.StringLiteral: { if (tokenValue.Length <= 10 && Tokenizer.DateLiteralRegex.IsMatch(tokenValue)) { try { object rval = HsqlDateTime.dateValue(tokenValue); tokenType = SqlTokenType.DateLiteral; return rval; } catch (Exception) { } } else if (tokenValue.Length <= 10 && Tokenizer.TimeLiteralRegex.IsMatch(tokenValue)) { try { object rval = HsqlDateTime.timeValue(tokenValue); tokenType = SqlTokenType.TimeLiteral; return rval; } catch (Exception) { } } else if (Tokenizer.TimestampLiteralRegex.IsMatch(tokenValue)) { try { object rval = HsqlDateTime.timestampValue(tokenValue); tokenType = SqlTokenType.TimestampLiteral; return rval; } catch (Exception) { } } return tokenValue; } case SqlTokenType.BigIntLiteral: { return HsqlValuePool.getLong(JavaLong.parseLong(tokenValue)); } case SqlTokenType.NumberLiteral: { // Returns unsigned values which may later need to be negated. // As a result, Integer.MIN_VALUE or Long.MIN_VALUE are promoted // to a wider type. if (tokenValue.Length < 11) { try { object rval = HsqlValuePool.getInt( JavaInteger.parseInt(tokenValue)); tokenType = SqlTokenType.IntegerLiteral; return rval; } catch (Exception) { } } if (tokenValue.Length < 20) { try { object rval = HsqlValuePool.getLong( JavaLong.parseLong(tokenValue)); tokenType = SqlTokenType.BigIntLiteral; return rval; } catch (Exception) { } } tokenType = SqlTokenType.DecimalLiteral; return new JavaBigDecimal(tokenValue); } case SqlTokenType.FloatLiteral: { double d = JavaSystem.parseDouble(tokenValue); long l = JavaDouble.doubleToLongBits(d); return HsqlValuePool.getDouble(l); } case SqlTokenType.DecimalLiteral: { return new JavaBigDecimal(tokenValue); } case SqlTokenType.BooleanLiteral: { return Token.ValueFor.TRUE.Equals(tokenValue, InvariantCultureIgnoreCase) ? JavaBoolean.TRUE : JavaBoolean.FALSE; } case SqlTokenType.DateLiteral: { return HsqlDateTime.dateValue(tokenValue); } case SqlTokenType.TimeLiteral: { return HsqlDateTime.timeValue(tokenValue); } case SqlTokenType.TimestampLiteral: { return HsqlDateTime.timestampValue(tokenValue); } default: { return tokenValue; } } }