public RegexToken(string query, QueryTokenType type) { this.query = query; this.type = type; tokens.Add(this); }
private QueryTokenType SkipString(char quoteChar) { QueryTokenType type = QueryTokenType.String; char ch = this.NextChar(); while (true) { if (ch == quoteChar) { _ = this.NextChar(); break; } else if (ch == '\\') { type = QueryTokenType.EncodedString; ch = this.NextChar(); switch (ch) { case '\"': case '\'': case '\\': case '/': case 'b': case 'f': case 'n': case 'r': case 't': ch = this.NextChar(); break; case 'u': if (!Tokenizer.IsHexDigit(this.NextChar()) || !Tokenizer.IsHexDigit(this.NextChar()) || !Tokenizer.IsHexDigit(this.NextChar()) || !Tokenizer.IsHexDigit(this.NextChar())) { return(QueryTokenType.Error); } ch = this.NextChar(); break; default: return(QueryTokenType.Error); } } else if (ch < ' ') { return(QueryTokenType.Error); } else { ch = this.NextChar(); } } return(type); }
public bool Accept(QueryTokenType t) { if (Current.type == t) { Next(); return(true); } return(false); }
private static IBinaryQueryOperator CreateOperator(QueryTokenType tokenType, IQueryPart leftPart, IQueryPart rightPart, int tolerance) { return(tokenType switch { QueryTokenType.AndOperator => new AndQueryOperator(leftPart, rightPart), QueryTokenType.OrOperator => new OrQueryOperator(leftPart, rightPart), QueryTokenType.NearOperator => new NearQueryOperator(leftPart, rightPart, tolerance), QueryTokenType.PrecedingNearOperator => new PrecedingNearQueryOperator(leftPart, rightPart, tolerance), QueryTokenType.PrecedingOperator => new PrecedingQueryOperator(leftPart, rightPart), _ => throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType), });
private QueryToken ReadToken() { char ch = this.SkipSpaces(this.curChar); QueryTokenType type = QueryTokenType.Error; int start = this.pos; switch (ch) { case '\0': type = QueryTokenType.None; break; case '\"': case '\'': type = this.SkipString(ch); if (type != QueryTokenType.Error) { // Don't include quotes in the string token return(new QueryToken(type, start + 1, this.pos - start - 2)); } break; case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': type = this.SkipNumber(ch); break; default: if (QueryTokenizer.IsIdentifierStart(ch)) { type = this.SkipIdentifier(); } break; } return(new QueryToken(type, start, this.pos - start)); }
private static OperatorPrecedence TokenPrecedence(QueryTokenType tokenType) { switch (tokenType) { case QueryTokenType.AndOperator: return(OperatorPrecedence.And); case QueryTokenType.OrOperator: return(OperatorPrecedence.Or); case QueryTokenType.PrecedingNearOperator: case QueryTokenType.NearOperator: case QueryTokenType.PrecedingOperator: return(OperatorPrecedence.Positional); default: throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType); } }
/// <summary> /// Gets tokens from the token stream until the specified token type is encountered. The terminating /// token will be consumed but not returned. If the terminating token is not encountered before the /// end of the token stream, a <see cref="QueryParserException"/> will be thrown. /// </summary> /// <param name="terminatingToken">The terminating token.</param> /// <returns>The tokens that appear before the terminating token.</returns> public IEnumerable <QueryToken> GetTokensUntil(QueryTokenType terminatingToken) { var matchedTerminator = false; while (this.enumerator.MoveNext()) { if (this.enumerator.Current.TokenType == terminatingToken) { matchedTerminator = true; break; } yield return(this.enumerator.Current); } if (!matchedTerminator) { throw new QueryParserException(ExceptionMessages.ExpectedToken, terminatingToken); } }
private static IBinaryQueryOperator CreateOperator(QueryTokenType tokenType, IQueryPart leftPart, IQueryPart rightPart, int tolerance) { switch (tokenType) { case QueryTokenType.AndOperator: return(new AndQueryOperator(leftPart, rightPart)); case QueryTokenType.OrOperator: return(new OrQueryOperator(leftPart, rightPart)); case QueryTokenType.NearOperator: return(new NearQueryOperator(leftPart, rightPart, tolerance)); case QueryTokenType.PrecedingNearOperator: return(new PrecedingNearQueryOperator(leftPart, rightPart, tolerance)); case QueryTokenType.PrecedingOperator: return(new PrecedingQueryOperator(leftPart, rightPart)); default: throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType); } }
private static IBinaryQueryOperator CombineParts(IQueryPart?existingPart, IQueryPart newPart, QueryTokenType operatorType, int tolerance) { if (existingPart == null) { throw new QueryParserException(ExceptionMessages.UnexpectedOperator, operatorType); } var existingBinaryOperator = existingPart as IBinaryQueryOperator; if (existingBinaryOperator != null) { if (existingBinaryOperator.Precedence >= TokenPrecedence(operatorType)) { existingBinaryOperator.Right = CreateOperator(operatorType, existingBinaryOperator.Right, newPart, tolerance); return(existingBinaryOperator); } return(CreateOperator(operatorType, existingBinaryOperator, newPart, tolerance)); } return(CreateOperator(operatorType, existingPart, newPart, tolerance)); }
private QueryToken(string tokenText, QueryTokenType tokenType, int tolerance) { this.TokenText = tokenText; this.TokenType = tokenType; this.Tolerance = tolerance; }
/// <summary> /// Creates a new <see cref="QueryToken"/> instance representing a query operator that has additional positional constraints. /// </summary> /// <param name="operatorType"> /// The type of operator the token should represent. /// </param> /// <param name="tolerance"> /// The number of tokens to use as the tolerance for the operator. /// </param> public static QueryToken ForOperatorWithTolerance(QueryTokenType operatorType, int tolerance) => new QueryToken(string.Empty, operatorType, tolerance == 0 ? 5 : tolerance);
/// <summary> /// Creates a new <see cref="QueryToken"/> instance representing a query operator. /// </summary> /// <param name="operatorType"> /// The type of operator the token should represent. /// </param> public static QueryToken ForOperator(QueryTokenType operatorType) => new QueryToken(string.Empty, operatorType, 0);
public static QueryToken ForOperator(QueryTokenType operatorType) => new QueryToken(null, operatorType, 0);
public QueryToken(QueryTokenType type, int start, int length) { this.Type = type; this.Start = start; this.Length = length; }