public RegexToken(string query, QueryTokenType type)
{
this.query = query;
this.type = type;
tokens.Add(this);
}
private QueryTokenType SkipString(char quoteChar)
{
QueryTokenType type = QueryTokenType.String;
char ch = this.NextChar();
while (true)
{
if (ch == quoteChar)
{
_ = this.NextChar();
break;
}
else if (ch == '\\')
{
type = QueryTokenType.EncodedString;
ch = this.NextChar();
switch (ch)
{
case '\"':
case '\'':
case '\\':
case '/':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
ch = this.NextChar();
break;
case 'u':
if (!Tokenizer.IsHexDigit(this.NextChar()) ||
!Tokenizer.IsHexDigit(this.NextChar()) ||
!Tokenizer.IsHexDigit(this.NextChar()) ||
!Tokenizer.IsHexDigit(this.NextChar()))
{
return(QueryTokenType.Error);
}
ch = this.NextChar();
break;
default:
return(QueryTokenType.Error);
}
}
else if (ch < ' ')
{
return(QueryTokenType.Error);
}
else
{
ch = this.NextChar();
}
}
return(type);
}
public bool Accept(QueryTokenType t)
{
if (Current.type == t)
{
Next();
return(true);
}
return(false);
}
private static IBinaryQueryOperator CreateOperator(QueryTokenType tokenType, IQueryPart leftPart, IQueryPart rightPart, int tolerance)
{
return(tokenType switch
{
QueryTokenType.AndOperator => new AndQueryOperator(leftPart, rightPart),
QueryTokenType.OrOperator => new OrQueryOperator(leftPart, rightPart),
QueryTokenType.NearOperator => new NearQueryOperator(leftPart, rightPart, tolerance),
QueryTokenType.PrecedingNearOperator => new PrecedingNearQueryOperator(leftPart, rightPart, tolerance),
QueryTokenType.PrecedingOperator => new PrecedingQueryOperator(leftPart, rightPart),
_ => throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType),
});
private QueryToken ReadToken()
{
char ch = this.SkipSpaces(this.curChar);
QueryTokenType type = QueryTokenType.Error;
int start = this.pos;
switch (ch)
{
case '\0':
type = QueryTokenType.None;
break;
case '\"':
case '\'':
type = this.SkipString(ch);
if (type != QueryTokenType.Error)
{
// Don't include quotes in the string token
return(new QueryToken(type, start + 1, this.pos - start - 2));
}
break;
case '-':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
type = this.SkipNumber(ch);
break;
default:
if (QueryTokenizer.IsIdentifierStart(ch))
{
type = this.SkipIdentifier();
}
break;
}
return(new QueryToken(type, start, this.pos - start));
}
private static OperatorPrecedence TokenPrecedence(QueryTokenType tokenType)
{
switch (tokenType)
{
case QueryTokenType.AndOperator:
return(OperatorPrecedence.And);
case QueryTokenType.OrOperator:
return(OperatorPrecedence.Or);
case QueryTokenType.PrecedingNearOperator:
case QueryTokenType.NearOperator:
case QueryTokenType.PrecedingOperator:
return(OperatorPrecedence.Positional);
default:
throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType);
}
}
/// <summary>
/// Gets tokens from the token stream until the specified token type is encountered. The terminating
/// token will be consumed but not returned. If the terminating token is not encountered before the
/// end of the token stream, a <see cref="QueryParserException"/> will be thrown.
/// </summary>
/// <param name="terminatingToken">The terminating token.</param>
/// <returns>The tokens that appear before the terminating token.</returns>
public IEnumerable <QueryToken> GetTokensUntil(QueryTokenType terminatingToken)
{
var matchedTerminator = false;
while (this.enumerator.MoveNext())
{
if (this.enumerator.Current.TokenType == terminatingToken)
{
matchedTerminator = true;
break;
}
yield return(this.enumerator.Current);
}
if (!matchedTerminator)
{
throw new QueryParserException(ExceptionMessages.ExpectedToken, terminatingToken);
}
}
private static IBinaryQueryOperator CreateOperator(QueryTokenType tokenType, IQueryPart leftPart, IQueryPart rightPart, int tolerance)
{
switch (tokenType)
{
case QueryTokenType.AndOperator:
return(new AndQueryOperator(leftPart, rightPart));
case QueryTokenType.OrOperator:
return(new OrQueryOperator(leftPart, rightPart));
case QueryTokenType.NearOperator:
return(new NearQueryOperator(leftPart, rightPart, tolerance));
case QueryTokenType.PrecedingNearOperator:
return(new PrecedingNearQueryOperator(leftPart, rightPart, tolerance));
case QueryTokenType.PrecedingOperator:
return(new PrecedingQueryOperator(leftPart, rightPart));
default:
throw new QueryParserException(ExceptionMessages.UnexpectedOperatorInternal, tokenType);
}
}
private static IBinaryQueryOperator CombineParts(IQueryPart?existingPart, IQueryPart newPart, QueryTokenType operatorType, int tolerance)
{
if (existingPart == null)
{
throw new QueryParserException(ExceptionMessages.UnexpectedOperator, operatorType);
}
var existingBinaryOperator = existingPart as IBinaryQueryOperator;
if (existingBinaryOperator != null)
{
if (existingBinaryOperator.Precedence >= TokenPrecedence(operatorType))
{
existingBinaryOperator.Right = CreateOperator(operatorType, existingBinaryOperator.Right, newPart, tolerance);
return(existingBinaryOperator);
}
return(CreateOperator(operatorType, existingBinaryOperator, newPart, tolerance));
}
return(CreateOperator(operatorType, existingPart, newPart, tolerance));
}
private QueryToken(string tokenText, QueryTokenType tokenType, int tolerance)
{
this.TokenText = tokenText;
this.TokenType = tokenType;
this.Tolerance = tolerance;
}
/// <summary> /// Creates a new <see cref="QueryToken"/> instance representing a query operator that has additional positional constraints. /// </summary> /// <param name="operatorType"> /// The type of operator the token should represent. /// </param> /// <param name="tolerance"> /// The number of tokens to use as the tolerance for the operator. /// </param> public static QueryToken ForOperatorWithTolerance(QueryTokenType operatorType, int tolerance) => new QueryToken(string.Empty, operatorType, tolerance == 0 ? 5 : tolerance);
/// <summary> /// Creates a new <see cref="QueryToken"/> instance representing a query operator. /// </summary> /// <param name="operatorType"> /// The type of operator the token should represent. /// </param> public static QueryToken ForOperator(QueryTokenType operatorType) => new QueryToken(string.Empty, operatorType, 0);
public static QueryToken ForOperator(QueryTokenType operatorType) => new QueryToken(null, operatorType, 0);
public QueryToken(QueryTokenType type, int start, int length)
{
this.Type = type;
this.Start = start;
this.Length = length;
}