private static IWordQueryPart CreateWordPart(QueryToken queryToken, ITokenizer wordTokenizer) { var tokenText = queryToken.TokenText.AsSpan(); var hasWildcard = tokenText.Length > 0 && tokenText[tokenText.Length - 1] == '*'; if (hasWildcard) { tokenText = tokenText.Slice(0, tokenText.Length - 1); } var tokenizedWord = wordTokenizer.Process(tokenText).Single(); return(hasWildcard ? (IWordQueryPart) new StartsWithWordQueryPart(tokenizedWord.Value) : new ExactWordQueryPart(tokenizedWord.Value)); }
private static IEnumerable <IWordQueryPart> CreateWordParts(QueryToken queryToken, ITokenizer wordTokenizer) { if (queryToken.TokenType != QueryTokenType.Text) { throw new QueryParserException(ExceptionMessages.ExpectedTextToken, queryToken.TokenType); } var tokenText = queryToken.TokenText.AsSpan(); var hasWildcard = tokenText.Length > 0 && tokenText[tokenText.Length - 1] == '*'; if (hasWildcard) { tokenText = tokenText.Slice(0, tokenText.Length - 1); } return(wordTokenizer.Process(tokenText) .Select(tokenizedWord => hasWildcard ? (IWordQueryPart) new StartsWithWordQueryPart(tokenizedWord.Value) : new ExactWordQueryPart(tokenizedWord.Value))); }
internal override IEnumerable <Token> Tokenize(ITokenizer tokenizer, TItem item) { return(tokenizer.Process(this.reader(item))); }
internal override async ValueTask <IEnumerable <Token> > TokenizeAsync(ITokenizer tokenizer, TItem item) { return(tokenizer.Process(await this.reader(item).ConfigureAwait(false))); }