private int SkipWhitespaces(LexerStringReader reader) { var whitespaceRuntimeInfo = new LexerRuntimeInfo(reader); var whitespaceResult = _whitespaceTokenizer.Tokenize(ref whitespaceRuntimeInfo); return(whitespaceResult.Token?.Length ?? 0); }
public LexerResult Tokenize(string query) { if (string.IsNullOrWhiteSpace(query)) { throw new ArgumentException(); } var finalResult = new LexerResult(); var reader = new LexerStringReader(query, 0); while (reader.IsEndOfQuery() == false) { reader.MoveBy(SkipWhitespaces(reader)); var tokenFound = false; foreach (var tokenizer in _languageConfig.TokenizersSource.OrderedTokenizers) { var runtimeInfo = new LexerRuntimeInfo(reader); var result = tokenizer.Tokenize(ref runtimeInfo); if (result.Success || result.IsPartiallySuccessful) { finalResult.Tokens.Add(result.Token); reader.ForceMoveBy(result.Token.Length); tokenFound = true; break; } } if (tokenFound == false) { var startPosition = reader.CurrentPosition; var word = reader.ReadTillEndOfWord(); var unknownToken = new UnknownToken(startPosition, word.Length); finalResult.Tokens.Add(unknownToken); } } return(finalResult); }