private int SkipWhitespaces(LexerStringReader reader)
{
var whitespaceRuntimeInfo = new LexerRuntimeInfo(reader);
var whitespaceResult = _whitespaceTokenizer.Tokenize(ref whitespaceRuntimeInfo);
return(whitespaceResult.Token?.Length ?? 0);
}
public LexerResult Tokenize(string query)
{
if (string.IsNullOrWhiteSpace(query))
{
throw new ArgumentException();
}
var finalResult = new LexerResult();
var reader = new LexerStringReader(query, 0);
while (reader.IsEndOfQuery() == false)
{
reader.MoveBy(SkipWhitespaces(reader));
var tokenFound = false;
foreach (var tokenizer in _languageConfig.TokenizersSource.OrderedTokenizers)
{
var runtimeInfo = new LexerRuntimeInfo(reader);
var result = tokenizer.Tokenize(ref runtimeInfo);
if (result.Success || result.IsPartiallySuccessful)
{
finalResult.Tokens.Add(result.Token);
reader.ForceMoveBy(result.Token.Length);
tokenFound = true;
break;
}
}
if (tokenFound == false)
{
var startPosition = reader.CurrentPosition;
var word = reader.ReadTillEndOfWord();
var unknownToken = new UnknownToken(startPosition, word.Length);
finalResult.Tokens.Add(unknownToken);
}
}
return(finalResult);
}
