Exemplo n.º 1
0
        private int SkipWhitespaces(LexerStringReader reader)
        {
            var whitespaceRuntimeInfo = new LexerRuntimeInfo(reader);
            var whitespaceResult      = _whitespaceTokenizer.Tokenize(ref whitespaceRuntimeInfo);

            return(whitespaceResult.Token?.Length ?? 0);
        }
Exemplo n.º 2
0
        public LexerResult Tokenize(string query)
        {
            if (string.IsNullOrWhiteSpace(query))
            {
                throw new ArgumentException();
            }

            var finalResult = new LexerResult();
            var reader      = new LexerStringReader(query, 0);

            while (reader.IsEndOfQuery() == false)
            {
                reader.MoveBy(SkipWhitespaces(reader));

                var tokenFound = false;

                foreach (var tokenizer in _languageConfig.TokenizersSource.OrderedTokenizers)
                {
                    var runtimeInfo = new LexerRuntimeInfo(reader);
                    var result      = tokenizer.Tokenize(ref runtimeInfo);
                    if (result.Success || result.IsPartiallySuccessful)
                    {
                        finalResult.Tokens.Add(result.Token);
                        reader.ForceMoveBy(result.Token.Length);
                        tokenFound = true;
                        break;
                    }
                }

                if (tokenFound == false)
                {
                    var startPosition = reader.CurrentPosition;
                    var word          = reader.ReadTillEndOfWord();
                    var unknownToken  = new UnknownToken(startPosition, word.Length);
                    finalResult.Tokens.Add(unknownToken);
                }
            }

            return(finalResult);
        }