NextToken() public method

public NextToken ( ) : string
return string
Tokenizer Class Documentation
Example #1
0
        bool IScanner.ScanTokenAndProvideInfoAboutIt(TokenInfo tokenInfo, ref int state)
        {
            tokenizer.BlockComment = state == 1 ? true : false;

            string token = tokenizer.NextToken();

            if (token == null)
            {
                return(false);
            }

            token = token.Trim();
            tokenInfo.StartIndex = tokenizer.StartIndex;
            tokenInfo.EndIndex   = tokenizer.StopIndex;
            tokenInfo.Type       = GetTokenType(token);
            tokenInfo.Color      = GetTokenColor(tokenInfo);
            state = (tokenizer.BlockComment && !token.EndsWith("*/")) ? 1 : 0;
            return(true);
        }