Example #1
0
        /// <summary>
        /// Gets the next token from the input.
        /// </summary>
        /// <param name="token">Current token.</param>
        /// <returns>The next token from the input.</returns>
        internal Token Next(Token token)
        {
            // do we already know the next token?
            if (token != null && token.Link != null)
                return token.Link;

            // start scanning at the position of the last token
            var pos = token != null ? token.Position.Offset + token.Position.Length : 0;

            // scan for the next token
            Token next = null;
            while (true) {
                // skip leading whitespace
                var ch = GetChar(pos);
                while (char.IsWhiteSpace(ch))
                    ch = GetChar(++pos);

                // are we at the end of the input?
                if (pos >= _input.Length) {
                    next = MakeToken(Tokens.End);
                    next.Position = new Position(pos, 0);
                    if (token != null)
                        token.Link = next;
                    break;
                }

                // deal with the various input characters
                var start = pos;
                switch (ch) {
                // character and string literals
                case '\'':
                    next = ScanCharacter(ref pos);
                    break;

                case '"':
                    next = ScanString(ref pos);
                    break;

                // punctuation
                case '.':
                    pos++;
                    next = MakeToken(Tokens.Dot);
                    break;

                case ',':
                    pos++;
                    next = MakeToken(Tokens.Comma);
                    break;

                case ':':
                    pos++;
                    next = MakeToken(Tokens.Colon);
                    break;

                case ';':
                    pos++;
                    next = MakeToken(Tokens.Semi);
                    break;

                case '(':
                    pos++;
                    next = MakeToken(Tokens.LeftParen);
                    break;

                case ')':
                    pos++;
                    next = MakeToken(Tokens.RightParen);
                    break;

                case '{':
                    pos++;
                    next = MakeToken(Tokens.LeftBrace);
                    break;

                case '}':
                    pos++;
                    next = MakeToken(Tokens.RightBrace);
                    break;

                case '[':
                    pos++;
                    next = MakeToken(Tokens.LeftBracket);
                    break;

                case ']':
                    pos++;
                    next = MakeToken(Tokens.RightBracket);
                    break;

                // operators
                case '=':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.DoubleEquals);
                    } else if (ch == '>') {
                        pos++;
                        next = MakeToken(Tokens.EqualsRightChevron);
                    } else {
                        next = MakeToken(Tokens.Equals);
                    }
                    break;

                case '+':
                    ch = GetChar(++pos);
                    if (ch == '+') {
                        pos++;
                        next = MakeToken(Tokens.DoublePlus);
                    } else if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.PlusEquals);
                    } else {
                        next = MakeToken(Tokens.Plus);
                    }
                    break;

                case '-':
                    ch = GetChar(++pos);
                    if (ch == '-') {
                        pos++;
                        next = MakeToken(Tokens.DoubleMinus);
                    } else if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.MinusEquals);
                    } else {
                        next = MakeToken(Tokens.Minus);
                    }
                    break;

                case '*':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.AsteriskEquals);
                    } else {
                        next = MakeToken(Tokens.Asterisk);
                    }
                    break;

                case '/':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.SlashEquals);
                    } else {
                        next = MakeToken(Tokens.Slash);
                    }
                    break;

                case '%':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.PercentEquals);
                    } else {
                        next = MakeToken(Tokens.Percent);
                    }
                    break;

                case '!':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.ExclamationEquals);
                    } else {
                        next = MakeToken(Tokens.Exclamation);
                    }
                    break;

                case '?':
                    ch = GetChar(++pos);
                    if (ch == '?') {
                        pos++;
                        next = MakeToken(Tokens.DoubleQuestion);
                    } else {
                        next = MakeToken(Tokens.Question);
                    }
                    break;

                case '~':
                    next = MakeToken(Tokens.Tilde);
                    break;

                case '&':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.AmpersandEquals);
                    } else if (ch == '&') {
                        pos++;
                        next = MakeToken(Tokens.DoubleAmpersand);
                    } else {
                        next = MakeToken(Tokens.Ampersand);
                    }
                    break;

                case '|':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.BarEquals);
                    } else if (ch == '|') {
                        pos++;
                        next = MakeToken(Tokens.DoubleBar);
                    } else {
                        next = MakeToken(Tokens.Bar);
                    }
                    break;

                case '^':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.CaretEquals);
                    } else {
                        next = MakeToken(Tokens.Caret);
                    }
                    break;

                case '<':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.LeftChevronEquals);
                    } else if (ch == '<') {
                        ch = GetChar(++pos);
                        if (ch == '=') {
                            pos++;
                            next = MakeToken(Tokens.DoubleLeftChevronEquals);
                        } else {
                            next = MakeToken(Tokens.DoubleLeftChevron);
                        }
                    } else {
                        next = MakeToken(Tokens.LeftChevron);
                    }
                    break;

                case '>':
                    ch = GetChar(++pos);
                    if (ch == '=') {
                        pos++;
                        next = MakeToken(Tokens.RightChevronEquals);
                    } else if (ch == '>') {
                        ch = GetChar(++pos);
                        if (ch == '=') {
                            pos++;
                            next = MakeToken(Tokens.DoubleRightChevronEquals);
                        } else {
                            next = MakeToken(Tokens.DoubleRightChevron);
                        }
                    } else {
                        next = MakeToken(Tokens.RightChevron);
                    }
                    break;

                default:
                    if (IsIdentifierStart(ch) || ch == '\\')
                        next = ScanIdentifier(ref pos);
                    else if (IsDecimalDigit(ch))
                        next = ScanNumber(ref pos);
                    else {
                        Error(new Position(pos, 1), Messages.UnexpectedInput(ch));
                        pos++;
                        continue;
                    }
                    break;
                }

                // set position and attach to preceding token
                next.Position = new Position(start, pos - start);
                if (token != null)
                    token.Link = next;

                break;
            }

            return next;
        }
Example #2
0
 /// <summary>
 /// Place all tokens from the given token and up to and including the end on the free-list, making them ready for re-use.
 /// </summary>
 /// <param name="token">First token to free</param>
 /// <param name="end">Last token to free</param>
 private void Free(Token token, Token end)
 {
     while (token != end) {
         var next = token.Link;
         Free(token);
         token = next;
     }
 }
Example #3
0
        /// <summary>
        /// Make a new token.
        /// </summary>
        /// <param name="kind">Kind of token to make.</param>
        /// <param name="value">Value to associate with the token.</param>
        private Token MakeToken(Tokens kind, object value = null)
        {
            // re-use a token off the free-list, or allocate a new one
            Token token;
            if (_freeList != null) {
                token = _freeList;
                _freeList = _freeList.Link;
            } else {
                token = new Token();
            }

            // initialise it
            token.Kind = kind;
            token.Value = value;
            token.Link = null;

            return token;
        }
Example #4
0
 /// <summary>
 /// Place the given token on the free-list, making it ready for re-use.
 /// </summary>
 /// <param name="token">Token to free.</param>
 private void Free(Token token)
 {
     token.Link = _freeList;
     _freeList = token;
 }