Example #1
0
        private Token GetNextToken()
        {
            // First, we handle the whitespace.

            // We first make sure the initial whitespace is eaten if it exists:
            if (IsFirstToken)
                EatWhitespace();
            // Now save this whitespace and clear the buffer, we'll check for more whitespace after the token.
            string preWhitespace = whitespace.ToString();
            whitespace.Clear();

            // Get the token
            Token result = null;
            switch (currentChar)
            {
                case '.':
                    Read();
                    if (currentChar == '.')
                    {
                        Read();
                        result = new Token(Token.Type.VARGS, new Location(file, line, col - 2, line, col));
                    }
                    else result = new Token(Token.Type.DOT, new Location(file, line, col - 1, line, col));
                    break;
                case ',': Read(); result = new Token(Token.Type.COMMA, new Location(file, line, col - 1, line, col)); break;
                case ':': Read(); result = new Token(Token.Type.COLON, new Location(file, line, col - 1, line, col)); break;

                case '(': Read(); result = new Token(Token.Type.OPEN_BRACE, new Location(file, line, col - 1, line, col)); break;
                case ')': Read(); result = new Token(Token.Type.CLOSE_BRACE, new Location(file, line, col - 1, line, col)); break;
                case '[': Read(); result = new Token(Token.Type.OPEN_SQUARE_BRACE, new Location(file, line, col - 1, line, col)); break;
                case ']': Read(); result = new Token(Token.Type.CLOSE_SQUARE_BRACE, new Location(file, line, col - 1, line, col)); break;
                case '{': Read(); result = new Token(Token.Type.OPEN_CURLY_BRACE, new Location(file, line, col - 1, line, col)); break;
                case '}': Read(); result = new Token(Token.Type.CLOSE_CURLY_BRACE, new Location(file, line, col - 1, line, col)); break;

                case '`': ReadLineComment(); result = null; break;

                case '~': case '!': case '@': case '#':
                case '%': case '^': case '&': case '*':
                case '-': case '=': case '+': case '\\':
                case '|': case '<': case '>': case '/':
                case '?': case ';': result = ReadOperator(); break;

                case '"':  result = ReadString(); break;

                case '0': case '1': case '2': case '3': case '4':
                case '5': case '6': case '7': case '8': case '9':
                    result = ReadNumber(); break;

                case '$': result = ReadParamIndex(); break;
                case '\'':
                    var loc = new Location(file, line, col);
                    Read();
                    var sym = ReadOthers();
                    if (sym.type == Token.Type.WILDCARD)
                        log.Error(sym.location, "The wildcard character cannot be used as a symbol literal.");
                    else if (sym.type != Token.Type.IDENTIFIER)
                        log.Error(sym.location, "Keyword '{0}' cannot be used as a symbol literal.", sym.image);
                    result = new Token(Token.Type.SYMBOL, loc, sym.image);
                    break;

                default:
                    if (currentChar == '_' || char.IsLetter((char)currentChar))
                        result = ReadOthers();
                    else
                    {
                        // TODO error, skip the character in attempt to recover.
                        log.Error(new Location(file, line, col),
                            "Unexpected character '{0}'. This character should only exist in a string, are you missing quotes?",
                            char.ConvertFromUtf32(currentChar));
                        Read();
                        return GetNextToken();
                    }
                    break;
            }

            EatWhitespaceSansNewline();
            if (result != null)
            {
            // Add the whitespace to the token.
                result.PreWhitespace = preWhitespace;
                result.PostWhitespace = whitespace.ToString();
            }
            // Once the token has its whitespace, make sure we've handled newlines now.
            // If the tokens are on the same line, this does nothing so that's good.
            EatWhitespace();

            // That's it!
            return result;
        }
Example #2
0
 /// <summary>
 /// Checks if the current token is of the given type.
 /// If it is, the stream advances and this method returns true.
 /// Otherwise a message is added to the error list if one was given, then this method returns false.
 /// </summary>
 /// <param name="type"></param>
 /// <param name="failMessage"></param>
 /// <returns></returns>
 internal bool Expect(Token.Type type, string failMessage)
 {
     if (failMessage == null)
         throw new ArgumentNullException("failMessage");
     if (!IsOver && Current.type == type)
     {
         Advance();
         return true;
     }
     // We make this optional because the parser could provide much more detailed information
     // on its own after some thought, we don't HAVE to expect that error message up front.
     if (failMessage != null)
         LogError(failMessage);
     return false;
 }