Ejemplo n.º 1
0
        Token ExtractWord()
        {
            StringBuilder sb;
            Token retval;
            sb = new StringBuilder();
            this.ReadAppendWord(sb);
            string word = sb.ToString();

            HappyTokenKind maybeKeyword;
            if (_keywords.TryGetValue(word, out maybeKeyword))
                retval = new Token(_reader.GetHappySourceLocation(), maybeKeyword, word);
            else
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Identifier, word);

            return retval;
        }
Ejemplo n.º 2
0
 Token ExtractStatementToken()
 {
     Token retval;
     this.EatWhite();
     if (_reader.Eof)
         retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput);
     else
     {
         if (Char.IsLetter(_reader.Peek()))
             retval = ExtractWord();
         else if (Char.IsNumber(_reader.Peek(0)) || (_reader.Peek(0) == '-' && Char.IsNumber(_reader.Peek(1))))
             retval = ExtractNumber();
         else
             retval = ExtractOtherStatementToken();
     }
     return retval;
 }
Ejemplo n.º 3
0
        //public void FailIfEof()
        //{
        //    if (_reader.Eof)
        //    {
        //        foreach (Token t in _readAhead)
        //            if (t.HappyTokenKind != HappyTokenKind.EndOfInput)
        //                return;
        //        SourceSpan span = new SourceSpan(_enteredLastStatementGroupAt, _reader.CurrentLocation);
        //        if (_state == LexerState.LexingStatement)
        //            _errorCollector.UnexpectedEndOfInputWhileParsingSection(_reader.GetHappySourceLocation());
        //        else
        //            _errorCollector.UnexpectedEndOfInputInComment(_reader.GetHappySourceLocation());
        //        throw new AbortParseException(_reader.GetHappySourceLocation());
        //    }
        //}
        Token ExtractToken()
        {
            Token retval;

            _reader.ResetStartLocation();

            if (_reader.Eof)
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput);
            else
            {
                switch (this.Mode)
                {
                case LexerMode.LexingVerbatimText:
                    retval = ExtractVerbatimText();
                    break;
                case LexerMode.LexingVerbatimOutputExpression:
                case LexerMode.LexingStatement:
                    //read until an end of statement is encountered
                    retval = ExtractStatementToken();
                    break;
                default:
                    throw new UnhandledCaseSourceException(_reader.GetHappySourceLocation());
                }
            }

            DebugAssert.IsFalse(_reader.Peek() == 0xFFFF && !_reader.Eof, "_reader.Peek() returned 0xFFFF yet _reader.Eof is false");
            DebugAssert.IsNotNull(retval, "retval cannot be null");
            return retval;
        }
Ejemplo n.º 4
0
        Token ExtractNumber()
        {
            StringBuilder sb;
            Token retval = null;
            HappySourceLocation startedAt = _reader.GetHappySourceLocation();
            sb = new StringBuilder();
            if(_reader.Peek(0) == '0' && _reader.Peek(1) == 'x')
            {
                _reader.Read();
                _reader.Read();

                char peeked;
                do
                {
                    sb.Append(_reader.Read());
                    peeked = _reader.Peek();
                } while (Char.IsNumber(peeked) || (peeked >= 'a' && peeked <= 'z' || peeked >='A' && peeked <='Z'));
                if(sb.Length <= 8)
                    retval = new Token(startedAt, HappyTokenKind.LiteralHexInt32, sb.ToString());
                else if(sb.Length <= 16)
                    retval = new Token(startedAt, HappyTokenKind.LiteralHexInt64, sb.ToString());
                else
                    _errorCollector.TooManyCharactersInHexLiteral(startedAt);
            }
            else
            {

                do
                {
                    sb.Append(_reader.Read());
                }
                while(Char.IsNumber(_reader.Peek(0)) || _reader.Peek(0) == '.');

                string aNumber = sb.ToString();

                if(aNumber.IndexOf('.') < 0)
                    retval = new Token(startedAt, HappyTokenKind.LiteralDecimalInt32, aNumber);
                else
                    retval = new Token(startedAt, HappyTokenKind.LiteralDouble, aNumber);
            }
            return retval;
        }
Ejemplo n.º 5
0
        Token ExtractOtherStatementToken()
        {
            Token retval;
            //lexer state changes occur in the first few cases below.
            switch (_reader.Peek())
            {
            case '<':
                _reader.Read();
                switch (_reader.Peek(0))
                {
                case '=':
                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThanOrEqual, "<=");
                    break;
                case '|':
                    _reader.Read();

                    if(this.Mode == LexerMode.LexingVerbatimOutputExpression)
                        _errorCollector.CannotNestTemplatesWithinTemplateOutputExpression(_reader.GetHappySourceLocation());

                    this.PushState(LexerMode.LexingVerbatimText);
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.BeginTemplate, "<|");
                    break;
                default:
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThan, "<");
                    break;
                }
                break;
            case '%':
                _reader.Read();
                if (_reader.Peek(0) != '|')
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMod, "%");
                else
                {
                    _reader.Read();

                    if(_stateStack.Count <= 1 || this.Mode == LexerMode.LexingVerbatimOutputExpression)
                        _errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());

                    this.PopState();

                    //I don't believe this case can ever happen...
                    //if(this.Mode != LexerMode.LexingVerbatimText)
                    //    _errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());
                    DebugAssert.AreEqual(this.Mode, LexerMode.LexingVerbatimText, "Unexpected case.  Debug me.");

                    retval = this.ExtractToken();
                }
                break;
            case '|':
                _reader.Read();
                switch (_reader.Peek(0))
                {
                case '>':

                    if (this.Mode == LexerMode.LexingVerbatimOutputExpression)
                        _errorCollector.MismatchedVerbatimOutputExpressionDelimiter(_reader.GetHappySourceLocation());

                    //Do not PopState here as it is already popped in ExtractVerbatimText()
                    //this.PopState();

                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndTemplate, "|>");

                    break;
                case '%':
                    _reader.Read();
                    _errorCollector.BeginTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());
                    throw new AbortParseException(_reader.GetHappySourceLocation());
                case '|':
                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalOr, "||");
                    break;
                default:
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseOr, "|");
                    break;
                }
                break;
            case '$':
                //First time this executes is the next extraction after the switch from LexingVerbatimText.
                //The state should just have been changed to LexingVerbatimOutputExpression
                //If not, we're in a state (LexingStatement, currently) where $ is not allowed.
                if(this.Mode != LexerMode.LexingVerbatimOutputExpression)
                    _errorCollector.VerbatimOutputExpressionDelimiterNotAllowedHere(_reader.GetHappySourceLocation());

                _reader.Read();
                this.VerbatimOuputExpressionDelimiterCount++;
                if (this.VerbatimOuputExpressionDelimiterCount >= 2)
                {
                    this.PopState();
                    DebugAssert.AreEqual(LexerMode.LexingVerbatimText, this.Mode, "Previous lexer mode was not LexingVerbatimText.  That shoulnd't have happened.");
                }

                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.VerbatimOutputExpressionDelimiter, "$");
                break;
            case ';':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfStatement, ";");
                break;
            case '(':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenParen, "(");
                break;
            case ')':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseParen, ")");
                break;
            case '!':
                _reader.Read();
                if (_reader.Peek(0) == '=')
                {
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorNotEqual, "!=");
                    _reader.Read();
                }
                else
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.UnaryOperatorNot, "!");
                break;
            case '&':
                _reader.Read();
                if(_reader.Peek() == '&')
                {
                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalAnd, "&&");
                }
                else
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseAnd, "&");
                break;
            case '^':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorXor, "^");
                break;
            case ',':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Comma, ",");
                break;
            case '.':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDot, ".");
                break;
            case '[':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenBracket, "[");
                break;
            case ']':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseBracket, "]");
                break;
            case '{':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OpenBrace, "{");
                break;
            case '}':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.CloseBrace, "}");
                break;
            case '>':
                _reader.Read();
                if (_reader.Peek(0) == '=')
                {
                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThanOrEqual, ">=");
                }
                else
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThan, ">");
                break;
            case '=':
                _reader.Read();
                if (_reader.Peek(0) == '=')
                {
                    _reader.Read();
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorEqual, "==");
                }
                else
                    retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAssign, "=");
                break;
            case '+':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAdd, "+");
                break;
            case '-':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorSubtract, "-");
                break;
            case '/':
                _reader.Read();
                //if (_reader.Peek(0) == '*')
                //    do
                //        _reader.Read();
                //    while(_reader.Peek(0) != '*' && _reader.Peek(1) != '/');
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDivide, "/");
                break;
            case '*':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMultiply, "*");
                break;
            case '~':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Output, "~");
                break;
            case ':':
                _reader.Read();
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Colon, ":");
                break;
            case '\'':
                _reader.Read();
                this.FailIfEof(_reader.GetHappySourceLocation());
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.LiteralChar, _reader.Read().ToString());
                this.FailIfEof(_reader.GetHappySourceLocation());
                if (_reader.Read() != '\'')
                    _errorCollector.LiteralCharsMustBeOneCharLong(_reader.GetHappySourceLocation());
                break;
            case '"':
                retval = ExtractLiteralString();
                break;
            default:
                char invalidChar = _reader.Read();
                _errorCollector.InvalidCharacter(_reader.GetHappySourceLocation(), invalidChar);
                retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.InvalidCharacter, Resources.MiscMessages.InvalidCharacter);
                break;
            }
            return retval;
        }
Ejemplo n.º 6
0
        Token ExtractLiteralString()
        {
            StringBuilder sb;
            Token retval;
            sb = new StringBuilder();
            //Eat opening quote
            _reader.Read();
            while (_reader.Peek() != '"')
            {
                this.FailIfEof(_reader.GetHappySourceLocation());
                if (_reader.Peek() == '\\')
                {
                    _reader.Read();
                    char escapedChar = _reader.Read();
                    switch (escapedChar)
                    {
                    case 't':
                        sb.Append('\t');
                        break;
                    case 'r':
                        sb.Append('\r');
                        break;
                    case 'n':
                        sb.Append('\n');
                        break;
                    case '"':
                        sb.Append('"');
                        break;
                    case '\\':
                        sb.Append('\\');
                        break;
                    default:
                        _errorCollector.InvalidEscapeSequence(_reader.GetHappySourceLocation(), escapedChar);
                        break;
                    }
                }
                else
                    sb.Append(_reader.Read());

                if (_reader.Peek() == '\n')
                    _errorCollector.NewLineInStringLiteral(_reader.GetHappySourceLocation());

            }
            //Eat closing quote
            _reader.Read();

            retval = new Token(_reader.GetHappySourceLocation(),
                               HappyTokenKind.LiteralString, sb.ToString());
            return retval;
        }
Ejemplo n.º 7
0
        private ExpressionNodeBase PromoteToExpression(Token firstToken)
        {
            ExpressionNodeBase retval;
            int intValue;
            switch(firstToken.HappyTokenKind)
            {
                case HappyTokenKind.Identifier:
                    HappySourceLocation startsAt, endsAt;
                    if (this.EatNextTokenIf(HappyTokenKind.OperatorOpenParen, out startsAt))
                    {
                        List<ExpressionNodeBase> arguments = new List<ExpressionNodeBase>();
                        bool parseMore = !this.EatNextTokenIf(HappyTokenKind.OperatorCloseParen, out endsAt);
                        while (parseMore)
                        {
                            arguments.Add(this.ParseExpression(ExpressionContext.ArgumentList));

                            var commaOrCloseParen = this.Expect(HappyTokenKind.Comma, HappyTokenKind.OperatorCloseParen, MiscMessages.CommaOrCloseParen);
                            endsAt = commaOrCloseParen.Location;
                            parseMore = commaOrCloseParen.HappyTokenKind != HappyTokenKind.OperatorCloseParen;
                        }
                        retval = new FunctionCallExpression(startsAt, endsAt, firstToken.ToIdentifier(), arguments.ToArray());
                    }
                    else
                        retval = new IdentifierExpression(firstToken.ToIdentifier());

                    break;
                case HappyTokenKind.LiteralBool:
                    switch(firstToken.Text)
                    {
                        case "true":
                            retval = new LiteralExpression(firstToken.Location, true);
                            break;
                        case "false":
                            retval = new LiteralExpression(firstToken.Location, false);
                            break;
                        default:
                            throw new UnhandledCaseSourceException(firstToken.Location);
                    }
                    break;
                case HappyTokenKind.LiteralDecimalInt32:
                    if(!Int32.TryParse(firstToken.Text, out intValue))
                        throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralInt32?!?!", firstToken.Text);
                    retval = new LiteralExpression(firstToken.Location, intValue);
                    break;
                case HappyTokenKind.LiteralHexInt32:
                    if(!Int32.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out intValue))
                        throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt32?!?!", firstToken.Text);
                    retval = new LiteralExpression(firstToken.Location, intValue);
                    break;
                case HappyTokenKind.LiteralHexInt64:
                    long longValue;
                    if (!Int64.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out longValue))
                        throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt64?!?!", firstToken.Text);
                    retval = new LiteralExpression(firstToken.Location, longValue);
                    break;
                case HappyTokenKind.LiteralDouble:
                    double doubleValue;
                    try
                    {
                        doubleValue = Double.Parse(firstToken.Text);
                    } catch(Exception e)
                    {
                        throw new InternalSourceException(e, firstToken.Location,
                                                          "Failed to parse a Double from \"{0}\" but _tokenKind is LiteralDouble?!?!", firstToken.Text);
                    }
                    retval = new LiteralExpression(firstToken.Location, doubleValue);
                    break;
                case HappyTokenKind.LiteralString:
                    retval = new LiteralExpression(firstToken.Location, firstToken.Text);
                    break;
                case HappyTokenKind.LiteralNull:
                    retval = new NullExpression(firstToken.Location);
                    break;
                case HappyTokenKind.BeginTemplate:
                    retval = this.ParseAnonymousTemplateExpression(firstToken.Location);
                    break;
                default:
                    _errorCollector.UnexpectedToken(firstToken);
                    throw new AbortParseException(firstToken.Location);
            }
            return retval;
        }
Ejemplo n.º 8
0
 public void Expected(string expected, Token actual)
 {
     this.Add(actual.Location, ErrorCode.Expected_Expected_Actual, expected, actual.Text);
 }
Ejemplo n.º 9
0
 public void DuplicateInputValue(Token value)
 {
     this.Add(value.Location, ErrorCode.DuplicateInputValue_Value, value.Text);
 }
Ejemplo n.º 10
0
 public void UnexpectedToken(Token t)
 {
     this.Add(t.Location, ErrorCode.UnexpectedToken_TokenText, t.Text);
 }