Token ExtractWord() { StringBuilder sb; Token retval; sb = new StringBuilder(); this.ReadAppendWord(sb); string word = sb.ToString(); HappyTokenKind maybeKeyword; if (_keywords.TryGetValue(word, out maybeKeyword)) retval = new Token(_reader.GetHappySourceLocation(), maybeKeyword, word); else retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Identifier, word); return retval; }
Token ExtractStatementToken() { Token retval; this.EatWhite(); if (_reader.Eof) retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput); else { if (Char.IsLetter(_reader.Peek())) retval = ExtractWord(); else if (Char.IsNumber(_reader.Peek(0)) || (_reader.Peek(0) == '-' && Char.IsNumber(_reader.Peek(1)))) retval = ExtractNumber(); else retval = ExtractOtherStatementToken(); } return retval; }
//public void FailIfEof() //{ // if (_reader.Eof) // { // foreach (Token t in _readAhead) // if (t.HappyTokenKind != HappyTokenKind.EndOfInput) // return; // SourceSpan span = new SourceSpan(_enteredLastStatementGroupAt, _reader.CurrentLocation); // if (_state == LexerState.LexingStatement) // _errorCollector.UnexpectedEndOfInputWhileParsingSection(_reader.GetHappySourceLocation()); // else // _errorCollector.UnexpectedEndOfInputInComment(_reader.GetHappySourceLocation()); // throw new AbortParseException(_reader.GetHappySourceLocation()); // } //} Token ExtractToken() { Token retval; _reader.ResetStartLocation(); if (_reader.Eof) retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput); else { switch (this.Mode) { case LexerMode.LexingVerbatimText: retval = ExtractVerbatimText(); break; case LexerMode.LexingVerbatimOutputExpression: case LexerMode.LexingStatement: //read until an end of statement is encountered retval = ExtractStatementToken(); break; default: throw new UnhandledCaseSourceException(_reader.GetHappySourceLocation()); } } DebugAssert.IsFalse(_reader.Peek() == 0xFFFF && !_reader.Eof, "_reader.Peek() returned 0xFFFF yet _reader.Eof is false"); DebugAssert.IsNotNull(retval, "retval cannot be null"); return retval; }
Token ExtractNumber() { StringBuilder sb; Token retval = null; HappySourceLocation startedAt = _reader.GetHappySourceLocation(); sb = new StringBuilder(); if(_reader.Peek(0) == '0' && _reader.Peek(1) == 'x') { _reader.Read(); _reader.Read(); char peeked; do { sb.Append(_reader.Read()); peeked = _reader.Peek(); } while (Char.IsNumber(peeked) || (peeked >= 'a' && peeked <= 'z' || peeked >='A' && peeked <='Z')); if(sb.Length <= 8) retval = new Token(startedAt, HappyTokenKind.LiteralHexInt32, sb.ToString()); else if(sb.Length <= 16) retval = new Token(startedAt, HappyTokenKind.LiteralHexInt64, sb.ToString()); else _errorCollector.TooManyCharactersInHexLiteral(startedAt); } else { do { sb.Append(_reader.Read()); } while(Char.IsNumber(_reader.Peek(0)) || _reader.Peek(0) == '.'); string aNumber = sb.ToString(); if(aNumber.IndexOf('.') < 0) retval = new Token(startedAt, HappyTokenKind.LiteralDecimalInt32, aNumber); else retval = new Token(startedAt, HappyTokenKind.LiteralDouble, aNumber); } return retval; }
Token ExtractOtherStatementToken() { Token retval; //lexer state changes occur in the first few cases below. switch (_reader.Peek()) { case '<': _reader.Read(); switch (_reader.Peek(0)) { case '=': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThanOrEqual, "<="); break; case '|': _reader.Read(); if(this.Mode == LexerMode.LexingVerbatimOutputExpression) _errorCollector.CannotNestTemplatesWithinTemplateOutputExpression(_reader.GetHappySourceLocation()); this.PushState(LexerMode.LexingVerbatimText); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.BeginTemplate, "<|"); break; default: retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThan, "<"); break; } break; case '%': _reader.Read(); if (_reader.Peek(0) != '|') retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMod, "%"); else { _reader.Read(); if(_stateStack.Count <= 1 || this.Mode == LexerMode.LexingVerbatimOutputExpression) _errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation()); this.PopState(); //I don't believe this case can ever happen... //if(this.Mode != LexerMode.LexingVerbatimText) // _errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation()); DebugAssert.AreEqual(this.Mode, LexerMode.LexingVerbatimText, "Unexpected case. Debug me."); retval = this.ExtractToken(); } break; case '|': _reader.Read(); switch (_reader.Peek(0)) { case '>': if (this.Mode == LexerMode.LexingVerbatimOutputExpression) _errorCollector.MismatchedVerbatimOutputExpressionDelimiter(_reader.GetHappySourceLocation()); //Do not PopState here as it is already popped in ExtractVerbatimText() //this.PopState(); _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndTemplate, "|>"); break; case '%': _reader.Read(); _errorCollector.BeginTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation()); throw new AbortParseException(_reader.GetHappySourceLocation()); case '|': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalOr, "||"); break; default: retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseOr, "|"); break; } break; case '$': //First time this executes is the next extraction after the switch from LexingVerbatimText. //The state should just have been changed to LexingVerbatimOutputExpression //If not, we're in a state (LexingStatement, currently) where $ is not allowed. if(this.Mode != LexerMode.LexingVerbatimOutputExpression) _errorCollector.VerbatimOutputExpressionDelimiterNotAllowedHere(_reader.GetHappySourceLocation()); _reader.Read(); this.VerbatimOuputExpressionDelimiterCount++; if (this.VerbatimOuputExpressionDelimiterCount >= 2) { this.PopState(); DebugAssert.AreEqual(LexerMode.LexingVerbatimText, this.Mode, "Previous lexer mode was not LexingVerbatimText. That shoulnd't have happened."); } retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.VerbatimOutputExpressionDelimiter, "$"); break; case ';': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfStatement, ";"); break; case '(': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenParen, "("); break; case ')': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseParen, ")"); break; case '!': _reader.Read(); if (_reader.Peek(0) == '=') { retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorNotEqual, "!="); _reader.Read(); } else retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.UnaryOperatorNot, "!"); break; case '&': _reader.Read(); if(_reader.Peek() == '&') { _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalAnd, "&&"); } else retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseAnd, "&"); break; case '^': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorXor, "^"); break; case ',': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Comma, ","); break; case '.': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDot, "."); break; case '[': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenBracket, "["); break; case ']': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseBracket, "]"); break; case '{': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OpenBrace, "{"); break; case '}': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.CloseBrace, "}"); break; case '>': _reader.Read(); if (_reader.Peek(0) == '=') { _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThanOrEqual, ">="); } else retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThan, ">"); break; case '=': _reader.Read(); if (_reader.Peek(0) == '=') { _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorEqual, "=="); } else retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAssign, "="); break; case '+': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAdd, "+"); break; case '-': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorSubtract, "-"); break; case '/': _reader.Read(); //if (_reader.Peek(0) == '*') // do // _reader.Read(); // while(_reader.Peek(0) != '*' && _reader.Peek(1) != '/'); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDivide, "/"); break; case '*': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMultiply, "*"); break; case '~': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Output, "~"); break; case ':': _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Colon, ":"); break; case '\'': _reader.Read(); this.FailIfEof(_reader.GetHappySourceLocation()); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.LiteralChar, _reader.Read().ToString()); this.FailIfEof(_reader.GetHappySourceLocation()); if (_reader.Read() != '\'') _errorCollector.LiteralCharsMustBeOneCharLong(_reader.GetHappySourceLocation()); break; case '"': retval = ExtractLiteralString(); break; default: char invalidChar = _reader.Read(); _errorCollector.InvalidCharacter(_reader.GetHappySourceLocation(), invalidChar); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.InvalidCharacter, Resources.MiscMessages.InvalidCharacter); break; } return retval; }
Token ExtractLiteralString() { StringBuilder sb; Token retval; sb = new StringBuilder(); //Eat opening quote _reader.Read(); while (_reader.Peek() != '"') { this.FailIfEof(_reader.GetHappySourceLocation()); if (_reader.Peek() == '\\') { _reader.Read(); char escapedChar = _reader.Read(); switch (escapedChar) { case 't': sb.Append('\t'); break; case 'r': sb.Append('\r'); break; case 'n': sb.Append('\n'); break; case '"': sb.Append('"'); break; case '\\': sb.Append('\\'); break; default: _errorCollector.InvalidEscapeSequence(_reader.GetHappySourceLocation(), escapedChar); break; } } else sb.Append(_reader.Read()); if (_reader.Peek() == '\n') _errorCollector.NewLineInStringLiteral(_reader.GetHappySourceLocation()); } //Eat closing quote _reader.Read(); retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.LiteralString, sb.ToString()); return retval; }
private ExpressionNodeBase PromoteToExpression(Token firstToken) { ExpressionNodeBase retval; int intValue; switch(firstToken.HappyTokenKind) { case HappyTokenKind.Identifier: HappySourceLocation startsAt, endsAt; if (this.EatNextTokenIf(HappyTokenKind.OperatorOpenParen, out startsAt)) { List<ExpressionNodeBase> arguments = new List<ExpressionNodeBase>(); bool parseMore = !this.EatNextTokenIf(HappyTokenKind.OperatorCloseParen, out endsAt); while (parseMore) { arguments.Add(this.ParseExpression(ExpressionContext.ArgumentList)); var commaOrCloseParen = this.Expect(HappyTokenKind.Comma, HappyTokenKind.OperatorCloseParen, MiscMessages.CommaOrCloseParen); endsAt = commaOrCloseParen.Location; parseMore = commaOrCloseParen.HappyTokenKind != HappyTokenKind.OperatorCloseParen; } retval = new FunctionCallExpression(startsAt, endsAt, firstToken.ToIdentifier(), arguments.ToArray()); } else retval = new IdentifierExpression(firstToken.ToIdentifier()); break; case HappyTokenKind.LiteralBool: switch(firstToken.Text) { case "true": retval = new LiteralExpression(firstToken.Location, true); break; case "false": retval = new LiteralExpression(firstToken.Location, false); break; default: throw new UnhandledCaseSourceException(firstToken.Location); } break; case HappyTokenKind.LiteralDecimalInt32: if(!Int32.TryParse(firstToken.Text, out intValue)) throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralInt32?!?!", firstToken.Text); retval = new LiteralExpression(firstToken.Location, intValue); break; case HappyTokenKind.LiteralHexInt32: if(!Int32.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out intValue)) throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt32?!?!", firstToken.Text); retval = new LiteralExpression(firstToken.Location, intValue); break; case HappyTokenKind.LiteralHexInt64: long longValue; if (!Int64.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out longValue)) throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt64?!?!", firstToken.Text); retval = new LiteralExpression(firstToken.Location, longValue); break; case HappyTokenKind.LiteralDouble: double doubleValue; try { doubleValue = Double.Parse(firstToken.Text); } catch(Exception e) { throw new InternalSourceException(e, firstToken.Location, "Failed to parse a Double from \"{0}\" but _tokenKind is LiteralDouble?!?!", firstToken.Text); } retval = new LiteralExpression(firstToken.Location, doubleValue); break; case HappyTokenKind.LiteralString: retval = new LiteralExpression(firstToken.Location, firstToken.Text); break; case HappyTokenKind.LiteralNull: retval = new NullExpression(firstToken.Location); break; case HappyTokenKind.BeginTemplate: retval = this.ParseAnonymousTemplateExpression(firstToken.Location); break; default: _errorCollector.UnexpectedToken(firstToken); throw new AbortParseException(firstToken.Location); } return retval; }
public void Expected(string expected, Token actual) { this.Add(actual.Location, ErrorCode.Expected_Expected_Actual, expected, actual.Text); }
public void DuplicateInputValue(Token value) { this.Add(value.Location, ErrorCode.DuplicateInputValue_Value, value.Text); }
public void UnexpectedToken(Token t) { this.Add(t.Location, ErrorCode.UnexpectedToken_TokenText, t.Text); }