private NumericSuffix ScanRealSuffixOpt() { NumericSuffix suffix = NumericSuffix.None; switch (PeekChar()) { case 'F': case 'f': suffix = NumericSuffix.F; break; case 'D': case 'd': suffix = NumericSuffix.D; break; case 'M': case 'm': suffix = NumericSuffix.M; break; default: return(NumericSuffix.None); } NextChar(); return(suffix); }
private NumericSuffix ScanIntegerSuffixOpt() { NumericSuffix suffix = NumericSuffix.None; while (true) { char ch = PeekChar(); if (ch == 'U' || ch == 'u') { if ((suffix & NumericSuffix.U) != 0) { break; } suffix |= NumericSuffix.U; } else if (ch == 'l' || ch == 'L') { if ((suffix & NumericSuffix.L) != 0) { break; } suffix |= NumericSuffix.L; } else { break; } NextChar(); } return(suffix); }
private Token CreateIntegerConstant(ulong value, NumericSuffix suffix) { switch (suffix) { case NumericSuffix.None: if (value <= int.MaxValue) { return(new IntToken((int)value, path, TakePosition())); } else if (value <= uint.MaxValue) { return(new UIntToken((uint)value, path, TakePosition())); } else if (value <= long.MaxValue) { return(new LongToken((long)value, path, TakePosition())); } goto case NumericSuffix.Ul; case NumericSuffix.U: if (value <= uint.MaxValue) { return(new UIntToken((uint)value, path, TakePosition())); } goto case NumericSuffix.Ul; case NumericSuffix.L: if (value <= long.MaxValue) { return(new LongToken((long)value, path, TakePosition())); } goto case NumericSuffix.Ul; case NumericSuffix.Ul: default: return(new ULongToken(value, path, TakePosition())); } }
private Token NextToken() { SkipWhiteSpace(); StartToken(); bool atIdentifier = false; char ch = PeekChar(); if (ch == '\0') { return(NewToken(TokenType.Eof)); } ch = NextChar(); switch (ch) { case '\0': Debug.Fail("Checked for EOF above"); return(null); case '#': if (text.Line == lastLine) { ReportError(LexError.UnexpectedCharacter, ch.ToString()); return(ErrorToken()); } else { ClearPosition(); text.Reverse(); return(null); } // operators case '{': return(NewToken(TokenType.OpenCurly)); case '}': return(NewToken(TokenType.CloseCurly)); case '[': return(NewToken(TokenType.OpenSquare)); case ']': return(NewToken(TokenType.CloseSquare)); case '(': return(NewToken(TokenType.OpenParen)); case ')': return(NewToken(TokenType.CloseParen)); case ',': return(NewToken(TokenType.Comma)); case ':': ch = PeekChar(); if (ch == ':') { NextChar(); return(NewToken(TokenType.ColonColon)); } return(NewToken(TokenType.Colon)); case ';': return(NewToken(TokenType.Semicolon)); case '~': return(NewToken(TokenType.Tilde)); case '?': ch = PeekChar(); if (ch == '?') { NextChar(); return(NewToken(TokenType.Coalesce)); } return(NewToken(TokenType.Question)); case '+': ch = PeekChar(); if (ch == '+') { NextChar(); return(NewToken(TokenType.PlusPlus)); } else if (ch == '=') { NextChar(); return(NewToken(TokenType.PlusEqual)); } return(NewToken(TokenType.Plus)); case '-': ch = PeekChar(); if (ch == '-') { NextChar(); return(NewToken(TokenType.MinusMinus)); } else if (ch == '=') { NextChar(); return(NewToken(TokenType.MinusEqual)); } else if (ch == '>') { NextChar(); return(NewToken(TokenType.Arrow)); } return(NewToken(TokenType.Minus)); case '*': if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.StarEqual)); } return(NewToken(TokenType.Star)); case '/': ch = PeekChar(); if (ch == '=') { NextChar(); return(NewToken(TokenType.SlashEqual)); } else if (ch == '/') { NextChar(); CommentTokenType commentType; if (!Eof && PeekChar() == '/') { commentType = CommentTokenType.TripleSlash; NextChar(); } else { commentType = CommentTokenType.DoubleSlash; } value.Length = 0; while (!Eof && !IsLineSeparator(PeekChar())) { value.Append(NextChar()); } if (includeComments) { return(new CommentToken(commentType, value.ToString(), path, TakePosition())); } TakePosition(); return(NextToken()); } else if (ch == '*') { NextChar(); value.Length = 0; while (!Eof && (PeekChar() != '*' || PeekChar(1) != '/')) { value.Append(NextChar()); } if (Eof) { ReportError(LexError.UnexpectedEndOfFileStarSlash); return(ErrorToken()); } NextChar(); NextChar(); lastLine = text.Line; if (includeComments) { return(new CommentToken(CommentTokenType.SlashStar, value.ToString(), path, TakePosition())); } TakePosition(); return(NextToken()); } return(NewToken(TokenType.Slash)); case '%': if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.PercentEqual)); } return(NewToken(TokenType.Percent)); case '&': ch = PeekChar(); if (ch == '&') { NextChar(); return(NewToken(TokenType.LogAnd)); } else if (ch == '=') { NextChar(); return(NewToken(TokenType.AndEqual)); } return(NewToken(TokenType.Ampersand)); case '|': ch = PeekChar(); if (ch == '|') { NextChar(); return(NewToken(TokenType.LogOr)); } else if (ch == '=') { NextChar(); return(NewToken(TokenType.BarEqual)); } return(NewToken(TokenType.Bar)); case '^': if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.HatEqual)); } return(NewToken(TokenType.Hat)); case '!': if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.NotEqual)); } return(NewToken(TokenType.Bang)); case '=': if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.EqualEqual)); } return(NewToken(TokenType.Equal)); case '<': ch = PeekChar(); if (ch == '=') { NextChar(); return(NewToken(TokenType.LessEqual)); } else if (ch == '<') { NextChar(); if (PeekChar() == '=') { NextChar(); return(NewToken(TokenType.ShiftLeftEqual)); } return(NewToken(TokenType.ShiftLeft)); } return(NewToken(TokenType.Less)); case '>': ch = PeekChar(); if (ch == '=') { NextChar(); return(NewToken(TokenType.GreaterEqual)); } return(NewToken(TokenType.Greater)); // literals case '\'': // char literal { char ch2; if (ScanCharValue('\'', false, out ch, out ch2)) { if (PeekChar() != '\'') { ReportError(LexError.BadCharConstant); } else { NextChar(); } return(new CharToken(ch, path, TakePosition())); } return(ErrorToken()); } case '"': // string literal { value.Length = 0; while (!Eof && ScanCharValue('"', true, out ch, out char ch2)) { value.Append(ch); if (ch2 != 0) { value.Append(ch2); } } if (Eof) { ReportError(LexError.UnexpectedEndOfFileString); return(ErrorToken()); } return(new StringToken(value.ToString(), path, TakePosition())); } case '@': ch = PeekChar(); if (ch == '"') { // verbatim string literal NextChar(); value.Length = 0; while (!Eof && (PeekChar() != '"' || PeekChar(1) == '"')) { // this is the one place where a CR/LF pair is significant ch = NextChar(out bool wasCrlf); value.Append(ch); if (wasCrlf) { value.Append('\xA'); } else if (ch == '"') { NextChar(); } } if (Eof) { ReportError(LexError.UnexpectedEndOfFileString); return(ErrorToken()); } NextChar(); return(new StringToken(value.ToString(), path, TakePosition())); } atIdentifier = true; goto default; case '0': if (PeekChar() == 'x' || PeekChar() == 'X') { NextChar(); // hexadecimal constant ulong value = 0; while (IsHexDigit(PeekChar())) { if ((value & 0xF000000000000000) != 0) { ReportError(LexError.NumericConstantOverflow); return(ErrorToken()); } value = (value << 4) | (uint)HexValue(NextChar()); } return(CreateIntegerConstant(value, ScanIntegerSuffixOpt())); } goto case '1'; case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': DoNumber: { bool foundDecimalPoint = ch == '.'; bool foundExponent = false; NumericSuffix suffix = NumericSuffix.None; value.Length = 0; value.Append(ch); while (true) { ch = PeekChar(); if (ch == '.') { if (foundDecimalPoint || !IsDigit(PeekChar(1))) { break; } foundDecimalPoint = true; } else if (ch == 'e' || ch == 'E') { char nextChar = PeekChar(1); if (IsDigit(nextChar) || (nextChar == '+' || nextChar == '-') && IsDigit(PeekChar(2))) { foundExponent = true; value.Append(NextChar()); value.Append(NextChar()); while (IsDigit(PeekChar())) { value.Append(NextChar()); } } break; } else if (!IsDigit(ch)) { break; } value.Append(NextChar()); } if (!foundDecimalPoint && !foundExponent) { suffix = ScanIntegerSuffixOpt(); } if (suffix == NumericSuffix.None) { suffix = ScanRealSuffixOpt(); } if (suffix < NumericSuffix.F && !foundDecimalPoint && !foundExponent) { // decimal integer constant ulong numericValue = 0; foreach (char digit in value.ToString()) { ulong value10 = numericValue * 10; if (value10 < numericValue) { ReportError(LexError.NumericConstantOverflow); return(ErrorToken()); } numericValue = value10 + (uint)(digit - '0'); } return(CreateIntegerConstant(numericValue, suffix)); } try { // real constant switch (suffix) { case NumericSuffix.F: { float f = float.Parse(value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture); return(new FloatToken(f, path, TakePosition())); } case NumericSuffix.D: case NumericSuffix.None: { double d = double.Parse(value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture); return(new DoubleToken(d, path, TakePosition())); } case NumericSuffix.M: { decimal dec = decimal.Parse(value.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture); return(new DecimalToken(dec, path, TakePosition())); } } } catch (Exception) { // catch overflow exceptions from the numeric parse } ReportError(LexError.NumericConstantOverflow); return(ErrorToken()); } case '.': if (IsDigit(PeekChar())) { goto DoNumber; } return(NewToken(TokenType.Dot)); default: { value.Length = 0; if (!IsIdentifierChar(ch)) { ReportError(LexError.UnexpectedCharacter, ch.ToString()); return(ErrorToken()); } value.Append(ch); while (ScanIdentifierChar(out ch)) { value.Append(ch); } Debug.Assert(value.Length > 0); if (value.Length > IdentifierToken.MAX_IDENTIFIER_LENGTH) { ReportError(LexError.IdentifierTooLong); return(ErrorToken()); } Name name = nameTable.Add(value); if (!atIdentifier) { // check for keywords TokenType keyword = keywords.IsKeyword(name); if (Token.IsKeyword(keyword)) { return(new Token(keyword, path, TakePosition())); } } return(new IdentifierToken(name, atIdentifier, path, TakePosition())); } } }
private Token CreateIntegerConstant(ulong value, NumericSuffix suffix) { switch (suffix) { case NumericSuffix.None: if (value <= int.MaxValue) { return new IntToken((int)value, _path, TakePosition()); } else if (value <= uint.MaxValue) { return new UIntToken((uint)value, _path, TakePosition()); } else if (value <= long.MaxValue) { return new LongToken((long)value, _path, TakePosition()); } goto case NumericSuffix.UL; case NumericSuffix.U: if (value <= uint.MaxValue) { return new UIntToken((uint)value, _path, TakePosition()); } goto case NumericSuffix.UL; case NumericSuffix.L: if (value <= long.MaxValue) { return new LongToken((long)value, _path, TakePosition()); } goto case NumericSuffix.UL; case NumericSuffix.UL: default: return new ULongToken(value, _path, TakePosition()); } }