// OCTAL: [0-7]([0-7_]*[0-7])? // HEXA: [0-9a-fA-F]([0-9a-fA-F_]*[0-9a-fA-F])? // BINARY: [01]([01_]*[01])? // DECIMAL: [0-9]([0-9_]*[0-9])? // FLOAT: [1-9]([0-9_]*[0-9])?[.][0-9_]*[0-9]([eE][+-]?[0-9]([0-9_]*[0-9])?) // // Previous digit caused an integer overflow. // numberStartIndex ... index of the first (most significant) digit // underscoreCount ... number of underscores already read private Tokens ReadBigNumber(long value, int @base, int numberStartIndex, int underscoreCount, bool allowDouble) { Debug.Assert(!allowDouble || @base == 10, "Only decimal based doubles supported"); Debug.Assert(@base <= 16); // the previous char is a digit: NumericCharKind prev = NumericCharKind.Digit; while (true) { int c = Peek(); int digit = ToDigit(c); if (digit < @base) { prev = NumericCharKind.Digit; Skip(c); } else { if (prev == NumericCharKind.Underscore) { ReportError(Errors.TrailingUnderscoreInNumber); } else if (c == '_') { Skip(c); prev = NumericCharKind.Underscore; underscoreCount++; continue; } else if (allowDouble) { int sign; if ((c == 'e' || c == 'E') && TryReadExponentSign(1, out sign)) { return ReadDoubleExponent(numberStartIndex, sign); } else if (c == '.') { if (IsDecimalDigit(Peek(1))) { Skip('.'); return ReadDouble(numberStartIndex); } } } // TODO: store only the digit count, the actual value will be parsed later: // TODO: skip initial zeros if (_bigIntParser == null) { _bigIntParser = new BignumParser(); } _bigIntParser.Position = numberStartIndex; _bigIntParser.Buffer = _lineBuffer; BigInteger result = _bigIntParser.Parse(_bufferPos - numberStartIndex - underscoreCount, @base); Debug.Assert(value > 0, "Cannot be zero since we are parsing a number greater than Int32.MaxValue"); _tokenValue.SetBigInteger(result); return Tokens.BigInteger; } } }
// @base == 0: // [:whitespace:]*[+-]?(0x|0X|ob|0B|0d|0D|0o|0O)?([:base-digit:][_]?)*[:base-digit:].* // otherwise: // [:whitespace:]*[+-]?([:base-digit:][_]?)*[:base-digit:].* public static IntegerValue ParseInteger(string/*!*/ str, int @base, ref int i) { ContractUtils.RequiresNotNull(str, "str"); int c; do { c = NextChar(str, ref i); } while (IsWhiteSpace(c)); int sign; if (c == '+') { sign = +1; c = NextChar(str, ref i); } else if (c == '-') { sign = -1; c = NextChar(str, ref i); } else { sign = +1; } if (c == '0') { c = NextChar(str, ref i); int newBase = 0; switch (c) { case 'x': case 'X': newBase = 16; break; case 'b': case 'B': newBase = 2; break; case 'd': case 'D': newBase = 10; break; case 'o': case 'O': newBase = 8; break; } if (newBase != 0) { // no base specified -> set the base // base specified -> skip prefix of that base if (@base == 0 || newBase == @base) { @base = newBase; c = NextChar(str, ref i); } } else if (@base == 0) { @base = 8; } } else if (@base == 0) { @base = 10; } bool underAllowed = false; long value = 0; int digitCount = 0; int start = i - 1; while (true) { if (c != '_') { int digit = ToDigit(c); if (digit < @base) { if (value <= Int32.MaxValue) { value = value * @base + digit; } digitCount++; } else { break; } underAllowed = true; } else if (underAllowed) { underAllowed = false; } else { break; } c = NextChar(str, ref i); } if (digitCount == 0) { return 0; } if (value <= Int32.MaxValue) { value *= sign; if (value >= Int32.MinValue && value <= Int32.MaxValue) { return (int)value; } else { return BigInteger.Create(value); } } else { var parser = new BignumParser(); parser.Position = start; parser.Buffer = str.ToCharArray(); return parser.Parse(digitCount, @base) * sign; } }
public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) { ContractUtils.RequiresNotNull(errorSink, "errorSink"); _bigIntParser = new BignumParser(); _errorSink = errorSink; _sourceUnit = null; _parser = null; _verbatim = verbatim; _compatibility = RubyCompatibility.Default; // _buffer = null; _initialLocation = SourceLocation.Invalid; _tokenSpan = SourceSpan.Invalid; _tokenValue = new TokenValue(); _bufferPos = 0; // TODO: _input = null; }