Exemple #1
0
        // OCTAL:   [0-7]([0-7_]*[0-7])?
        // HEXA:    [0-9a-fA-F]([0-9a-fA-F_]*[0-9a-fA-F])?
        // BINARY:  [01]([01_]*[01])?
        // DECIMAL: [0-9]([0-9_]*[0-9])?
        // FLOAT:   [1-9]([0-9_]*[0-9])?[.][0-9_]*[0-9]([eE][+-]?[0-9]([0-9_]*[0-9])?)
        //
        // Previous digit caused an integer overflow.
        // numberStartIndex ... index of the first (most significant) digit
        // underscoreCount  ... number of underscores already read
        private Tokens ReadBigNumber(long value, int @base, int numberStartIndex, int underscoreCount, bool allowDouble) {
            Debug.Assert(!allowDouble || @base == 10, "Only decimal based doubles supported");
            Debug.Assert(@base <= 16);

            // the previous char is a digit:
            NumericCharKind prev = NumericCharKind.Digit; 

            while (true) {
                int c = Peek();
                int digit = ToDigit(c);

                if (digit < @base) {
                    prev = NumericCharKind.Digit;
                    Skip(c);
                } else {

                    if (prev == NumericCharKind.Underscore) {
                        ReportError(Errors.TrailingUnderscoreInNumber);                        
                    } else if (c == '_') {
                        Skip(c);
                        prev = NumericCharKind.Underscore;
                        underscoreCount++;
                        continue;
                    } else if (allowDouble) {
                        int sign;
                        if ((c == 'e' || c == 'E') && TryReadExponentSign(1, out sign)) {
                            return ReadDoubleExponent(numberStartIndex, sign);
                        } else if (c == '.') {
                            if (IsDecimalDigit(Peek(1))) {
                                Skip('.');
                                return ReadDouble(numberStartIndex);
                            }
                        }
                    }

                    // TODO: store only the digit count, the actual value will be parsed later:
                    // TODO: skip initial zeros
                    if (_bigIntParser == null) {
                        _bigIntParser = new BignumParser();
                    }

                    _bigIntParser.Position = numberStartIndex;
                    _bigIntParser.Buffer = _lineBuffer;

                    BigInteger result = _bigIntParser.Parse(_bufferPos - numberStartIndex - underscoreCount, @base);

                    Debug.Assert(value > 0, "Cannot be zero since we are parsing a number greater than Int32.MaxValue");

                    _tokenValue.SetBigInteger(result);
                    return Tokens.BigInteger;
                }
            }
        }
Exemple #2
0
        // @base == 0:
        //    [:whitespace:]*[+-]?(0x|0X|ob|0B|0d|0D|0o|0O)?([:base-digit:][_]?)*[:base-digit:].*
        // otherwise:
        //    [:whitespace:]*[+-]?([:base-digit:][_]?)*[:base-digit:].*
        public static IntegerValue ParseInteger(string/*!*/ str, int @base, ref int i) {
            ContractUtils.RequiresNotNull(str, "str");

            int c;
            do { c = NextChar(str, ref i); } while (IsWhiteSpace(c));

            int sign;
            if (c == '+') {
                sign = +1;
                c = NextChar(str, ref i);
            } else if (c == '-') {
                sign = -1;
                c = NextChar(str, ref i);
            } else {
                sign = +1;
            }

            if (c == '0') {
                c = NextChar(str, ref i);
                int newBase = 0;
                switch (c) {
                    case 'x':
                    case 'X': newBase = 16; break;
                    case 'b':
                    case 'B': newBase = 2; break;
                    case 'd':
                    case 'D': newBase = 10; break;
                    case 'o':
                    case 'O': newBase = 8; break;
                }

                if (newBase != 0) {
                    // no base specified -> set the base
                    // base specified -> skip prefix of that base
                    if (@base == 0 || newBase == @base) {
                        @base = newBase;
                        c = NextChar(str, ref i);
                    }
                } else if (@base == 0) {
                    @base = 8;
                }
            } else if (@base == 0) {
                @base = 10;
            }

            bool underAllowed = false;
            long value = 0;
            int digitCount = 0;
            int start = i - 1;
            while (true) {
                if (c != '_') {
                    int digit = ToDigit(c);
                    if (digit < @base) {
                        if (value <= Int32.MaxValue) {
                            value = value * @base + digit;
                        }
                        digitCount++;
                    } else {
                        break;
                    }
                    underAllowed = true;
                } else if (underAllowed) {
                    underAllowed = false;
                } else {
                    break;
                }
                c = NextChar(str, ref i);
            }

            if (digitCount == 0) {
                return 0;
            }
            
            if (value <= Int32.MaxValue) {
                value *= sign;
                if (value >= Int32.MinValue && value <= Int32.MaxValue) {
                    return (int)value;
                } else {
                    return BigInteger.Create(value);
                }
            } else {
                var parser = new BignumParser();
                parser.Position = start;
                parser.Buffer = str.ToCharArray();
                
                return parser.Parse(digitCount, @base) * sign;
            }
        }
Exemple #3
0
        public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) {
            ContractUtils.RequiresNotNull(errorSink, "errorSink");

            _bigIntParser = new BignumParser();
            _errorSink = errorSink;
            _sourceUnit = null;
            _parser = null;
            _verbatim = verbatim;
            _compatibility = RubyCompatibility.Default;
//            _buffer = null;
            _initialLocation = SourceLocation.Invalid;
            _tokenSpan = SourceSpan.Invalid;
            _tokenValue = new TokenValue();
            _bufferPos = 0;
            
            // TODO:
            _input = null;
        }