/// <summary> /// Create a token. /// </summary> /// <param name="token">Token.</param> /// <param name="tokenValue">Token data.</param> /// <param name="tokenPosition">Token span.</param> /// <param name="tokenText">Token text.</param> public CompleteToken(Tokens token, SemanticValueType tokenValue, Span tokenPosition, string tokenText) { Token = token; TokenValue = tokenValue; TokenPosition = tokenPosition; TokenText = tokenText; }
/// <summary> /// Reads token as a number (accepts tokens with any reasonable base [0-9a-zA-Z]*). /// Parsed value is stored in <paramref name="val"/> as integer (when value is less than MaxInt), /// as Long (when value is less then MaxLong) or as double. /// </summary> /// <param name="startIndex">Starting read position of the token.</param> /// <param name="base">The base of the number.</param> /// <param name="val">Parsed value is stored in this union</param> /// <returns>Returns one of T_LNUMBER (int), T_L64NUMBER (long) or T_DNUMBER (double)</returns> protected Tokens GetTokenAsDecimalNumber(int startIndex, int @base, ref SemanticValueType val) { long lresult = 0; double dresult = 0; int digit; int buffer_pos = token_start + startIndex; // try to parse INT value // most of the literals are parsed using the following loop while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base && lresult <= Int32.MaxValue) { lresult = lresult * @base + digit; buffer_pos++; } if (lresult > Int32.MaxValue) { // try to parse LONG value (check for the overflow and if it occurs converts data to double) bool longOverflow = false; while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base) { try { lresult = checked (lresult * @base + digit); } catch (OverflowException) { longOverflow = true; break; } buffer_pos++; } if (longOverflow) { // too big for LONG - use double dresult = (double)lresult; while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base) { dresult = dresult * @base + digit; buffer_pos++; } val.Double = dresult; return(Tokens.T_DNUMBER); } else { val.Long = lresult; return(Tokens.T_L64NUMBER); } } else { val.Integer = (int)lresult; return(Tokens.T_LNUMBER); } }
/// <summary> /// Checks whether {LNUM} fits to integer, long or double /// and returns appropriate value from Tokens enum. /// </summary> protected Tokens GetIntegerTokenType(int startIndex) { int i = token_start + startIndex; while (i < token_end && buffer[i] == '0') { i++; } int number_length = token_end - i; if (i != token_start + startIndex) { // starts with zero - octal // similar to GetHexIntegerTokenType code if ((number_length < 11) || (number_length == 11 && buffer[i] == '1')) { return(Tokens.T_LNUMBER); } if (number_length < 22) { return(Tokens.T_L64NUMBER); } return(Tokens.T_DNUMBER); } else { // decimal if (number_length < 10) { return(Tokens.T_LNUMBER); } if (number_length > 19) { return(Tokens.T_DNUMBER); } if (number_length >= 11 && number_length <= 18) { return(Tokens.T_L64NUMBER); } // can't easily check for numbers of different length SemanticValueType val = default(SemanticValueType); return(GetTokenAsDecimalNumber(startIndex, 10, ref val)); } }
protected int?GetTokenAsLinePragma() { int start_offset = GetPragmaValueStart("line".Length); int sign = +1; if (GetTokenChar(start_offset) == '-') { sign = -1; start_offset++; } // TP_COMMENT: modified call to GetTokenAsDecimalNumber SemanticValueType val = default(SemanticValueType); Tokens ret = GetTokenAsDecimalNumber(start_offset, 10, ref val); // multiplication cannot overflow as ivalue >= 0 return((ret != Tokens.T_LNUMBER) ? null : (int?)(val.Integer * sign)); }
/// <summary> /// Reads token as a number (accepts tokens with any reasonable base [0-9a-zA-Z]*). /// Parsed value is stored in <paramref name="val"/> as integer (when value is less than MaxInt), /// as Long (when value is less then MaxLong) or as double. /// </summary> /// <param name="startIndex">Starting read position of the token.</param> /// <param name="base">The base of the number.</param> /// <param name="val">Parsed value is stored in this union</param> /// <returns>Returns one of T_LNUMBER (int), T_L64NUMBER (long) or T_DNUMBER (double)</returns> protected Tokens GetTokenAsDecimalNumber(int startIndex, int @base, ref SemanticValueType val) { long lresult = 0; double dresult = 0; int digit; int buffer_pos = token_start + startIndex; // try to parse INT value // most of the literals are parsed using the following loop while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base && lresult <= Int32.MaxValue) { lresult = lresult * @base + digit; buffer_pos++; } if (lresult > Int32.MaxValue) { // try to parse LONG value (check for the overflow and if it occurs converts data to double) bool longOverflow = false; while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base) { try { lresult = checked(lresult * @base + digit); } catch (OverflowException) { longOverflow = true; break; } buffer_pos++; } if (longOverflow) { // too big for LONG - use double dresult = (double)lresult; while (buffer_pos < buffer.Length && (digit = Convert.AlphaNumericToDigit(buffer[buffer_pos])) < @base) { dresult = dresult * @base + digit; buffer_pos++; } val.Double = dresult; return Tokens.T_DNUMBER; } else { val.Long = lresult; return Tokens.T_L64NUMBER; } } else { val.Integer = (int)lresult; return Tokens.T_LNUMBER; } }