private bool QuickConvertToInt32(CompoundTokenDetails details, ParsingContext context)
{
int radix = GetRadix(details);
if (radix == 10 && details.Body.Length > 10)
{
return(false); //10 digits is maximum for int32; int32.MaxValue = 2 147 483 647
}
try {
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
int iValue = 0;
if (radix == 10)
{
iValue = Convert.ToInt32(details.Body, context.Culture);
}
else
{
iValue = Convert.ToInt32(details.Body, radix);
}
details.Value = iValue;
return(true);
} catch {
return(false);
}
}//method
}//method
private void AssignTypeCodes(CompoundTokenDetails details)
{
//Type could be assigned when we read suffix; if so, just exit
if (details.TypeCodes != null)
{
return;
}
//Decide on float types
var hasDot = details.IsSet((short)(NumberFlagsInternal.HasDot));
var hasExp = details.IsSet((short)(NumberFlagsInternal.HasExp));
var isFloat = (hasDot || hasExp);
if (!isFloat)
{
details.TypeCodes = DefaultIntTypes;
return;
}
//so we have a float. If we have exponent symbol then use it to select type
if (hasExp)
{
TypeCode code;
if (_exponentsTable.TryGetValue(details.ExponentSymbol[0], out code))
{
details.TypeCodes = new TypeCode[] { code };
return;
}
}//if hasExp
//Finally assign default float type
details.TypeCodes = new TypeCode[] { DefaultFloatType };
}
protected virtual void ReadPrefix(ISourceStream source, CompoundTokenDetails details)
{
if (!_prefixesFirsts.Contains(source.PreviewChar))
{
return;
}
var comparisonType = CaseSensitivePrefixesSuffixes ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
foreach (string pfx in Prefixes)
{
// Prefixes are usually case insensitive, even if language is case-sensitive. So we cannot use source.MatchSymbol here,
// we need case-specific comparison
if (string.Compare(source.Text, source.PreviewPosition, pfx, 0, pfx.Length, comparisonType) != 0)
{
continue;
}
//We found prefix
details.Prefix = pfx;
source.PreviewPosition += pfx.Length;
//Set flag from prefix
short pfxFlags;
if (!string.IsNullOrEmpty(details.Prefix) && PrefixFlags.TryGetValue(details.Prefix, out pfxFlags))
{
details.Flags |= (short)pfxFlags;
}
return;
} //foreach
} //method
}//method
private bool TryConvertToLong(CompoundTokenDetails details, bool useULong, ParsingContext context)
{
try {
int radix = GetRadix(details);
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
{
if (useULong)
{
details.Value = Convert.ToUInt64(details.Body, context.Culture);
}
else
{
details.Value = Convert.ToInt64(details.Body, context.Culture);
}
}
else
if (useULong)
{
details.Value = Convert.ToUInt64(details.Body, radix);
}
else
{
details.Value = Convert.ToInt64(details.Body, radix);
}
return(true);
} catch (OverflowException) {
details.Error = string.Format(Resources.ErrCannotConvertValueToType, details.Value, TypeCode.Int64.ToString());
return(false);
}
}
protected virtual void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
if (!_suffixesFirsts.Contains(source.PreviewChar))
{
return;
}
var comparisonType = CaseSensitivePrefixesSuffixes ? StringComparison.InvariantCulture : StringComparison.InvariantCultureIgnoreCase;
foreach (string sfx in Suffixes)
{
//Suffixes are usually case insensitive, even if language is case-sensitive. So we cannot use source.MatchSymbol here,
// we need case-specific comparison
if (string.Compare(source.GetText(source.PreviewPosition, sfx.Length), 0, sfx, 0, sfx.Length, comparisonType) != 0)
{
continue;
}
//We found suffix
details.Suffix = sfx;
source.PreviewPosition += sfx.Length;
//Set TypeCode from suffix
TypeCode[] codes;
if (!string.IsNullOrEmpty(details.Suffix) && SuffixTypeCodes.TryGetValue(details.Suffix, out codes))
{
details.TypeCodes = codes;
}
return;
} //foreach
} //method
}//method
private char ReadUnicodeEscape(ISourceStream source, CompoundTokenDetails details)
{
//Position is currently at "\" symbol
source.PreviewPosition++; //move to U/u char
int len;
switch (source.PreviewChar)
{
case 'u': len = 4; break;
case 'U': len = 8; break;
default:
details.Error = O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrInvEscSymbol; // "Invalid escape symbol, expected 'u' or 'U' only."
return('\0');
}
if (source.PreviewPosition + len > source.Text.Length)
{
details.Error = O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrInvEscSeq; // "Invalid escape sequence";
return('\0');
}
source.PreviewPosition++; //move to the first digit
string digits = source.Text.Substring(source.PreviewPosition, len);
char result = (char)Convert.ToUInt32(digits, 16);
source.PreviewPosition += len;
details.Flags |= (int)IdFlagsInternal.HasEscapes;
return(result);
}
private char ReadUnicodeEscape(ISourceStream source, CompoundTokenDetails details)
{
//Position is currently at "\" symbol
source.Position++; //move to U/u char
int len;
switch (source.CurrentChar)
{
case 'u': len = 4; break;
case 'U': len = 8; break;
default:
details.Error = "Invalid escape symbol, expected 'u' or 'U' only.";
return('\0');
}
if (source.Position + len > source.Text.Length)
{
details.Error = "Invalid escape symbol";
return('\0');
}
source.Position++; //move to the first digit
string digits = source.Text.Substring(source.Position, len);
char result = (char)Convert.ToUInt32(digits, 16);
source.Position += len;
details.Flags |= (int)IdFlags.HasEscapes;
return(result);
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
Token token;
//Try quick parse first, but only if we're not continuing
if (context.ScannerState.Value == 0)
{
token = QuickParse(context, source);
if (token != null)
{
return(token);
}
source.Position = source.TokenStart.Position; //revert the position
}
CompoundTokenDetails details = new CompoundTokenDetails();
InitDetails(context, details);
if (context.ScannerState.Value == 0)
{
ReadPrefix(source, details);
}
if (!ReadBody(source, details))
{
return(null);
}
if (details.Error != null)
{
return(context.CreateErrorTokenAndReportError(source.TokenStart, source.CurrentChar.ToString(), details.Error));
}
if (details.IsPartial)
{
details.Value = details.Body;
}
else
{
ReadSuffix(source, details);
if (!ConvertValue(details))
{
return(context.CreateErrorTokenAndReportError(source.TokenStart, source.CurrentChar.ToString(), "Failed to convert the value: " + details.Error));
}
}
token = CreateToken(context, source, details);
if (details.IsPartial)
{
//Save terminal state so we can continue
context.ScannerState.TokenSubType = (byte)details.SubTypeIndex;
context.ScannerState.TerminalFlags = (short)details.Flags;
context.ScannerState.TerminalIndex = this.MultilineIndex;
}
else
{
context.ScannerState.Value = 0;
}
return(token);
}
}//method
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
base.ReadSuffix(source, details);
if (string.IsNullOrEmpty(details.Suffix))
{
details.TypeCodes = details.IsSet((short)(NumberFlags.HasDot | NumberFlags.HasExp)) ? _defaultFloatTypes : DefaultIntTypes;
}
}
private void ProcessPartialBody(ISourceStream source, CompoundTokenDetails details)
{
int from = source.PreviewPosition;
source.PreviewPosition = source.Text.Length;
details.Body = source.Text.Substring(from, source.PreviewPosition - from);
details.IsPartial = true;
}
private bool ConvertToBigInteger(CompoundTokenDetails details)
{
//ignore leading zeros and sign
details.Body = details.Body.TrimStart('+').TrimStart('-').TrimStart('0');
if (string.IsNullOrEmpty(details.Body))
{
details.Body = "0";
}
int bodyLength = details.Body.Length;
int radix = GetRadix(details);
int wordLength = GetSafeWordLength(details);
int sectionCount = GetSectionCount(bodyLength, wordLength);
ulong[] numberSections = new ulong[sectionCount]; //big endian
try
{
int startIndex = details.Body.Length - wordLength;
for (int sectionIndex = sectionCount - 1; sectionIndex >= 0; sectionIndex--)
{
if (startIndex < 0)
{
wordLength += startIndex;
startIndex = 0;
}
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
{
numberSections[sectionIndex] = Convert.ToUInt64(details.Body.Substring(startIndex, wordLength));
}
else
{
numberSections[sectionIndex] = Convert.ToUInt64(details.Body.Substring(startIndex, wordLength), radix);
}
startIndex -= wordLength;
}
}
catch
{
details.Error = Resources.ErrInvNumber;// "Invalid number.";
return(false);
}
//produce big integer
ulong safeWordRadix = GetSafeWordRadix(details);
BigInteger bigIntegerValue = numberSections[0];
for (int i = 1; i < sectionCount; i++)
{
bigIntegerValue = checked (bigIntegerValue * safeWordRadix + numberSections[i]);
}
if (details.Sign == "-")
{
bigIntegerValue = -bigIntegerValue;
}
details.Value = bigIntegerValue;
return(true);
}
protected override void ReadPrefix(ISourceStream source, CompoundTokenDetails details)
{
//check that is not a 0 followed by dot;
//this may happen in Python for number "0.123" - we can mistakenly take "0" as octal prefix
if (source.PreviewChar == '0' && source.NextPreviewChar == '.')
{
return;
}
base.ReadPrefix(source, details);
}//method
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
if (!details.PartialContinues)
{
if (!ReadStartSymbol(source, details))
{
return(false);
}
}
return(CompleteReadBody(source, details));
}
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
int start = source.PreviewPosition;
bool allowEscapes = details.IsSet((short)IdOptions.AllowsEscapes);
CharList outputChars = new CharList();
while (!source.EOF())
{
char current = source.PreviewChar;
if (Grammar.IsWhitespaceOrDelimiter(current))
{
break;
}
if (allowEscapes && current == this.EscapeChar)
{
current = ReadUnicodeEscape(source, details);
//We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
//This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
// char is at position of last digit of escape sequence.
source.PreviewPosition--;
if (details.Error != null)
{
return(false);
}
}
//Check if current character is OK
if (!CharOk(current, source.PreviewPosition == start))
{
break;
}
//Check if we need to skip this char
#if NETSTANDARD
UnicodeCategory currCat = CharUnicodeInfo.GetUnicodeCategory(current);
#else
UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
#endif
if (!this.CharsToRemoveCategories.Contains(currCat))
{
outputChars.Add(current); //add it to output (identifier)
}
source.PreviewPosition++;
}//while
if (outputChars.Count == 0)
{
return(false);
}
//Convert collected chars to string
details.Body = new string(outputChars.ToArray());
if (!CheckCaseRestriction(details.Body))
{
return(false);
}
return(!string.IsNullOrEmpty(details.Body));
}
protected override bool ConvertValue(CompoundTokenDetails details)
{
if (details.IsSet((short)IdOptions.NameIncludesPrefix))
{
details.Value = details.Prefix + details.Body;
}
else
{
details.Value = details.Body;
}
return(true);
}
protected override void InitDetails(ParsingContext context, CompoundTokenDetails details)
{
base.InitDetails(context, details);
if (context.VsLineScanState.Value == 0)
{
return;
}
//we are continuing partial string on the next line
details.Flags = context.VsLineScanState.TerminalFlags;
details.SubTypeIndex = context.VsLineScanState.TokenSubType;
var stringInfo = _subTypes[context.VsLineScanState.TokenSubType];
details.StartSymbol = stringInfo.Start;
}
private int GetRadix(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(16);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(8);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(2);
}
return(10);
}
private string GetDigits(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(Strings.HexDigits);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(Strings.OctalDigits);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(Strings.BinaryDigits);
}
return(Strings.DecimalDigits);
}
private int GetSafeWordLength(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(15);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(21); //maxWordLength 22
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(63);
}
return(19); //maxWordLength 20
}
//radix^safeWordLength
private ulong GetSafeWordRadix(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(1152921504606846976);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(9223372036854775808);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(9223372036854775808);
}
return(10000000000000000000);
}
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
base.ReadSuffix(source, details);
//"char" type can be identified by suffix (like VB where c suffix identifies char)
// in this case we have details.TypeCodes[0] == char and we need to set the IsChar flag
if (details.TypeCodes != null && details.TypeCodes[0] == TypeCode.Char)
{
details.Flags |= (int)StringOptions.IsChar;
}
else if (details.IsSet((short)StringOptions.IsChar))
{
//we may have IsChar flag set (from startEndSymbol, like in c# single quote identifies char)
// in this case set type code
details.TypeCodes = new[] { TypeCode.Char };
}
}
private bool TryCastToIntegerType(TypeCode typeCode, CompoundTokenDetails details)
{
if (details.Value == null)
{
return(false);
}
try {
if (typeCode != TypeCode.UInt64)
{
details.Value = Convert.ChangeType(details.Value, typeCode, CultureInfo.InvariantCulture);
}
return(true);
} catch (Exception) {
details.Error = string.Format(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrCannotConvertValueToType, details.Value, typeCode.ToString());
return(false);
}
}//method
//Extract the string content from lexeme, adjusts the escaped and double-end symbols
protected override bool ConvertValue(CompoundTokenDetails details)
{
var value = details.Body;
var escapeEnabled = !details.IsSet((short)HereDocOptions.NoEscapes);
//Fix all escapes
if (escapeEnabled && value.IndexOf(EscapeChar) >= 0)
{
details.Flags |= (int)StringFlagsInternal.HasEscapes;
var arr = value.Split(EscapeChar);
var ignoreNext = false;
//we skip the 0 element as it is not preceeded by "\"
for (var i = 1; i < arr.Length; i++)
{
if (ignoreNext)
{
ignoreNext = false;
continue;
}
var s = arr[i];
if (string.IsNullOrEmpty(s))
{
//it is "\\" - escaped escape symbol.
arr[i] = @"\";
ignoreNext = true;
continue;
}
//The char is being escaped is the first one; replace it with char in Escapes table
var first = s[0];
if (Escapes.TryGetValue(first, out char newFirst))
{
arr[i] = newFirst + s.Substring(1);
}
else
{
arr[i] = HandleSpecialEscape(arr[i], details);
}
}
value = string.Join(string.Empty, arr);
}
details.TypeCodes = new[] { TypeCode.String };
details.Value = value;
return(true);
}
private bool TryCastToIntegerType(TypeCode typeCode, CompoundTokenDetails details, ParsingContext context)
{
if (details.Value == null)
{
return(false);
}
try {
if (typeCode != TypeCode.UInt64)
{
details.Value = Convert.ChangeType(details.Value, typeCode, context.Culture);
}
return(true);
} catch (Exception) {
details.Error = string.Format(Resources.ErrCannotConvertValueToType, details.Value, typeCode.ToString());
return(false);
}
}//method
private bool TryCastToIntegerType(TypeCode typeCode, CompoundTokenDetails details)
{
if (details.Value == null)
{
return(false);
}
try {
if (typeCode != TypeCode.UInt64)
{
details.Value = Convert.ChangeType(details.Value, typeCode, CultureInfo.InvariantCulture);
}
return(true);
} catch (Exception e) {
Trace.WriteLine("Error converting to integer: text=[" + details.Body + "], type=" + typeCode + ", error: " + e.Message);
return(false);
}
}//method
protected override string HandleSpecialEscape(string segment, CompoundTokenDetails details)
{
if (string.IsNullOrEmpty(segment))
{
return(string.Empty);
}
char first = segment[0];
switch (first)
{
case 'a':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
case 'v':
case '\\':
case '"':
case '\'':
break;
case '0':
case '1':
case '2':
{
bool success = false;
if (segment.Length >= 3)
{
//Verify that a numeric escape is 3 characters
string value = segment.Substring(0, 3);
int dummy = 0;
success = Int32.TryParse(value, out dummy);
}
if (!success)
{
details.Error = "Invalid escape sequence: \000 must be a valid number.";
}
}
break;
}
details.Error = "Invalid escape sequence: \\" + segment;
return(segment);
}
}//method
private bool TryConvertToUlong(CompoundTokenDetails details)
{
try {
int radix = GetRadix(details);
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
{
details.Value = Convert.ToUInt64(details.Body, CultureInfo.InvariantCulture);
}
else
{
details.Value = Convert.ToUInt64(details.Body, radix);
}
return(true);
} catch (OverflowException) {
return(false);
}
}
protected override bool ConvertValue(CompoundTokenDetails details, ParsingContext context)
{
if (base.ConvertValue(details, context) || details.Error != null)
{
return(true);
}
switch (details.TypeCodes[0])
{
case TypeCode.Double:
if (QuickConvertToDouble(details))
{
return(true);
}
break;
}
return(false);
}
}//method
private bool QuickConvertToDouble(CompoundTokenDetails details)
{
if (details.IsSet((short)(NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex | NumberFlags.HasExp)))
{
return(false);
}
if (DecimalSeparator != '.')
{
return(false);
}
double dvalue;
if (!double.TryParse(details.Body, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out dvalue))
{
return(false);
}
details.Value = dvalue;
return(true);
}
}//method
private bool TryConvertToUlong(CompoundTokenDetails details)
{
try {
int radix = GetRadix(details);
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
{
details.Value = Convert.ToUInt64(details.Body, CultureInfo.InvariantCulture);
}
else
{
details.Value = Convert.ToUInt64(details.Body, radix);
}
return(true);
} catch (OverflowException) {
details.Error = string.Format(O2_Misc_Microsoft_MPL_Libs.Irony_Parser.Resources.ErrCannotConvertValueToType, details.Value, TypeCode.UInt64.ToString());
return(false);
}
}
protected override string HandleSpecialEscape(string segment, CompoundTokenDetails details)
{
if (string.IsNullOrEmpty(segment)) return string.Empty;
char first = segment[0];
switch (first)
{
case 'a':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
case 'v':
case '\\':
case '"':
case '\'':
break;
case '0':
case '1':
case '2':
{
bool success = false;
if (segment.Length >=3)
{
//Verify that a numeric escape is 3 characters
string value = segment.Substring(0, 3);
int dummy = 0;
success = Int32.TryParse(value, out dummy);
}
if(!success)
details.Error = "Invalid escape sequence: \000 must be a valid number.";
}
break;
}
details.Error = "Invalid escape sequence: \\" + segment;
return segment;
}
}//method
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details) {
base.ReadSuffix(source, details);
if (string.IsNullOrEmpty(details.Suffix))
details.TypeCodes = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp)) ? _defaultFloatTypes : DefaultIntTypes;
}
private bool ConvertToBigInteger(CompoundTokenDetails details) {
//ignore leading zeros and sign
details.Body = details.Body.TrimStart('+').TrimStart('-').TrimStart('0');
int bodyLength = details.Body.Length;
int radix = GetRadix(details);
int wordLength = GetSafeWordLength(details);
int sectionCount = GetSectionCount(bodyLength, wordLength);
ulong[] numberSections = new ulong[sectionCount]; //big endian
try {
int startIndex = details.Body.Length - wordLength;
for (int sectionIndex = sectionCount - 1; sectionIndex >= 0; sectionIndex--) {
if (startIndex < 0) {
wordLength += startIndex;
startIndex = 0;
}
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
numberSections[sectionIndex] = Convert.ToUInt64(details.Body.Substring(startIndex, wordLength));
else
numberSections[sectionIndex] = Convert.ToUInt64(details.Body.Substring(startIndex, wordLength), radix);
startIndex -= wordLength;
}
} catch {
details.Error = "Invalid number.";
return false;
}
//produce big integer
ulong safeWordRadix = GetSafeWordRadix(details);
BigInteger bigIntegerValue = numberSections[0];
for (int i = 1; i < sectionCount; i++)
bigIntegerValue = checked(bigIntegerValue * safeWordRadix + numberSections[i]);
if (details.Sign == "-")
bigIntegerValue = -bigIntegerValue;
details.Value = bigIntegerValue;
return true;
}
protected override void InitDetails(CompilerContext context, CompoundTokenDetails details) {
base.InitDetails(context, details);
details.Flags = (short) this.Flags;
}
protected override void ReadPrefix(ISourceStream source, CompoundTokenDetails details) {
//check that is not a 0 followed by dot;
//this may happen in Python for number "0.123" - we can mistakenly take "0" as octal prefix
if (source.CurrentChar == '0' && source.NextChar == '.') return;
base.ReadPrefix(source, details);
}//method
protected static bool FlagIsSet(CompoundTokenDetails details, NumberFlags flag)
{
return (details.Flags & (int) flag ) != 0;
}
private int GetRadix(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 16;
if (details.IsSet((short)NumberFlags.Octal))
return 8;
if (details.IsSet((short)NumberFlags.Binary))
return 2;
return 10;
}
private string GetDigits(CompoundTokenDetails details)
{
if (FlagIsSet(details, NumberFlags.Hex))
return TextUtils.HexDigits;
if (FlagIsSet(details, NumberFlags.Octal))
return TextUtils.OctalDigits;
if (FlagIsSet(details, NumberFlags.Binary))
return TextUtils.BinaryDigits;
return TextUtils.DecimalDigits;
}
private bool ConvertToFloat(TypeCode typeCode, CompoundTokenDetails details) {
//only decimal numbers can be fractions
if (details.IsSet((short)(NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex))) {
details.Error = "Invalid number.";
return false;
}
string body = details.Body;
//Some languages allow exp symbols other than E. Check if it is the case, and change it to E
// - otherwise .NET conversion methods may fail
if (details.IsSet((short)NumberFlags.HasExp) && details.ExponentSymbol.ToUpper() != "E")
body = body.Replace(details.ExponentSymbol, "E");
//'.' decimal seperator required by invariant culture
if (details.IsSet((short)NumberFlags.HasDot) && DecimalSeparator != '.')
body = body.Replace(DecimalSeparator, '.');
switch (typeCode) {
case TypeCode.Double:
case TypeCodeImaginary:
double dValue;
if (!Double.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out dValue)) return false;
if (typeCode == TypeCodeImaginary)
details.Value = new Complex64(0, dValue);
else
details.Value = dValue;
return true;
case TypeCode.Single:
float fValue;
if (!Single.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out fValue)) return false;
details.Value = fValue;
return true;
case TypeCode.Decimal:
decimal decValue;
if (!Decimal.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out decValue)) return false;
details.Value = decValue;
return true;
}//switch
return false;
}
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details) {
//remember start - it may be different from source.TokenStart, we may have skipped prefix
int start = source.Position;
char current = source.CurrentChar;
if (current == '-' || current == '+') {
details.Sign = current.ToString();
source.Position++;
}
//Figure out digits set
string digits = GetDigits(details);
bool isDecimal = !details.IsSet((short) (NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex));
bool allowFloat = !IsSet(NumberFlags.IntOnly);
bool foundDigits = false;
while (!source.EOF()) {
current = source.CurrentChar;
//1. If it is a digit, just continue going
if (digits.IndexOf(current) >= 0) {
source.Position++;
foundDigits = true;
continue;
}
//2. Check if it is a dot in float number
bool isDot = current == DecimalSeparator;
if (allowFloat && isDot) {
//If we had seen already a dot or exponent, don't accept this one;
bool hasDotOrExp = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp));
if (hasDotOrExp) break; //from while loop
//In python number literals (NumberAllowPointFloat) a point can be the first and last character,
//We accept dot only if it is followed by a digit
if (digits.IndexOf(source.NextChar) < 0 && !IsSet(NumberFlags.AllowStartEndDot))
break; //from while loop
details.Flags |= (int) NumberFlags.HasDot;
source.Position++;
continue;
}
//3. Check if it is int number followed by dot or exp symbol
bool isExpSymbol = (details.ExponentSymbol == null) && ExponentSymbols.IndexOf(current) >= 0;
if (!allowFloat && foundDigits && (isDot || isExpSymbol)) {
//If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
if (IsSet(NumberFlags.AvoidPartialFloat)) return false;
//otherwise break, it is integer and we're done reading digits
break;
}
//4. Only for decimals - check if it is (the first) exponent symbol
if (allowFloat && isDecimal && isExpSymbol) {
char next = source.NextChar;
bool nextIsSign = next == '-' || next == '+';
bool nextIsDigit = digits.IndexOf(next) >= 0;
if (!nextIsSign && !nextIsDigit)
break; //Exponent should be followed by either sign or digit
//ok, we've got real exponent
details.ExponentSymbol = current.ToString(); //remember the exp char
details.Flags |= (int) NumberFlags.HasExp;
source.Position++;
if (nextIsSign)
source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
continue;
}
//4. It is something else (not digit, not dot or exponent) - we're done
break; //from while loop
}//while
int end = source.Position;
if (!foundDigits)
return false;
details.Body = source.Text.Substring(start, end - start);
return true;
}
private string GetDigits(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return Strings.HexDigits;
if (details.IsSet((short)NumberFlags.Octal))
return Strings.OctalDigits;
if (details.IsSet((short)NumberFlags.Binary))
return Strings.BinaryDigits;
return Strings.DecimalDigits;
}
protected override bool ConvertValue(CompoundTokenDetails details) {
if (String.IsNullOrEmpty(details.Body)) {
details.Error = "Invalid number.";
return false;
}
//Try quick paths
switch (details.TypeCodes[0]) {
case TypeCode.Int32:
if (QuickConvertToInt32(details)) return true;
break;
case TypeCode.Double:
if (QuickConvertToDouble(details)) return true;
break;
}
//Go full cycle
details.Value = null;
foreach (TypeCode typeCode in details.TypeCodes) {
switch (typeCode) {
case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: case TypeCodeImaginary:
return ConvertToFloat(typeCode, details);
case TypeCode.SByte: case TypeCode.Byte: case TypeCode.Int16: case TypeCode.UInt16:
case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64:
if (details.Value == null) //if it is not done yet
TryConvertToUlong(details); //try to convert to ULong and place the result into details.Value field;
if(TryCastToIntegerType(typeCode, details)) //now try to cast the ULong value to the target type
return true;
break;
case TypeCodeBigInt:
if (ConvertToBigInteger(details)) return true;
break;
}//switch
}
return false;
}//method
}//method
#endregion
#region private utilities
private bool QuickConvertToInt32(CompoundTokenDetails details) {
int radix = GetRadix(details);
if (radix == 10 && details.Body.Length > 10) return false; //10 digits is maximum for int32; int32.MaxValue = 2 147 483 647
try {
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
int iValue = 0;
if (radix == 10)
iValue = Convert.ToInt32(details.Body, CultureInfo.InvariantCulture);
else
iValue = Convert.ToInt32(details.Body, radix);
details.Value = iValue;
return true;
} catch {
return false;
}
}//method
}//method
private bool QuickConvertToDouble(CompoundTokenDetails details) {
if (details.IsSet((short)(NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex | NumberFlags.HasExp))) return false;
if (DecimalSeparator != '.') return false;
double dvalue;
if (!double.TryParse(details.Body, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out dvalue)) return false;
details.Value = dvalue;
return true;
}
private int GetSafeWordLength(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 15;
if (details.IsSet((short)NumberFlags.Octal))
return 21; //maxWordLength 22
if (details.IsSet((short)NumberFlags.Binary))
return 63;
return 19; //maxWordLength 20
}
private bool TryCastToIntegerType(TypeCode typeCode, CompoundTokenDetails details) {
if (details.Value == null) return false;
try {
if (typeCode != TypeCode.UInt64)
details.Value = Convert.ChangeType(details.Value, typeCode, CultureInfo.InvariantCulture);
return true;
} catch (Exception e) {
Trace.WriteLine("Error converting to integer: text=[" + details.Body + "], type=" + typeCode + ", error: " + e.Message);
return false;
}
}//method
}//method
private bool TryConvertToUlong(CompoundTokenDetails details) {
try {
int radix = GetRadix(details);
//workaround for .Net FX bug: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=278448
if (radix == 10)
details.Value = Convert.ToUInt64(details.Body, CultureInfo.InvariantCulture);
else
details.Value = Convert.ToUInt64(details.Body, radix);
return true;
} catch(OverflowException) {
return false;
}
}
//radix^safeWordLength
private ulong GetSafeWordRadix(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 1152921504606846976;
if (details.IsSet((short)NumberFlags.Octal))
return 9223372036854775808;
if (details.IsSet((short) NumberFlags.Binary))
return 9223372036854775808;
return 10000000000000000000;
}