protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
//remember start - it may be different from source.TokenStart, we may have skipped
int start = source.Position;
//Figure out digits set
string digits = GetDigits(details);
bool isDecimal = !details.IsSet(ScanFlags.NonDecimal);
bool allowFloat = !IsSet(TermOptions.NumberIntOnly);
while (!source.EOF())
{
char current = source.CurrentChar;
//1. If it is a digit, just continue going
if (digits.IndexOf(current) >= 0)
{
source.Position++;
continue;
}
//2. Check if it is a dot
if (current == DecimalSeparator && allowFloat)
{
//If we had seen already a dot or exponent, don't accept this one;
//In python number literals (NumberAllowPointFloat) a point can be the first and last character,
//otherwise we accept dot only if it is followed by a digit
if (details.IsSet(ScanFlags.HasDotOrExp) || (digits.IndexOf(source.NextChar) < 0) && !IsSet(TermOptions.NumberAllowStartEndDot))
{
break; //from while loop
}
details.Flags |= ScanFlags.HasDot;
source.Position++;
continue;
}
//3. Only for decimals - check if it is (the first) exponent symbol
if (allowFloat && isDecimal && (details.ControlSymbol == null) && (ExponentSymbols.IndexOf(current) >= 0))
{
char next = source.NextChar;
bool nextIsSign = next == '-' || next == '+';
bool nextIsDigit = digits.IndexOf(next) >= 0;
if (!nextIsSign && !nextIsDigit)
{
break; //Exponent should be followed by either sign or digit
}
//ok, we've got real exponent
details.ControlSymbol = current.ToString(); //remember the exp char
details.Flags |= ScanFlags.HasExp;
source.Position++;
if (nextIsSign)
{
source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
}
continue;
}
//4. It is something else (not digit, not dot or exponent) - we're done
break; //from while loop
}//while
int end = source.Position;
details.Body = source.Text.Substring(start, end - start);
return(true);
}
//Override to assign IsKeyword flag to keyword tokens
protected override Token CreateToken(CompilerContext context, ISourceStream source, ScanDetails details)
{
if (details.IsSet(ScanFlags.IncludePrefix) && !string.IsNullOrEmpty(details.Prefix))
{
details.Value = details.Prefix + details.Body;
}
Token token = base.CreateToken(context, source, details);
if (details.IsSet(ScanFlags.IsNotKeyword))
{
return(token);
}
//check if it is keyword
string text = token.Text;
if (!Grammar.CaseSensitive)
{
text = text.ToLower();
}
if (_keywordHash.ContainsKey(text))
{
token.IsKeyword = true;
}
return(token);
}
protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
if (!ReadStartSymbol(source, details))
{
return(false);
}
bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
bool ignoreCase = IsSet(TermOptions.SpecialIgnoreCase);
int start = source.Position;
string startS = details.ControlSymbol;
string startS2 = startS + startS; //doubled start symbol
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = details.IsSet(ScanFlags.AllowLineBreak) ? -1 : source.Text.IndexOf('\n', source.Position);
while (!source.EOF())
{
int endPos = source.Text.IndexOf(startS, source.Position);
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.Position = endPos + 1;
}
details.Error = "Mal-formed string literal - cannot find termination symbol.";
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && source.Text[endPos - 1] == EscapeChar)
{
source.Position = endPos + startS.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.Position = endPos;
if (details.IsSet(ScanFlags.AllowDoubledQuote) && source.MatchSymbol(startS2, ignoreCase))
{
source.Position = endPos + startS.Length * 2;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.Text.Substring(start, endPos - start);
source.Position = endPos + startS.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
return(false);
}
//radix^safeWordLength
private ulong GetSafeWordRadix(ScanDetails details)
{
if (details.IsSet(ScanFlags.Hex))
{
return(1152921504606846976);
}
if (details.IsSet(ScanFlags.Octal))
{
return(9223372036854775808);
}
if (details.IsSet(ScanFlags.Binary))
{
return(9223372036854775808);
}
return(10000000000000000000);
}
private int GetSafeWordLength(ScanDetails details)
{
if (details.IsSet(ScanFlags.Hex))
{
return(15);
}
if (details.IsSet(ScanFlags.Octal))
{
return(21); //maxWordLength 22
}
if (details.IsSet(ScanFlags.Binary))
{
return(63);
}
return(19); //maxWordLength 20
}
private string GetDigits(ScanDetails details)
{
if (details.IsSet(ScanFlags.Hex))
{
return(TextUtils.HexDigits);
}
if (details.IsSet(ScanFlags.Octal))
{
return(TextUtils.OctalDigits);
}
if (details.IsSet(ScanFlags.Binary))
{
return(TextUtils.BinaryDigits);
}
return(TextUtils.DecimalDigits);
}
private int GetRadix(ScanDetails details)
{
if (details.IsSet(ScanFlags.Hex))
{
return(16);
}
if (details.IsSet(ScanFlags.Octal))
{
return(8);
}
if (details.IsSet(ScanFlags.Binary))
{
return(2);
}
return(10);
}
}//method
private bool QuickConvertToDouble(ScanDetails details)
{
if (details.IsSet(ScanFlags.Binary | ScanFlags.Octal | ScanFlags.Hex | ScanFlags.HasExp))
{
return(false);
}
if (DecimalSeparator != '.')
{
return(false);
}
double result;
#if PocketPC || SILVERLIGHT
try
{
result = Convert.ToDouble(details.Body, CultureInfo.InvariantCulture);
}
catch
{
return(false);
}
#else
if (!double.TryParse(details.Body, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out result))
{
return(false);
}
#endif
details.Value = result;
return(true);
}
}//method
protected override void ReadSuffix(ISourceStream source, ScanDetails details)
{
base.ReadSuffix(source, details);
if (string.IsNullOrEmpty(details.Suffix))
{
details.TypeCodes = details.IsSet(ScanFlags.HasDotOrExp) ? _defaultFloatTypes : DefaultIntTypes;
}
}
protected override bool ConvertValue(ScanDetails details)
{
if (details.IsSet(ScanFlags.IncludePrefix))
{
details.Value = details.Prefix + details.Body;
}
else
{
details.Value = details.Body;
}
return(true);
}
protected override bool ReadBody(ISourceStream source, ScanDetails details)
{
int start = source.Position;
bool allowEscapes = !details.IsSet(ScanFlags.DisableEscapes);
CharList outputChars = new CharList();
while (!source.EOF())
{
char current = source.CurrentChar;
if (_terminators.IndexOf(current) >= 0)
{
break;
}
if (allowEscapes && current == this.EscapeChar)
{
current = ReadUnicodeEscape(source, details);
//We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
//This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
// char is at position of last digit of escape sequence.
source.Position--;
if (details.HasError())
{
return(false);
}
}
//Check if current character is OK
if (!CharOk(current, source.Position == start))
{
break;
}
//Check if we need to skip this char
UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
if (!this.CharsToRemoveCategories.Contains(currCat))
{
outputChars.Add(current); //add it to output (identifier)
}
source.Position++;
}//while
if (outputChars.Count == 0)
{
return(false);
}
//Convert collected chars to string
details.Body = new string(outputChars.ToArray());
return(!string.IsNullOrEmpty(details.Body));
}
private bool ConvertToFloat(TypeCode typeCode, ScanDetails details)
{
//only decimal numbers can be fractions
if (details.IsSet(ScanFlags.Binary | ScanFlags.Octal | ScanFlags.Hex))
{
details.Error = "Invalid number.";
return(false);
}
string body = details.Body;
//Some languages allow exp symbols other than E. Check if it is the case, and change it to E
// - otherwise .NET conversion methods may fail
if (details.IsSet(ScanFlags.HasExp) && details.ControlSymbol.ToUpper() != "E")
{
body = body.Replace(details.ControlSymbol, "E");
}
//'.' decimal seperator required by invariant culture
if (details.IsSet(ScanFlags.HasDot) && DecimalSeparator != '.')
{
body = body.Replace(DecimalSeparator, '.');
}
switch (typeCode)
{
case TypeCode.Double:
case TypeCodeImaginary:
double dValue;
#if PocketPC || SILVERLIGHT
try
{
dValue = Convert.ToDouble(body, CultureInfo.InvariantCulture);
}
catch
{
return(false);
}
#else
if (!Double.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out dValue))
{
return(false);
}
#endif
if (typeCode == TypeCodeImaginary)
{
details.Value = new Complex64(0, dValue);
}
else
{
details.Value = dValue;
}
return(true);
case TypeCode.Single:
float fValue;
#if PocketPC || SILVERLIGHT
try
{
fValue = Convert.ToSingle(body, CultureInfo.InvariantCulture);
}
catch
{
return(false);
}
#else
if (!Single.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out fValue))
{
return(false);
}
#endif
details.Value = fValue;
return(true);
case TypeCode.Decimal:
decimal decValue;
#if PocketPC || SILVERLIGHT
try
{
decValue = Convert.ToDecimal(body, CultureInfo.InvariantCulture);
}
catch
{
return(false);
}
#else
if (!Decimal.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out decValue))
{
return(false);
}
#endif
details.Value = decValue;
return(true);
}//switch
return(false);
}
//Should support: \Udddddddd, \udddd, \xdddd, \N{name}, \0, \ddd (octal),
protected virtual string HandleSpecialEscape(string segment, ScanDetails details)
{
if (string.IsNullOrEmpty(segment))
{
return(string.Empty);
}
int len, p; string digits; char ch; string result;
char first = segment[0];
switch (first)
{
case 'u':
case 'U':
if (details.IsSet(ScanFlags.AllowUEscapes))
{
len = (first == 'u' ? 4 : 8);
if (segment.Length < len + 1)
{
details.Error = "Invalid unicode escape (" + segment.Substring(len + 1) + "), expected " + len + " hex digits.";
return(segment);
}
digits = segment.Substring(1, len);
ch = (char)Convert.ToUInt32(digits, 16);
result = ch + segment.Substring(len + 1);
return(result);
}//if
break;
case 'x':
if (details.IsSet(ScanFlags.AllowXEscapes))
{
//x-escape allows variable number of digits, from one to 4; let's count them
p = 1; //current position
while (p < 5 && p < segment.Length)
{
if (TextUtils.HexDigits.IndexOf(segment[p]) < 0)
{
break;
}
p++;
}
//p now point to char right after the last digit
if (p <= 1)
{
details.Error = "Invalid \\x escape, at least one digit expected.";
return(segment);
}
digits = segment.Substring(1, p - 1);
ch = (char)Convert.ToUInt32(digits, 16);
result = ch + segment.Substring(p);
return(result);
}//if
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
if (details.IsSet(ScanFlags.AllowOctalEscapes))
{
//octal escape allows variable number of digits, from one to 3; let's count them
p = 0; //current position
while (p < 3 && p < segment.Length)
{
if (TextUtils.OctalDigits.IndexOf(segment[p]) < 0)
{
break;
}
p++;
}
//p now point to char right after the last digit
digits = segment.Substring(0, p);
ch = (char)Convert.ToUInt32(digits, 8);
result = ch + segment.Substring(p);
return(result);
} //if
break;
} //switch
details.Error = "Invalid escape sequence: \\" + segment;
return(segment);
}//method
} //method
//Extract the string content from lexeme, adjusts the escaped and double-end symbols
protected override bool ConvertValue(ScanDetails details)
{
string value = details.Body;
bool escapeEnabled = !details.IsSet(ScanFlags.DisableEscapes);
//Fix all escapes
if (escapeEnabled && value.IndexOf(EscapeChar) >= 0)
{
details.Flags |= ScanFlags.HasEscapes;
string[] arr = value.Split(EscapeChar);
bool ignoreNext = false;
//we skip the 0 element as it is not preceeded by "\"
for (int i = 1; i < arr.Length; i++)
{
if (ignoreNext)
{
ignoreNext = false;
continue;
}
string s = arr[i];
if (string.IsNullOrEmpty(s))
{
//it is "\\" - escaped escape symbol.
arr[i] = @"\";
ignoreNext = true;
continue;
}
//The char is being escaped is the first one; replace it with char in Escapes table
char first = s[0];
char newFirst;
if (Escapes.TryGetValue(first, out newFirst))
{
arr[i] = newFirst + s.Substring(1);
}
else
{
arr[i] = HandleSpecialEscape(arr[i], details);
} //else
} //for i
value = string.Join(string.Empty, arr);
} // if EscapeEnabled
//Check for doubled end symbol
string startS = details.ControlSymbol;
if (details.IsSet(ScanFlags.AllowDoubledQuote) && value.IndexOf(startS) >= 0)
{
value = value.Replace(startS + startS, startS);
}
if (details.IsSet(ScanFlags.IsChar))
{
details.TypeCodes = new TypeCode[] { TypeCode.Char }
}
;
//Check char length - must be exactly 1
if (details.TypeCodes[0] == TypeCode.Char && value.Length != 1)
{
details.Error = "Invalid length of char literal - should be 1.";
return(false);
}
details.Value = (details.TypeCodes[0] == TypeCode.Char ? (object)value[0] : value);
return(true);
//TODO: Investigate unescaped linebreak, with Flags == BnfFlags.StringAllowLineBreak | BnfFlags.StringLineBreakEscaped
// also investigate what happens in this case in Windows where default linebreak is "\r\n", not "\n"
}