protected override Literal Next()
{
char ch;
while (true)
{
int nextChar = ReaderRead();
if (nextChar == -1)
{
break;
}
Literal token = null;
switch (nextChar)
{
case ' ':
case '\t':
continue;
case '\r':
case '\n':
HandleLineEnd((char)nextChar);
continue;
case '"':
token = ReadString();
break;
case '\'':
token = ReadChar();
break;
case '@':
int next = ReaderRead();
if (next == -1)
{
Error(Line, Col, String.Format("EOF after @"));
continue;
}
else
{
int x = Col - 1;
int y = Line;
ch = (char)next;
if (ch == '"')
{
token = ReadVerbatimString();
}
else if (Char.IsLetterOrDigit(ch) || ch == '_')
{
bool canBeKeyword;
string s = ReadIdent(ch, out canBeKeyword);
return(new Literal(null, null, LiteralFormat.None));
}
else
{
HandleLineEnd(ch);
Error(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
continue;
}
}
break;
default: // non-ws chars are handled here
ch = (char)nextChar;
if (Char.IsLetter(ch) || ch == '_' || ch == '\\')
{
int x = Col - 1; // Col was incremented above, but we want the start of the identifier
int y = Line;
bool canBeKeyword;
string s = ReadIdent(ch, out canBeKeyword);
return(new Literal(null, null, LiteralFormat.None));
}
else if (Char.IsDigit(ch))
{
token = ReadDigit(ch, Col - 1);
}
break;
}
// try error recovery (token = null -> continue with next char)
if (token != null)
{
return(token);
}
}
return(new Literal(null, null, LiteralFormat.None));
}
Literal ReadDigit(char ch, int x)
{
unchecked { // prevent exception when ReaderPeek() = -1 is cast to char
int y = Line;
sb.Length = 0;
sb.Append(ch);
string prefix = null;
string suffix = null;
bool ishex = false;
bool isunsigned = false;
bool islong = false;
bool isfloat = false;
bool isdouble = false;
bool isdecimal = false;
char peek = (char)ReaderPeek();
if (ch == '.')
{
isdouble = true;
while (Char.IsDigit((char)ReaderPeek())) // read decimal digits beyond the dot
{
sb.Append((char)ReaderRead());
}
peek = (char)ReaderPeek();
}
else if (ch == '0' && (peek == 'x' || peek == 'X'))
{
ReaderRead(); // skip 'x'
sb.Length = 0; // Remove '0' from 0x prefix from the stringvalue
while (IsHex((char)ReaderPeek()))
{
sb.Append((char)ReaderRead());
}
if (sb.Length == 0)
{
sb.Append('0'); // dummy value to prevent exception
Error(y, x, "Invalid hexadecimal integer literal");
}
ishex = true;
prefix = "0x";
peek = (char)ReaderPeek();
}
else
{
while (Char.IsDigit((char)ReaderPeek()))
{
sb.Append((char)ReaderRead());
}
peek = (char)ReaderPeek();
}
Literal nextToken = null; // if we accidently read a 'dot'
if (peek == '.') // read floating point number
{
ReaderRead();
peek = (char)ReaderPeek();
if (!Char.IsDigit(peek))
{
nextToken = new Literal(null, null, LiteralFormat.None);
peek = '.';
}
else
{
isdouble = true; // double is default
if (ishex)
{
Error(y, x, "No hexadecimal floating point values allowed");
}
sb.Append('.');
while (Char.IsDigit((char)ReaderPeek())) // read decimal digits beyond the dot
{
sb.Append((char)ReaderRead());
}
peek = (char)ReaderPeek();
}
}
if (peek == 'e' || peek == 'E') // read exponent
{
isdouble = true;
sb.Append((char)ReaderRead());
peek = (char)ReaderPeek();
if (peek == '-' || peek == '+')
{
sb.Append((char)ReaderRead());
}
while (Char.IsDigit((char)ReaderPeek())) // read exponent value
{
sb.Append((char)ReaderRead());
}
isunsigned = true;
peek = (char)ReaderPeek();
}
if (peek == 'f' || peek == 'F') // float value
{
ReaderRead();
suffix = "f";
isfloat = true;
}
else if (peek == 'd' || peek == 'D') // double type suffix (obsolete, double is default)
{
ReaderRead();
suffix = "d";
isdouble = true;
}
else if (peek == 'm' || peek == 'M') // decimal value
{
ReaderRead();
suffix = "m";
isdecimal = true;
}
else if (!isdouble)
{
if (peek == 'u' || peek == 'U')
{
ReaderRead();
suffix = "u";
isunsigned = true;
peek = (char)ReaderPeek();
}
if (peek == 'l' || peek == 'L')
{
ReaderRead();
peek = (char)ReaderPeek();
islong = true;
if (!isunsigned && (peek == 'u' || peek == 'U'))
{
ReaderRead();
suffix = "Lu";
isunsigned = true;
}
else
{
suffix = isunsigned ? "uL" : "L";
}
}
}
string digit = sb.ToString();
string stringValue = prefix + digit + suffix;
if (isfloat)
{
float num;
if (float.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num))
{
return(new Literal(stringValue, num, LiteralFormat.DecimalNumber));
}
else
{
Error(y, x, String.Format("Can't parse float {0}", digit));
return(new Literal(stringValue, 0f, LiteralFormat.DecimalNumber));
}
}
if (isdecimal)
{
decimal num;
if (decimal.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num))
{
return(new Literal(stringValue, num, LiteralFormat.DecimalNumber));
}
else
{
Error(y, x, String.Format("Can't parse decimal {0}", digit));
return(new Literal(stringValue, 0m, LiteralFormat.DecimalNumber));
}
}
if (isdouble)
{
double num;
if (double.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num))
{
return(new Literal(stringValue, num, LiteralFormat.DecimalNumber));
}
else
{
Error(y, x, String.Format("Can't parse double {0}", digit));
return(new Literal(stringValue, 0d, LiteralFormat.DecimalNumber));
}
}
// Try to determine a parsable value using ranges.
ulong result;
if (ishex)
{
if (!ulong.TryParse(digit, NumberStyles.HexNumber, null, out result))
{
Error(y, x, String.Format("Can't parse hexadecimal constant {0}", digit));
return(new Literal(stringValue.ToString(), 0, LiteralFormat.HexadecimalNumber));
}
}
else
{
if (!ulong.TryParse(digit, NumberStyles.Integer, null, out result))
{
Error(y, x, String.Format("Can't parse integral constant {0}", digit));
return(new Literal(stringValue.ToString(), 0, LiteralFormat.DecimalNumber));
}
}
if (result > long.MaxValue)
{
islong = true;
isunsigned = true;
}
else if (result > uint.MaxValue)
{
islong = true;
}
else if (islong == false && result > int.MaxValue)
{
isunsigned = true;
}
Literal token;
LiteralFormat literalFormat = ishex ? LiteralFormat.HexadecimalNumber : LiteralFormat.DecimalNumber;
if (islong)
{
if (isunsigned)
{
ulong num;
if (ulong.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num))
{
token = new Literal(stringValue, num, literalFormat);
}
else
{
Error(y, x, String.Format("Can't parse unsigned long {0}", digit));
token = new Literal(stringValue, 0UL, literalFormat);
}
}
else
{
long num;
if (long.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num))
{
token = new Literal(stringValue, num, literalFormat);
}
else
{
Error(y, x, String.Format("Can't parse long {0}", digit));
token = new Literal(stringValue, 0L, literalFormat);
}
}
}
else
{
if (isunsigned)
{
uint num;
if (uint.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num))
{
token = new Literal(stringValue, num, literalFormat);
}
else
{
Error(y, x, String.Format("Can't parse unsigned int {0}", digit));
token = new Literal(stringValue, (uint)0, literalFormat);
}
}
else
{
int num;
if (int.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num))
{
token = new Literal(stringValue, num, literalFormat);
}
else
{
Error(y, x, String.Format("Can't parse int {0}", digit));
token = new Literal(stringValue, 0, literalFormat);
}
}
}
token.next = nextToken;
return(token);
}
}
/// <summary>
/// Must be called before a peek operation.
/// </summary>
public void StartPeek()
{
peekToken = curToken;
}
