/// <summary>
/// </summary>
/// <param name="reader"></param>
public ILTokenizer (StreamReader reader)
{
this.reader = new ILReader (reader);
strBuilder = new StringHelper (this);
numBuilder = new NumberHelper (this);
lastToken = ILToken.Invalid.Clone () as ILToken;
}
/// <summary>
/// </summary>
/// <param name="ch"></param>
/// <returns></returns>
public static int Escape (ILReader reader, int ch)
{
int res = -1;
if (ch >= '0' && ch <='7') {
StringBuilder octal = new StringBuilder ();
octal.Append ((char)ch);
int possibleOctalChar = reader.Peek ();
if (possibleOctalChar >= '0' && possibleOctalChar <='7') {
octal.Append ((char)reader.Read ());
possibleOctalChar = reader.Peek ();
if (possibleOctalChar >= '0' && possibleOctalChar <='7')
octal.Append ((char)reader.Read ());
}
res = Convert.ToInt32(octal.ToString (), 8);
} else {
int id = "abfnrtv\"'\\".IndexOf ((char)ch);
if (id != -1) {
res = "\a\b\f\n\r\t\v\"'\\" [id];
}
}
return res;
}
/// <summary>
/// </summary>
/// <returns></returns>
public override string Build()
{
ILReader reader = host.Reader;
reader.MarkLocation();
StringBuilder num_builder = new StringBuilder();
string num;
int ch;
int peek;
bool is_real = false;
bool dec_found = false;
NumberStyles nstyles = NumberStyles.AllowExponent | NumberStyles.AllowDecimalPoint |
NumberStyles.AllowLeadingSign;
ch = reader.Read();
peek = reader.Peek();
reader.Unread(ch);
if (ch == '0' && (peek == 'x' || peek == 'X'))
{
return(BuildHex());
}
if (is_sign(reader.Peek()))
{
num_builder.Append((char)reader.Read());
}
do
{
ch = reader.Read();
peek = reader.Peek();
num_builder.Append((char)ch);
if (is_e(ch))
{
if (is_real)
{
throw new ILTokenizingException(reader.Location, num_builder.ToString());
}
is_real = true;
}
if (ch == '.')
{
dec_found = true;
}
if (!is_hex(peek) &&
!(peek == '.' && !dec_found) && !is_e(peek) &&
!(is_sign(peek) && is_real))
{
break;
}
}while (ch != -1);
num = num_builder.ToString();
// Check for hexbytes
if (num.Length == 2)
{
if (Char.IsLetter(num[0]) || Char.IsLetter(num[1]))
{
result.token = Token.HEXBYTE;
result.val = Byte.Parse(num, NumberStyles.HexNumber);
return(num);
}
}
if (ch == '.' && peek == '.')
{
num = num.Substring(0, num.Length - 1);
reader.Unread('.');
dec_found = false;
}
else if (ch == '.')
{
num += '0';
}
if (!dec_found && !is_real)
{
try
{
long i = Int64.Parse(num, nstyles);
result.token = Token.INT64;
result.val = i;
return(num);
}
catch
{
}
try
{
long i = (long)UInt64.Parse(num, nstyles);
result.token = Token.INT64;
result.val = i;
return(num);
}
catch
{
}
}
try
{
double d = Double.Parse(num, nstyles, NumberFormatInfo.InvariantInfo);
result.token = Token.FLOAT64;
result.val = d;
}
catch
{
reader.Unread(num.ToCharArray());
reader.RestoreLocation();
num = String.Empty;
Reset();
throw new ILTokenizingException(reader.Location, num_builder.ToString());
}
return(num);
}
/// <summary>
/// </summary>
/// <returns></returns>
public override string Build()
{
if (mode == Token.UNKNOWN)
{
return(String.Empty);
}
int ch = 0;
ILReader reader = host.Reader;
StringBuilder idsb = new StringBuilder();
if (mode == Token.SQSTRING || mode == Token.QSTRING)
{
int term = (mode == Token.SQSTRING) ? '\'' : '"';
reader.Read(); // skip quote
for (ch = reader.Read(); ch != -1; ch = reader.Read())
{
if (ch == term)
{
break;
}
if (ch == '\\')
{
ch = reader.Read();
/*
* Long string can be broken across multiple lines
* by using '\' as the last char in line.
* Any white space chars between '\' and the first
* char on the next line are ignored.
*/
if (ch == '\n')
{
reader.SkipWhitespace();
continue;
}
int escaped = Escape(reader, ch);
if (escaped == -1)
{
reader.Unread(ch);
ch = '\\';
}
else
{
ch = escaped;
}
}
idsb.Append((char)ch);
}
}
else // ID
{
while ((ch = reader.Read()) != -1)
{
if (IsIdChar(ch))
{
idsb.Append((char)ch);
}
else
{
reader.Unread(ch);
break;
}
}
}
return(idsb.ToString());
}
public string BuildHex()
{
ILReader reader = host.Reader;
reader.MarkLocation();
StringBuilder num_builder = new StringBuilder();
NumberStyles nstyles = NumberStyles.HexNumber;
string num;
int ch;
int peek;
ch = reader.Read();
if (ch != '0')
{
throw new ILTokenizingException(reader.Location, ((char)ch).ToString());
}
ch = reader.Read();
if (ch != 'x' && ch != 'X')
{
throw new ILTokenizingException(reader.Location, "0" + (char)ch);
}
do
{
ch = reader.Read();
peek = reader.Peek();
num_builder.Append((char)ch);
if (!is_hex((char)peek))
{
break;
}
if (num_builder.Length == 32)
{
throw new ILTokenizingException(reader.Location, num_builder.ToString());
}
}while (ch != -1);
num = num_builder.ToString();
try
{
long i = (long)UInt64.Parse(num, nstyles);
//if (i < Int32.MinValue || i > Int32.MaxValue) {
result.token = Token.INT64;
result.val = i;
//} else {
// result.token = Token.INT32;
// result.val = (int) i;
//}
}
catch
{
string tnum = num;
reader.Unread(num.ToCharArray());
reader.RestoreLocation();
num = String.Empty;
Reset();
throw new ILTokenizingException(reader.Location, tnum);
}
return(num);
}