private Token ReadIntegerToken(ref string content, ref int pos, TokenPosition position)
{
var literal = ReadLiteral(ref content, ref pos).ToLower();
if (literal.StartsWith("0x")) {
return ParsePrefixedHexInteger(literal, position);
}
if (literal.EndsWith("h")) {
return ParsePostfixedHexInteger(literal, position);
}
return ReadDecimalInteger(ref literal, position);
}
private Token ParsePrefixedHexInteger(string literal, TokenPosition position)
{
long val = 0;
var pos = 2;
if (pos >= literal.Length) throw new TokenException("Unexpeced end of hex constant", new Token { Position = position, StringValue = literal });
while (pos < literal.Length) {
var ch = literal[pos++];
var hex = GetHexValue(ch);
if (hex < 0) throw new TokenException("unexpected hex symbol '" + ch + "'", new Token { Position = position, StringValue = literal });
val = val * 16 + hex;
}
return new Token {
Type = TokenType.Integer,
IntegerValue = val,
StringValue = literal,
Position = position
};
}
private static Token ReadDecimalInteger(ref string literal, TokenPosition position)
{
long val = 0;
var pos = 0;
while (pos < literal.Length) {
var ch = literal[pos++];
if (char.IsDigit(ch)) {
val = val * 10 + (ch - '0');
} else {
throw new TokenException("unexpected decimal symbol '" + ch + "'", new Token { Position = position, StringValue = literal });
}
}
return new Token {
Type = TokenType.Integer,
IntegerValue = val,
StringValue = literal,
Position = position
};
}
private static string ReadString(ref string content, ref int pos, TokenPosition position)
{
var quoteCh = content[pos++];
var token = "";
while (pos < content.Length) {
var ch = content[pos++];
if (ch == '\r' || ch == '\n') {
throw new TokenException("missing end quote", new Token { Position = position });
}
if (ch == '\\') {
if (pos >= content.Length) throw new TokenException("missing end quote", new Token { Position = position });
ch = content[pos++];
switch (ch) {
case 'n':
token += '\n';
break;
case 'r':
token += '\r';
break;
case 'a':
token += '\a';
break;
case 'b':
token += '\b';
break;
case 'f':
token += '\f';
break;
case 't':
token += '\t';
break;
case 'v':
token += '\v';
break;
case '\\':
token += '\\';
break;
case '\'':
token += '\'';
break;
case '\"':
token += '\"';
break;
case '0':
token += '\0';
break;
default:
throw new TokenException("invalid escape character " + ch, new Token { Position = position });
}
} else if (ch == quoteCh) {
if (pos >= content.Length) return token;
var next = content[pos++];
if (next == quoteCh) {
token += ch;
} else {
pos--;
return token;
}
} else {
token += ch;
}
}
throw new TokenException("missing end quote", new Token { Position = position });
}
public IList<Token> Read(FileSource source)
{
var content = source.Content;
var res = new List<Token>();
var pos = 0;
var lineNumber = 1;
var lineStart = 0;
while (pos < content.Length) {
SkipWhitespace(ref content, ref pos);
if (pos >= content.Length) break;
var position = new TokenPosition { File = source, Line = lineNumber, LineStart = lineStart };
var ch = content[pos++];
var preview = (char)(0xffff);
if (pos < content.Length) {
preview = content[pos];
}
if (ch == '>' && preview == '>') {
res.Add(new Token {
Type = TokenType.RightShift,
StringValue = ">",
Position = position
});
pos++;
continue;
}
if (ch == '<' && preview == '<') {
res.Add(new Token {
Type = TokenType.LeftShift,
StringValue = "<",
Position = position
});
pos++;
continue;
}
switch (ch) {
case '\r':
case '\n':
SkipNewLineChar(ref content, ref pos, ch);
lineNumber++;
lineStart = pos;
res.Add(new Token {
Type = TokenType.NewLine,
Position = position
});
break;
case ';':
SkipLine(ref content, ref pos);
lineNumber++;
lineStart = pos;
res.Add(new Token {
Type = TokenType.NewLine,
Position = position
});
break;
case '"':
case '\'':
pos--;
res.Add(new Token {
Type = TokenType.String,
StringValue = ReadString(ref content, ref pos, position),
Position = position
});
break;
case '\\':
SkipWhitespace(ref content, ref pos);
if (pos >= content.Length) throw new Exception("Unexpected end of file");
ch = content[pos++];
if (ch == '\r' || ch == '\n') {
SkipNewLineChar(ref content, ref pos, ch);
lineNumber++;
lineStart = pos;
} else {
throw new Exception("Unexpected character after backslash");
}
break;
default:
if (IsPunctuation(ch)) {
res.Add(new Token {
Type = GetPunctuationTokenType(ch),
StringValue = ch.ToString(),
Position = position
});
} else if (char.IsDigit(ch)) {
pos--;
res.Add(ReadIntegerToken(ref content, ref pos, position));
} else {
pos--;
res.Add(new Token {
Type = TokenType.Literal,
StringValue = ReadLiteral(ref content, ref pos),
Position = position
});
}
break;
}
}
return res;
}