protected override AstToken ReadNextToken()
{
while (true)
{
if (!MoveToNextToken())
return new CSharpAstToken(EOF, string.Empty, new TextRange(Location, Location));
char currentChar;
Peek(out currentChar);
CSharpAstToken token;
if (currentChar == '/')
{
var node = TokenizeForwardSlash();
token = node as CSharpAstToken;
if (token == null)
{
SpecialBag.SpecialNodes.Add(node);
continue;
}
}
else if (char.IsDigit(currentChar))
{
token = ReadNumberToken();
}
else if (IsWordElement(currentChar))
{
token = ReadWordToken();
CSharpAstTokenCode keywordCode;
if (CSharpAstToken.KeywordMapping.TryGetValue(token.Value, out keywordCode))
{
switch (keywordCode)
{
case TRUE:
token.UserData["InterpretedValue"] = true;
break;
case FALSE:
token.UserData["InterpretedValue"] = false;
break;
case DEFAULT:
// Check whether next char is ':' to disambiguate default(T) expression and default case label.
char next;
if (Peek(out next) && next == ':')
keywordCode = DEFAULT_COLON;
break;
}
token.Code = keywordCode;
}
}
else if (currentChar == '\'' || currentChar == '"')
{
token = ReadStringToken();
}
else
{
var start = Location;
StartReadBuffer();
var code = TokenizeCurrentSymbol();
var end = Location;
token = new CSharpAstToken(code, EndReadBuffer().ToString(), new TextRange(start, end));
}
return token;
}
}
private CSharpAstToken ReadStringToken()
{
var startLocation = Location;
var startCharacter = Read();
TextRange range;
bool escaped = false;
string stringToken = startCharacter + ReadCharacters(x =>
{
if (x == startCharacter)
{
if (escaped)
escaped = false;
else
return false;
}
else if (x == '\\')
{
escaped = !escaped;
}
else
{
escaped = false;
}
return true;
}, out range);
char character;
if (Peek(out character) && character == startCharacter)
stringToken += Read();
var endLocation = Location;
var astToken = new CSharpAstToken(LITERAL, stringToken, new TextRange(startLocation, endLocation));
var formatter = CSharpLanguage.Instance.StringFormatter;
astToken.UserData["InterpretedValue"] = stringToken[0] == '\'' ? (object) formatter.EvaluateFormattedChar(stringToken) : formatter.EvaluateFormattedString(stringToken);
return astToken;
}
private AstNode TokenizeForwardSlash()
{
var start = Location;
Read();
char nextChar;
Peek(out nextChar);
AstNode node;
switch (nextChar)
{
case '/':
node = ReadSingleLineComment();
break;
case '*':
node = ReadCommentBlock();
break;
case '=':
Read();
node = new CSharpAstToken(OP_DIV_ASSIGN, "/=");
break;
default:
node = new CSharpAstToken(DIV, "/");
break;
}
var end = Location;
node.Range = new TextRange(start, end);
return node;
}
protected CSharpAstToken ReadNumberToken()
{
TextRange range;
bool hasSuffix = false;
bool hasDecimalSpecifier = false;
string token = ReadCharacters(x =>
{
if (char.IsLetter(x))
{
hasSuffix = true;
return true;
}
if (char.IsDigit(x))
return !hasSuffix;
if (x == '.')
{
if (hasDecimalSpecifier)
throw new Exception("Too many decimal specifiers in number.");
hasDecimalSpecifier = true;
return true;
}
return false;
}, out range);
var astToken = new CSharpAstToken(LITERAL, token, range);
astToken.UserData["InterpretedValue"] = CSharpLanguage.Instance.NumberFormatter.EvaluateFormattedNumber(token);
return astToken;
}
