protected override AstToken ReadNextToken() { while (true) { if (!MoveToNextToken()) return new CSharpAstToken(EOF, string.Empty, new TextRange(Location, Location)); char currentChar; Peek(out currentChar); CSharpAstToken token; if (currentChar == '/') { var node = TokenizeForwardSlash(); token = node as CSharpAstToken; if (token == null) { SpecialBag.SpecialNodes.Add(node); continue; } } else if (char.IsDigit(currentChar)) { token = ReadNumberToken(); } else if (IsWordElement(currentChar)) { token = ReadWordToken(); CSharpAstTokenCode keywordCode; if (CSharpAstToken.KeywordMapping.TryGetValue(token.Value, out keywordCode)) { switch (keywordCode) { case TRUE: token.UserData["InterpretedValue"] = true; break; case FALSE: token.UserData["InterpretedValue"] = false; break; case DEFAULT: // Check whether next char is ':' to disambiguate default(T) expression and default case label. char next; if (Peek(out next) && next == ':') keywordCode = DEFAULT_COLON; break; } token.Code = keywordCode; } } else if (currentChar == '\'' || currentChar == '"') { token = ReadStringToken(); } else { var start = Location; StartReadBuffer(); var code = TokenizeCurrentSymbol(); var end = Location; token = new CSharpAstToken(code, EndReadBuffer().ToString(), new TextRange(start, end)); } return token; } }
private CSharpAstToken ReadStringToken() { var startLocation = Location; var startCharacter = Read(); TextRange range; bool escaped = false; string stringToken = startCharacter + ReadCharacters(x => { if (x == startCharacter) { if (escaped) escaped = false; else return false; } else if (x == '\\') { escaped = !escaped; } else { escaped = false; } return true; }, out range); char character; if (Peek(out character) && character == startCharacter) stringToken += Read(); var endLocation = Location; var astToken = new CSharpAstToken(LITERAL, stringToken, new TextRange(startLocation, endLocation)); var formatter = CSharpLanguage.Instance.StringFormatter; astToken.UserData["InterpretedValue"] = stringToken[0] == '\'' ? (object) formatter.EvaluateFormattedChar(stringToken) : formatter.EvaluateFormattedString(stringToken); return astToken; }
private AstNode TokenizeForwardSlash() { var start = Location; Read(); char nextChar; Peek(out nextChar); AstNode node; switch (nextChar) { case '/': node = ReadSingleLineComment(); break; case '*': node = ReadCommentBlock(); break; case '=': Read(); node = new CSharpAstToken(OP_DIV_ASSIGN, "/="); break; default: node = new CSharpAstToken(DIV, "/"); break; } var end = Location; node.Range = new TextRange(start, end); return node; }
protected CSharpAstToken ReadNumberToken() { TextRange range; bool hasSuffix = false; bool hasDecimalSpecifier = false; string token = ReadCharacters(x => { if (char.IsLetter(x)) { hasSuffix = true; return true; } if (char.IsDigit(x)) return !hasSuffix; if (x == '.') { if (hasDecimalSpecifier) throw new Exception("Too many decimal specifiers in number."); hasDecimalSpecifier = true; return true; } return false; }, out range); var astToken = new CSharpAstToken(LITERAL, token, range); astToken.UserData["InterpretedValue"] = CSharpLanguage.Instance.NumberFormatter.EvaluateFormattedNumber(token); return astToken; }