public IEnumerable <HighlightToken> GetTokens(string text)
{
IronTjs.Compiler.Tokenizer tokenizer = new Compiler.Tokenizer();
int end = 0;
tokenizer.ErrorSink = Microsoft.Scripting.ErrorSink.Null;
tokenizer.Initialize(null, new StringReader(text), null, Microsoft.Scripting.SourceLocation.MinValue);
var tokens = new List <Token>();
while (tokenizer.NextToken.Type != TokenType.EndOfStream)
{
tokens.Add(tokenizer.Read());
}
foreach (var token in tokens)
{
var start = token.Span.Start.Index;
if (end != start)
{
yield return(new HighlightToken(end, start - end, Color.Green, Color.Empty));
}
end = token.Span.End.Index;
switch (token.Type)
{
case TokenType.KeywordClass:
case TokenType.KeywordConst:
case TokenType.KeywordEnum:
case TokenType.KeywordExtends:
case TokenType.KeywordFunction:
case TokenType.KeywordPrivate:
case TokenType.KeywordProperty:
case TokenType.KeywordProtected:
case TokenType.KeywordPublic:
case TokenType.KeywordStatic:
case TokenType.KeywordSynchronized:
case TokenType.KeywordInt:
case TokenType.KeywordOctet:
case TokenType.KeywordReal:
case TokenType.KeywordString:
case TokenType.KeywordVar:
case TokenType.KeywordBreak:
case TokenType.KeywordCase:
case TokenType.KeywordCatch:
case TokenType.KeywordContinue:
case TokenType.KeywordDebugger:
case TokenType.KeywordDefault:
case TokenType.KeywordDelete:
case TokenType.KeywordDo:
case TokenType.KeywordElse:
case TokenType.KeywordExport:
case TokenType.KeywordFinally:
case TokenType.KeywordFor:
case TokenType.KeywordGetter:
case TokenType.KeywordGlobal:
case TokenType.KeywordGoTo:
case TokenType.KeywordIf:
case TokenType.KeywordImport:
case TokenType.KeywordIn:
case TokenType.KeywordInContextOf:
case TokenType.KeywordInstanceOf:
case TokenType.KeywordInvalidate:
case TokenType.KeywordIsValid:
case TokenType.KeywordNew:
case TokenType.KeywordReturn:
case TokenType.KeywordSetter:
case TokenType.KeywordSuper:
case TokenType.KeywordSwitch:
case TokenType.KeywordThis:
case TokenType.KeywordThrow:
case TokenType.KeywordTry:
case TokenType.KeywordTypeOf:
case TokenType.KeywordWhile:
case TokenType.KeywordWith:
case TokenType.KeywordFalse:
case TokenType.KeywordNull:
case TokenType.KeywordTrue:
case TokenType.KeywordVoid:
yield return(new HighlightToken(start, end - start, Color.Blue, Color.Empty));
break;
case TokenType.LiteralString:
yield return(new HighlightToken(start, end - start, Color.FromArgb(163, 21, 21), Color.Empty));
break;
case TokenType.Unknown:
yield return(new HighlightToken(start, end - start, Color.White, Color.Red));
break;
}
}
if (end != text.Length)
{
yield return(new HighlightToken(end, text.Length - end, Color.Green, Color.Empty));
}
}