private static Token ParseWhitespace(TokenizerContext context)
{
var originalPos = context.Position;
var len = context.File.Content.Length;
var position = context.CreatePosition();
while (context.Position < len) {
var ch = context.File.Content[context.Position++];
if (ch == '\r' || ch == '\n') {
if (context.Position < len) {
var nextCh = context.File.Content[context.Position];
if ((nextCh == '\r' || nextCh == '\n') && nextCh != ch) {
context.Position++;
}
}
context.IncrementLine();
} else if (ch == ' ' || ch == '\t') {
} else {
context.Position--;
break;
}
}
var res = new Token {
Type = TokenType.Whitespace,
StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos),
Position = position
};
return res;
}
private static Token ParseMultiLineComment(TokenizerContext context)
{
var originalPos = context.Position;
var len = context.File.Content.Length;
var position = context.CreatePosition();
while (context.Position < len - 1) {
var ch = context.File.Content[context.Position++];
if (ch == '\r' || ch == '\n') {
if (context.Position < len) {
var nextCh = context.File.Content[context.Position];
if ((nextCh == '\r' || nextCh == '\n') && nextCh != ch) {
context.Position++;
}
}
context.IncrementLine();
} else if (ch == '*' && context.File.Content[context.Position] == '/') {
context.Position++;
return new Token {
Type = TokenType.MultiLineComment,
StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos),
Position = position
};
}
}
throw new TokenException("unclosed multiline comment", new Token {
Type = TokenType.MultiLineComment,
StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos),
Position = position
});
}
