private static Token ParseWhitespace(TokenizerContext context) { var originalPos = context.Position; var len = context.File.Content.Length; var position = context.CreatePosition(); while (context.Position < len) { var ch = context.File.Content[context.Position++]; if (ch == '\r' || ch == '\n') { if (context.Position < len) { var nextCh = context.File.Content[context.Position]; if ((nextCh == '\r' || nextCh == '\n') && nextCh != ch) { context.Position++; } } context.IncrementLine(); } else if (ch == ' ' || ch == '\t') { } else { context.Position--; break; } } var res = new Token { Type = TokenType.Whitespace, StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos), Position = position }; return res; }
private static Token ParseMultiLineComment(TokenizerContext context) { var originalPos = context.Position; var len = context.File.Content.Length; var position = context.CreatePosition(); while (context.Position < len - 1) { var ch = context.File.Content[context.Position++]; if (ch == '\r' || ch == '\n') { if (context.Position < len) { var nextCh = context.File.Content[context.Position]; if ((nextCh == '\r' || nextCh == '\n') && nextCh != ch) { context.Position++; } } context.IncrementLine(); } else if (ch == '*' && context.File.Content[context.Position] == '/') { context.Position++; return new Token { Type = TokenType.MultiLineComment, StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos), Position = position }; } } throw new TokenException("unclosed multiline comment", new Token { Type = TokenType.MultiLineComment, StringValue = context.File.Content.Substring(originalPos, context.Position - originalPos), Position = position }); }