static void Main(string[] args)
{
var lines = new List<string>();
Console.WriteLine("Enter python script to tokenize below. Leave line empty to tokenize");
string line;
do
{
Console.Write("> ");
line = Console.ReadLine();
lines.Add(line);
} while (!string.IsNullOrEmpty(line));
var script = string.Join("\n", lines);
var tokenizer = new Lexer.Tokenizer(new Lexer.Grammers.PythonGrammer());
foreach (var token in tokenizer.Tokenize(script))
{
Console.WriteLine("{0}:{1} - {2} => {3}", token.StartIndex, token.Length, token.Type, script.Substring(token.StartIndex, token.Length));
}
Console.ReadLine();
}
static void Main(string[] args)
{
var lines = new List <string>();
Console.WriteLine("Enter python script to tokenize below. Leave line empty to tokenize");
string line;
do
{
Console.Write("> ");
line = Console.ReadLine();
lines.Add(line);
} while (!string.IsNullOrEmpty(line));
var script = string.Join("\n", lines);
var tokenizer = new Lexer.Tokenizer(new Lexer.Grammers.PythonGrammer());
foreach (var token in tokenizer.Tokenize(script))
{
Console.WriteLine("{0}:{1} - {2} => {3}", token.StartIndex, token.Length, token.Type, script.Substring(token.StartIndex, token.Length));
}
Console.ReadLine();
}
