static void Main(string[] args)
{
var text = "foo 123 bar";
using (var sr = new StreamReader(@"..\..\Program.cs"))
text = sr.ReadToEnd();
Console.WriteLine("Lex: " + text);
var tokenizer = new TestTokenizer(text); // generated from Example.lx
Console.WriteLine("Disassembly:");
Console.WriteLine(Lex.Disassemble(TestTokenizer.Program));
Console.WriteLine();
foreach (var tok in tokenizer)
{
// we don't want errors or whitespace but we don't know the symbol
// id for whitespace because you can switch tokenizers around
// so we check tok.Value instead
if (-1 != tok.SymbolId && !string.IsNullOrWhiteSpace(tok.Value))
{
Console.WriteLine("{0}: {1}", tok.SymbolId, tok.Value);
}
}
Stopwatch sw = new Stopwatch();
const int ITER = 1000;
for (var i = 0; i < ITER; ++i)
{
var lc = LexContext.Create(text);
while (LexContext.EndOfInput != lc.Current)
{
lc.ClearCapture();
sw.Start();
var acc = Lex.Run(TestTokenizer.Program, lc);
sw.Stop();
}
}
Console.WriteLine("Lexed in " + sw.ElapsedMilliseconds / (float)ITER + " msec");
}
public void TestTokenizerParsing()
{
TestTokenizer tokenizer = new TestTokenizer();
{
string input = "abc def123.456ghi7...__xx;";
List <CodeToken> tokens = tokenizer.Tokenize(input.ToCharArray());
Assert.AreEqual(TestTokenizer.Id, tokens[0].Id);
Assert.AreEqual(new TextPosition(0, 0), tokens[0].Start);
Assert.AreEqual(new TextPosition(0, 3), tokens[0].End);
Assert.AreEqual("abc", tokens[0].Value);
Assert.AreEqual(-1, tokens[1].Id);
Assert.AreEqual(new TextPosition(0, 3), tokens[1].Start);
Assert.AreEqual(new TextPosition(0, 4), tokens[1].End);
Assert.AreEqual(" ", tokens[1].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[2].Id);
Assert.AreEqual(new TextPosition(0, 4), tokens[2].Start);
Assert.AreEqual(new TextPosition(0, 10), tokens[2].End);
Assert.AreEqual("def123", tokens[2].Value);
Assert.AreEqual(-1, tokens[3].Id);
Assert.AreEqual(new TextPosition(0, 10), tokens[3].Start);
Assert.AreEqual(new TextPosition(0, 11), tokens[3].End);
Assert.AreEqual(".", tokens[3].Value);
Assert.AreEqual(TestTokenizer.Integer, tokens[4].Id);
Assert.AreEqual(new TextPosition(0, 11), tokens[4].Start);
Assert.AreEqual(new TextPosition(0, 14), tokens[4].End);
Assert.AreEqual("456", tokens[4].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[5].Id);
Assert.AreEqual(new TextPosition(0, 14), tokens[5].Start);
Assert.AreEqual(new TextPosition(0, 18), tokens[5].End);
Assert.AreEqual("ghi7", tokens[5].Value);
Assert.AreEqual(-1, tokens[6].Id);
Assert.AreEqual(new TextPosition(0, 18), tokens[6].Start);
Assert.AreEqual(new TextPosition(0, 19), tokens[6].End);
Assert.AreEqual(".", tokens[6].Value);
Assert.AreEqual(-1, tokens[7].Id);
Assert.AreEqual(new TextPosition(0, 19), tokens[7].Start);
Assert.AreEqual(new TextPosition(0, 20), tokens[7].End);
Assert.AreEqual(".", tokens[7].Value);
Assert.AreEqual(-1, tokens[8].Id);
Assert.AreEqual(new TextPosition(0, 20), tokens[8].Start);
Assert.AreEqual(new TextPosition(0, 21), tokens[8].End);
Assert.AreEqual(".", tokens[8].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[9].Id);
Assert.AreEqual(new TextPosition(0, 21), tokens[9].Start);
Assert.AreEqual(new TextPosition(0, 25), tokens[9].End);
Assert.AreEqual("__xx", tokens[9].Value);
Assert.AreEqual(-1, tokens[10].Id);
Assert.AreEqual(new TextPosition(0, 25), tokens[10].Start);
Assert.AreEqual(new TextPosition(0, 26), tokens[10].End);
Assert.AreEqual(";", tokens[10].Value);
}
{
string input = "abc def;123.456gh?7...__xx;";
List <CodeToken> tokens = tokenizer.Tokenize(input.ToCharArray());
Assert.AreEqual(TestTokenizer.Id, tokens[0].Id);
Assert.AreEqual(new TextPosition(0, 0), tokens[0].Start);
Assert.AreEqual(new TextPosition(0, 3), tokens[0].End);
Assert.AreEqual("abc", tokens[0].Value);
Assert.AreEqual(-1, tokens[1].Id);
Assert.AreEqual(new TextPosition(0, 3), tokens[1].Start);
Assert.AreEqual(new TextPosition(0, 4), tokens[1].End);
Assert.AreEqual(" ", tokens[1].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[2].Id);
Assert.AreEqual(new TextPosition(0, 4), tokens[2].Start);
Assert.AreEqual(new TextPosition(0, 7), tokens[2].End);
Assert.AreEqual("def", tokens[2].Value);
Assert.AreEqual(-1, tokens[3].Id);
Assert.AreEqual(new TextPosition(0, 7), tokens[3].Start);
Assert.AreEqual(new TextPosition(0, 8), tokens[3].End);
Assert.AreEqual(";", tokens[3].Value);
Assert.AreEqual(TestTokenizer.Double, tokens[4].Id);
Assert.AreEqual(new TextPosition(0, 8), tokens[4].Start);
Assert.AreEqual(new TextPosition(0, 15), tokens[4].End);
Assert.AreEqual("123.456", tokens[4].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[5].Id);
Assert.AreEqual(new TextPosition(0, 15), tokens[5].Start);
Assert.AreEqual(new TextPosition(0, 17), tokens[5].End);
Assert.AreEqual("gh", tokens[5].Value);
Assert.AreEqual(-1, tokens[6].Id);
Assert.AreEqual(new TextPosition(0, 17), tokens[6].Start);
Assert.AreEqual(new TextPosition(0, 18), tokens[6].End);
Assert.AreEqual("?", tokens[6].Value);
Assert.AreEqual(TestTokenizer.Integer, tokens[7].Id);
Assert.AreEqual(new TextPosition(0, 18), tokens[7].Start);
Assert.AreEqual(new TextPosition(0, 19), tokens[7].End);
Assert.AreEqual("7", tokens[7].Value);
Assert.AreEqual(-1, tokens[8].Id);
Assert.AreEqual(new TextPosition(0, 19), tokens[8].Start);
Assert.AreEqual(new TextPosition(0, 20), tokens[8].End);
Assert.AreEqual(".", tokens[8].Value);
Assert.AreEqual(-1, tokens[9].Id);
Assert.AreEqual(new TextPosition(0, 20), tokens[9].Start);
Assert.AreEqual(new TextPosition(0, 21), tokens[9].End);
Assert.AreEqual(".", tokens[9].Value);
Assert.AreEqual(-1, tokens[10].Id);
Assert.AreEqual(new TextPosition(0, 21), tokens[10].Start);
Assert.AreEqual(new TextPosition(0, 22), tokens[10].End);
Assert.AreEqual(".", tokens[10].Value);
Assert.AreEqual(TestTokenizer.Id, tokens[11].Id);
Assert.AreEqual(new TextPosition(0, 22), tokens[11].Start);
Assert.AreEqual(new TextPosition(0, 26), tokens[11].End);
Assert.AreEqual("__xx", tokens[11].Value);
Assert.AreEqual(-1, tokens[12].Id);
Assert.AreEqual(new TextPosition(0, 26), tokens[12].Start);
Assert.AreEqual(new TextPosition(0, 27), tokens[12].End);
Assert.AreEqual(";", tokens[12].Value);
}
}
