public void GeneralReadTest()
        {
            var reader = StringInfo.GetTextElementEnumerator(
@"function test()
    local v = 12 -- this is a comment
    t ={ [12] = v, cat=12.345456 }
    str = ""this is a test string with \""escapes\""\n""
    ::potato::
end"
            );
            Tokenizer tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 9, Value = "function" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 14, EndPos = 15, Value = "(" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 15, EndPos = 16, Value = ")" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 2, EndLine = 2, StartPos = 5, EndPos = 10, Value = "local" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 2, EndLine = 2, StartPos = 11, EndPos = 12, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 2, EndLine = 2, StartPos = 13, EndPos = 14, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 2, EndLine = 2, StartPos = 15, EndPos = 17, Value = "12" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 5, EndPos = 6, Value = "t" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 7, EndPos = 8, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 8, EndPos = 9, Value = "{" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 10, EndPos = 11, Value = "[" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 11, EndPos = 13, Value = "12" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 13, EndPos = 14, Value = "]" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 15, EndPos = 16, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 17, EndPos = 18, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 18, EndPos = 19, Value = "," }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 20, EndPos = 23, Value = "cat" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 23, EndPos = 24, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 24, EndPos = 33, Value = "12.345456" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 3, EndLine = 3, StartPos = 34, EndPos = 35, Value = "}" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 4, EndLine = 4, StartPos = 5, EndPos = 8, Value = "str" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 4, EndLine = 4, StartPos = 9, EndPos = 10, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 4, EndLine = 4, StartPos = 11, EndPos = 53, Value = "\"this is a test string with \"escapes\"\n" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 5, EndLine = 5, StartPos = 5, EndPos = 7, Value = "::" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 5, EndLine = 5, StartPos = 7, EndPos = 13, Value = "potato" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 5, EndLine = 5, StartPos = 13, EndPos = 15, Value = "::" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 6, EndLine = 6, StartPos = 1, EndPos = 4, Value = "end" }, tokenizer.Read());

            Assert.AreEqual(new Token() { StartLine = 0, EndLine = 0, StartPos = 0, EndPos = 0, Value = null }, tokenizer.Read());
        }
        public void LongStringTest()
        {
            var reader = StringInfo.GetTextElementEnumerator(
@"v = [==[
this is a test of a
long string ]]
this is still a string
not an escape \n]==]
end"
            );
            Tokenizer tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 2, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 3, EndPos = 4, Value = "=" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 5, StartPos = 5, EndPos = 21, Value = "\"this is a test of a\nlong string ]]\nthis is still a string\nnot an escape \\n" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 6, EndLine = 6, StartPos = 1, EndPos = 4, Value = "end" }, tokenizer.Read());

            Assert.AreEqual(new Token() { StartLine = 0, EndLine = 0, StartPos = 0, EndPos = 0, Value = null }, tokenizer.Read());
        }
        public void StringErrorTest()
        {
            // newline in string literal.
            var reader = StringInfo.GetTextElementEnumerator(
@"v = 'error string

end"
            );
            Tokenizer tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 2, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 3, EndPos = 4, Value = "=" }, tokenizer.Read());

            try
            {
                tokenizer.Read();
                Assert.Fail("Supposed to cause error.");
            }
            catch (Exception e)
            {
                Assert.IsInstanceOfType(e, typeof(SyntaxException));
            }

            // only one grave (`) per literal.
            reader = StringInfo.GetTextElementEnumerator(@"v = foo``");
            tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 2, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 3, EndPos = 4, Value = "=" }, tokenizer.Read());

            try
            {
                tokenizer.Read();
                Assert.Fail("Supposed to cause error.");
            }
            catch (Exception e)
            {
                Assert.IsInstanceOfType(e, typeof(SyntaxException));
            }

            // only number after a grave(`).
            reader = StringInfo.GetTextElementEnumerator(@"v = foo`23e");
            tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 2, Value = "v" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 3, EndPos = 4, Value = "=" }, tokenizer.Read());

            try
            {
                tokenizer.Read();
                Assert.Fail("Supposed to cause error.");
            }
            catch (Exception e)
            {
                Assert.IsInstanceOfType(e, typeof(SyntaxException));
            }
        }
        public void ReadPeekPushBackTest()
        {
            var reader = StringInfo.GetTextElementEnumerator(@"function test()");
            Tokenizer tokenizer = new Tokenizer(reader, "Test");

            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 1, EndPos = 9, Value = "function" }, tokenizer.Read());

            // check for multiple calls to Peek
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Peek());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Peek());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Peek());

            // check that peek changes
            Token next = tokenizer.Read();
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, next);
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 14, EndPos = 15, Value = "(" }, tokenizer.Peek());

            Token next2 = tokenizer.Read();
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 14, EndPos = 15, Value = "(" }, next2);
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 15, EndPos = 16, Value = ")" }, tokenizer.Peek());

            // check that PushBack works putting things back
            tokenizer.PushBack(next2);
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 14, EndPos = 15, Value = "(" }, tokenizer.Peek());
            tokenizer.PushBack(next);
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Peek());

            // check that PushBack works after several calls and in the correct order.
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 10, EndPos = 14, Value = "test" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 14, EndPos = 15, Value = "(" }, tokenizer.Read());
            Assert.AreEqual(new Token() { StartLine = 1, EndLine = 1, StartPos = 15, EndPos = 16, Value = ")" }, tokenizer.Read());

            Assert.AreEqual(new Token() { StartLine = 0, EndLine = 0, StartPos = 0, EndPos = 0, Value = null }, tokenizer.Read());
        }