public void TokenStreamTest()
{
void Test(string input, int expectedLength, string output)
{
if (output == null)
{
output = input;
}
var tokens = new TextFileTokenStream(new StringReader(input), null).Tokens.ToArray();
Assert.AreEqual(expectedLength, tokens.Length);
Assert.AreEqual(output, tokens.Untokenize(new FormattingOptions()
{
Capitalize = false
}));
}
Test("\"Foo,\" I said. \"I don't like this.\"", 16, "\u201cFoo,\u201d I said. \u201cI don't like this.\u201d");
Test("This <i>is also</i> a test", 7, "This<i> is also</i> a test");
Test("this is a test", 4, null);
Test(" this is a test ", 4, "this is a test");
Test(" ", 0, "");
Test("", 0, "");
Test("[]", 2, "[]");
Test(" [ ] ", 2, "[]");
Test("[a]", 3, "[a]");
Test(">=", 1, ">=");
}
public void TokenizeCommentTest2()
{
var actual = TextFileTokenStream.Tokenize(@"Test: foo # comment
Baz: baz");
var expected = new[] { "Test", ":", "foo", "\n", "Baz", ":", "baz" };
Assert.AreEqual(actual.Length, expected.Length);
for (var i = 0; i < actual.Length; i++)
{
Assert.AreEqual(expected[i], actual[i]);
}
}
public void TokenizeCommentTest()
{
var actual = TextFileTokenStream.Tokenize(@"Foo:
a
#foo
#bar
b
[end]");
var expected = new[] { "Foo", ":", "\n", "a", "\n", "\n", "\n", "b", "\n", "[", "end", "]" };
Assert.AreEqual(actual.Length, expected.Length);
for (var i = 0; i < actual.Length; i++)
{
Assert.AreEqual(expected[i], actual[i]);
}
}
