public virtual void Test1to3()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("k")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "kkk" }, new int[] { 0 }, new int[] { 1 });
}
public virtual void TestChained()
{
CharStream cs = new MappingCharFilter(normMap, new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("aaaa ll h"))));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "a", "llllllll", "i" }, new int[] { 0, 5, 8 }, new int[] { 4, 7, 9 });
}
public virtual void TestTokenStream()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("h i j k ll cccc bbb aa")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "i", "i", "jj", "kkk", "llll", "cc", "b", "a" }, new int[] { 0, 2, 4, 6, 8, 11, 16, 20 }, new int[] { 1, 3, 5, 7, 10, 15, 19, 22 });
}
public virtual void Test5to0()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("empty")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[0]);
}
public virtual void TestReaderReset()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
char[] buf = new char[10];
int len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
len = cs.Read(buf, 0, 10);
Assert.AreEqual(- 1, len);
// rewind
cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
}
public virtual void TestReaderReset()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
char[] buf = new char[10];
int len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
len = cs.Read(buf, 0, 10);
Assert.AreEqual(-1, len);
// rewind
cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
}
public virtual void Test2to4()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("ll")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[]{"llll"}, new int[]{0}, new int[]{2});
}
public virtual void TestChained()
{
CharStream cs = new MappingCharFilter(normMap, new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("aaaa ll h"))));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[]{"a", "llllllll", "i"}, new int[]{0, 5, 8}, new int[]{4, 7, 9});
}
public virtual void TestTokenStream()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("h i j k ll cccc bbb aa")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[]{"i", "i", "jj", "kkk", "llll", "cc", "b", "a"}, new int[]{0, 2, 4, 6, 8, 11, 16, 20}, new int[]{1, 3, 5, 7, 10, 15, 19, 22});
}
public virtual void Test5to0()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("empty")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[0]);
}