public virtual void TestTokenStream()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("h i j k ll cccc bbb aa")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "i", "i", "jj", "kkk", "llll", "cc", "b", "a" }, new int[] { 0, 2, 4, 6, 8, 11, 16, 20 }, new int[] { 1, 3, 5, 7, 10, 15, 19, 22 });
}
public virtual void Test1to3()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("k")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "kkk" }, new int[] { 0 }, new int[] { 1 });
}
public virtual void Test5to0()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("empty")));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[0]);
}
public HTMLStripCharFilter(CharStream source, HashSet <String> escapedTags)
: base(CharReader.Get(source))
{
this.escapedTags = escapedTags;
safeReadAheadLimit = readAheadLimit - 3;
LoadEntityTable();
}
public virtual void TestReaderReset()
{
CharStream cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
char[] buf = new char[10];
int len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
len = cs.Read(buf, 0, 10);
Assert.AreEqual(-1, len);
// rewind
cs = new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("x")));
len = cs.Read(buf, 0, 10);
Assert.AreEqual(1, len);
Assert.AreEqual('x', buf[0]);
}
/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input) : base(source)
{
this.input = CharReader.Get(input);
}
/// <summary>Construct a token stream processing the given input using the given AttributeFactory. </summary>
protected internal Tokenizer(AttributeFactory factory, System.IO.TextReader input) : base(factory)
{
this.input = CharReader.Get(input);
}
/// <summary>Construct a token stream processing the given input. </summary>
protected internal Tokenizer(System.IO.TextReader input)
{
this.input = CharReader.Get(input);
}
/// Easy-use constructor that takes a {@link Reader}.
public MappingCharFilter(NormalizeCharMap normMap, System.IO.TextReader in_Renamed) : base(CharReader.Get(in_Renamed))
{
this.normMap = normMap;
}
public virtual void TestChained()
{
CharStream cs = new MappingCharFilter(normMap, new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("aaaa ll h"))));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "a", "llllllll", "i" }, new int[] { 0, 5, 8 }, new int[] { 4, 7, 9 });
}
public virtual void TestCharFilter11()
{
CharStream cs = new CharFilter1(new CharFilter1(CharReader.Get(new System.IO.StringReader(""))));
Assert.AreEqual(2, cs.CorrectOffset(0), "corrected offset is invalid");
}
/// Easy-use constructor that takes a <see cref="System.IO.TextReader" />.
public MappingCharFilter(NormalizeCharMap normMap, System.IO.TextReader @in)
: base(CharReader.Get(@in))
{
this.normMap = normMap;
}
public HTMLStripCharFilter(CharStream source)
: base(CharReader.Get(source))
{
safeReadAheadLimit = readAheadLimit - 3;
LoadEntityTable();
}