/// <summary>Construct a token stream processing the given input using the given AttributeSource. </summary>
protected internal Tokenizer(AttributeSource source, System.IO.TextReader input) : base(source)
{
this.input = CharReader.Get(input);
}
/// <summary>Construct a token stream processing the given input. </summary>
protected internal Tokenizer(System.IO.TextReader input)
{
this.input = CharReader.Get(input);
}
/// <summary>Construct a token stream processing the given input using the given AttributeFactory. </summary>
protected internal Tokenizer(AttributeFactory factory, System.IO.TextReader input) : base(factory)
{
this.input = CharReader.Get(input);
}
/// Easy-use constructor that takes a {@link Reader}.
public MappingCharFilter(NormalizeCharMap normMap, System.IO.TextReader in_Renamed) : base(CharReader.Get(in_Renamed))
{
this.normMap = normMap;
}
public virtual void TestChained()
{
CharStream cs = new MappingCharFilter(normMap, new MappingCharFilter(normMap, CharReader.Get(new System.IO.StringReader("aaaa ll h"))));
TokenStream ts = new WhitespaceTokenizer(cs);
AssertTokenStreamContents(ts, new System.String[] { "a", "llllllll", "i" }, new int[] { 0, 5, 8 }, new int[] { 4, 7, 9 });
}
public virtual void TestCharFilter11()
{
CharStream cs = new CharFilter1(new CharFilter1(CharReader.Get(new System.IO.StringReader(""))));
Assert.AreEqual(2, cs.CorrectOffset(0), "corrected offset is invalid");
}
/// Easy-use constructor that takes a <see cref="System.IO.TextReader" />.
public MappingCharFilter(NormalizeCharMap normMap, System.IO.TextReader @in)
: base(CharReader.Get(@in))
{
this.normMap = normMap;
}
public HTMLStripCharFilter(CharStream source)
: base(CharReader.Get(source))
{
safeReadAheadLimit = readAheadLimit - 3;
LoadEntityTable();
}