/// <summary>
/// Construct a new <see cref="ICUTokenizer"/> that breaks text into words from the given
/// <see cref="TextReader"/>, using a tailored <see cref="BreakIterator"/> configuration.
/// </summary>
/// <param name="factory"><see cref="Lucene.Net.Util.AttributeSource.AttributeFactory"/> to use.</param>
/// <param name="input"><see cref="TextReader"/> containing text to tokenize.</param>
/// <param name="config">Tailored <see cref="BreakIterator"/> configuration.</param>
public ICUTokenizer(AttributeFactory factory, TextReader input, ICUTokenizerConfig config)
: base(factory, input)
{
this.offsetAtt = AddAttribute <IOffsetAttribute>();
this.termAtt = AddAttribute <ICharTermAttribute>();
this.typeAtt = AddAttribute <ITypeAttribute>();
this.scriptAtt = AddAttribute <IScriptAttribute>();
this.config = config;
breaker = new CompositeBreakIterator(config);
}
public void TestTokenAttributes()
{
using TokenStream ts = a.GetTokenStream("dummy", "This is a test");
IScriptAttribute scriptAtt = ts.AddAttribute <IScriptAttribute>();
ts.Reset();
while (ts.IncrementToken())
{
assertEquals(UScript.Latin, scriptAtt.Code);
assertEquals(UScript.GetName(UScript.Latin), scriptAtt.GetName());
assertEquals(UScript.GetShortName(UScript.Latin), scriptAtt.GetShortName());
assertTrue(ts.ReflectAsString(false).Contains("script=Latin"));
}
ts.End();
}