public Tokenizer <T> getTokenizer(TextReader r, String extraOptions)
{
Dictionary <String, String> prop = StringUtils.stringToProperties(extraOptions);
bool tokenizeNewlines =
PropertiesUtils.getBool(prop, "tokenizeNLs", this.tokenizeNLs);
return(new WhitespaceTokenizer <T>(factory, r, tokenizeNewlines));
}
public WhitespaceTokenizerFactory(LexedTokenFactory <T> factory,
String options)
{
this.factory = factory;
Dictionary <String, String> prop = StringUtils.stringToProperties(options);
this.tokenizeNLs = PropertiesUtils.getBool(prop, "tokenizeNLs", false);
}
public void setOptions(String options)
{
Dictionary <String, String> prop = StringUtils.stringToProperties(options);
tokenizeNLs = PropertiesUtils.getBool(prop, "tokenizeNLs", tokenizeNLs);
}
