// create a new random tokenizer from classpath
private TokenizerSpec NewTokenizer(Random random, TextReader reader)
{
TokenizerSpec spec = new TokenizerSpec();
while (spec.tokenizer == null)
{
ConstructorInfo ctor = tokenizers[random.nextInt(tokenizers.size())];
StringBuilder descr = new StringBuilder();
CheckThatYouDidntReadAnythingReaderWrapper wrapper = new CheckThatYouDidntReadAnythingReaderWrapper(reader);
object[] args = NewTokenizerArgs(random, wrapper, ctor.GetParameters().Select(p => p.ParameterType).ToArray());
if (Broken(ctor, args))
{
continue;
}
spec.tokenizer = CreateComponent <Tokenizer>(ctor, args, descr);
if (spec.tokenizer != null)
{
spec.offsetsAreCorrect &= !BrokenOffsets(ctor, args);
spec.toString = descr.toString();
}
else
{
assertFalse(ctor.DeclaringType.Name + " has read something in ctor but failed with UOE/IAE", wrapper.readSomething);
}
}
return(spec);
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Random random = new Random(seed);
TokenizerSpec tokenizerSpec = NewTokenizer(random, reader);
//System.out.println("seed=" + seed + ",create tokenizer=" + tokenizerSpec.toString);
TokenFilterSpec filterSpec = NewFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
//System.out.println("seed=" + seed + ",create filter=" + filterSpec.toString);
return(new TokenStreamComponents(tokenizerSpec.tokenizer, filterSpec.stream));
}
public override string ToString()
{
Random random = new Random(seed);
StringBuilder sb = new StringBuilder();
CharFilterSpec charFilterSpec = NewCharFilterChain(random, new StringReader(""));
sb.Append("\ncharfilters=");
sb.Append(charFilterSpec.toString);
// intentional: initReader gets its own separate random
random = new Random(seed);
TokenizerSpec tokenizerSpec = NewTokenizer(random, charFilterSpec.reader);
sb.Append("\n");
sb.Append("tokenizer=");
sb.Append(tokenizerSpec.toString);
TokenFilterSpec tokenFilterSpec = NewFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
sb.Append("\n");
sb.Append("filters=");
sb.Append(tokenFilterSpec.toString);
sb.Append("\n");
sb.Append("offsetsAreCorrect=" + tokenFilterSpec.offsetsAreCorrect);
return(sb.ToString());
}
