protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader);
if (fieldName.Contains("payloadsFixed"))
{
TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);
return(new TokenStreamComponents(tokenizer, filter));
}
else if (fieldName.Contains("payloadsVariable"))
{
TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer);
return(new TokenStreamComponents(tokenizer, filter));
}
else
{
return(new TokenStreamComponents(tokenizer));
}
}
public virtual void Test()
{
Directory dir = NewDirectory();
Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
{
Tokenizer tokenizer = new MockTokenizer(reader);
if (fieldName.Contains("payloadsFixed"))
{
TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);
return(new TokenStreamComponents(tokenizer, filter));
}
else if (fieldName.Contains("payloadsVariable"))
{
TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer);
return(new TokenStreamComponents(tokenizer, filter));
}
else
{
return(new TokenStreamComponents(tokenizer));
}
}, reuseStrategy: Analyzer.PER_FIELD_REUSE_STRATEGY);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
iwc.SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()));
// TODO we could actually add more fields implemented with different PFs
// or, just put this test into the usual rotation?
RandomIndexWriter iw = new RandomIndexWriter(Random, dir, (IndexWriterConfig)iwc.Clone());
Document doc = new Document();
FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);
// turn this on for a cross-check
docsOnlyType.StoreTermVectors = true;
docsOnlyType.IndexOptions = IndexOptions.DOCS_ONLY;
FieldType docsAndFreqsType = new FieldType(TextField.TYPE_NOT_STORED);
// turn this on for a cross-check
docsAndFreqsType.StoreTermVectors = true;
docsAndFreqsType.IndexOptions = IndexOptions.DOCS_AND_FREQS;
FieldType positionsType = new FieldType(TextField.TYPE_NOT_STORED);
// turn these on for a cross-check
positionsType.StoreTermVectors = true;
positionsType.StoreTermVectorPositions = true;
positionsType.StoreTermVectorOffsets = true;
positionsType.StoreTermVectorPayloads = true;
FieldType offsetsType = new FieldType(positionsType);
offsetsType.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
Field field1 = new Field("field1docs", "", docsOnlyType);
Field field2 = new Field("field2freqs", "", docsAndFreqsType);
Field field3 = new Field("field3positions", "", positionsType);
Field field4 = new Field("field4offsets", "", offsetsType);
Field field5 = new Field("field5payloadsFixed", "", positionsType);
Field field6 = new Field("field6payloadsVariable", "", positionsType);
Field field7 = new Field("field7payloadsFixedOffsets", "", offsetsType);
Field field8 = new Field("field8payloadsVariableOffsets", "", offsetsType);
doc.Add(field1);
doc.Add(field2);
doc.Add(field3);
doc.Add(field4);
doc.Add(field5);
doc.Add(field6);
doc.Add(field7);
doc.Add(field8);
for (int i = 0; i < MAXDOC; i++)
{
string stringValue = Convert.ToString(i) + " verycommon " + English.Int32ToEnglish(i).Replace('-', ' ') + " " + TestUtil.RandomSimpleString(Random);
field1.SetStringValue(stringValue);
field2.SetStringValue(stringValue);
field3.SetStringValue(stringValue);
field4.SetStringValue(stringValue);
field5.SetStringValue(stringValue);
field6.SetStringValue(stringValue);
field7.SetStringValue(stringValue);
field8.SetStringValue(stringValue);
iw.AddDocument(doc);
}
iw.Dispose();
Verify(dir);
TestUtil.CheckIndex(dir); // for some extra coverage, checkIndex before we forceMerge
iwc.SetOpenMode(OpenMode.APPEND);
IndexWriter iw2 = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
iw2.ForceMerge(1);
iw2.Dispose();
Verify(dir);
dir.Dispose();
}
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader);
if (fieldName.Contains("payloadsFixed"))
{
TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);
return new TokenStreamComponents(tokenizer, filter);
}
else if (fieldName.Contains("payloadsVariable"))
{
TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
else
{
return new TokenStreamComponents(tokenizer);
}
}
