public virtual void TestBasic()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Iwc);
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
if (Random().NextBoolean())
{
ft.StoreTermVectors = true;
ft.StoreTermVectorPositions = Random().NextBoolean();
ft.StoreTermVectorOffsets = Random().NextBoolean();
}
Token[] tokens = new Token[] { MakeToken("a", 1, 0, 6), MakeToken("b", 1, 8, 9), MakeToken("a", 1, 9, 17), MakeToken("c", 1, 19, 50) };
doc.Add(new Field("content", new CannedTokenStream(tokens), ft));
w.AddDocument(doc);
IndexReader r = w.Reader;
w.Dispose();
DocsAndPositionsEnum dp = MultiFields.GetTermPositionsEnum(r, null, "content", new BytesRef("a"));
Assert.IsNotNull(dp);
Assert.AreEqual(0, dp.NextDoc());
Assert.AreEqual(2, dp.Freq());
Assert.AreEqual(0, dp.NextPosition());
Assert.AreEqual(0, dp.StartOffset());
Assert.AreEqual(6, dp.EndOffset());
Assert.AreEqual(2, dp.NextPosition());
Assert.AreEqual(9, dp.StartOffset());
Assert.AreEqual(17, dp.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dp.NextDoc());
dp = MultiFields.GetTermPositionsEnum(r, null, "content", new BytesRef("b"));
Assert.IsNotNull(dp);
Assert.AreEqual(0, dp.NextDoc());
Assert.AreEqual(1, dp.Freq());
Assert.AreEqual(1, dp.NextPosition());
Assert.AreEqual(8, dp.StartOffset());
Assert.AreEqual(9, dp.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dp.NextDoc());
dp = MultiFields.GetTermPositionsEnum(r, null, "content", new BytesRef("c"));
Assert.IsNotNull(dp);
Assert.AreEqual(0, dp.NextDoc());
Assert.AreEqual(1, dp.Freq());
Assert.AreEqual(3, dp.NextPosition());
Assert.AreEqual(19, dp.StartOffset());
Assert.AreEqual(50, dp.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dp.NextDoc());
r.Dispose();
dir.Dispose();
}
public virtual void TestEndOffsetPositionWithCachingTokenFilter()
{
Directory dir = NewDirectory();
Analyzer analyzer = new MockAnalyzer(Random());
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
IOException priorException = null;
TokenStream stream = analyzer.TokenStream("field", new StringReader("abcd "));
try
{
stream.Reset(); // TODO: weird to reset before wrapping with CachingTokenFilter... correct?
TokenStream cachedStream = new CachingTokenFilter(stream);
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
Field f = new Field("field", cachedStream, customType);
doc.Add(f);
doc.Add(f);
w.AddDocument(doc);
}
catch (IOException e)
{
priorException = e;
}
finally
{
IOUtils.CloseWhileHandlingException(priorException, stream);
}
w.Dispose();
IndexReader r = DirectoryReader.Open(dir);
TermsEnum termsEnum = r.GetTermVectors(0).Terms("field").Iterator(null);
Assert.IsNotNull(termsEnum.Next());
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.AreEqual(2, termsEnum.TotalTermFreq());
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(0, dpEnum.StartOffset());
Assert.AreEqual(4, dpEnum.EndOffset());
dpEnum.NextPosition();
Assert.AreEqual(8, dpEnum.StartOffset());
Assert.AreEqual(12, dpEnum.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
r.Dispose();
dir.Dispose();
}
public virtual void TestEndOffsetPositionStandard()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
Field f = NewField("field", "abcd the ", customType);
Field f2 = NewField("field", "crunch man", customType);
doc.Add(f);
doc.Add(f2);
w.AddDocument(doc);
w.Dispose();
IndexReader r = DirectoryReader.Open(dir);
TermsEnum termsEnum = r.GetTermVectors(0).Terms("field").Iterator(null);
Assert.IsNotNull(termsEnum.Next());
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(0, dpEnum.StartOffset());
Assert.AreEqual(4, dpEnum.EndOffset());
Assert.IsNotNull(termsEnum.Next());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(11, dpEnum.StartOffset());
Assert.AreEqual(17, dpEnum.EndOffset());
Assert.IsNotNull(termsEnum.Next());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(18, dpEnum.StartOffset());
Assert.AreEqual(21, dpEnum.EndOffset());
r.Dispose();
dir.Dispose();
}
public virtual void TestOffsetReader()
{
TermVectorsReader reader = Codec.Default.TermVectorsFormat().VectorsReader(Dir, Seg.Info, FieldInfos, NewIOContext(Random()));
Terms vector = reader.Get(0).Terms(TestFields[0]);
Assert.IsNotNull(vector);
TermsEnum termsEnum = vector.Iterator(null);
Assert.IsNotNull(termsEnum);
Assert.AreEqual(TestTerms.Length, vector.Size());
DocsAndPositionsEnum dpEnum = null;
for (int i = 0; i < TestTerms.Length; i++)
{
BytesRef text = termsEnum.Next();
Assert.IsNotNull(text);
string term = text.Utf8ToString();
Assert.AreEqual(TestTerms[i], term);
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsNotNull(dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(dpEnum.Freq(), Positions[i].Length);
for (int j = 0; j < Positions[i].Length; j++)
{
Assert.AreEqual(Positions[i][j], dpEnum.NextPosition());
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.IsNotNull(dpEnum);
Assert.AreEqual(dpEnum.Freq(), Positions[i].Length);
for (int j = 0; j < Positions[i].Length; j++)
{
Assert.AreEqual(Positions[i][j], dpEnum.NextPosition());
Assert.AreEqual(j * 10, dpEnum.StartOffset());
Assert.AreEqual(j * 10 + TestTerms[i].Length, dpEnum.EndOffset());
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
}
reader.Dispose();
}
/// <summary>
/// checks docs + freqs + positions + payloads, sequentially
/// </summary>
public void AssertDocsAndPositionsEnumEquals(string info, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs)
{
if (leftDocs == null || rightDocs == null)
{
Assert.IsNull(leftDocs);
Assert.IsNull(rightDocs);
return;
}
Assert.AreEqual(-1, leftDocs.DocID(), info);
Assert.AreEqual(-1, rightDocs.DocID(), info);
int docid;
while ((docid = leftDocs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.AreEqual(docid, rightDocs.NextDoc(), info);
int freq = leftDocs.Freq();
Assert.AreEqual(freq, rightDocs.Freq(), info);
for (int i = 0; i < freq; i++)
{
Assert.AreEqual(leftDocs.NextPosition(), rightDocs.NextPosition(), info);
Assert.AreEqual(leftDocs.Payload, rightDocs.Payload, info);
Assert.AreEqual(leftDocs.StartOffset(), rightDocs.StartOffset(), info);
Assert.AreEqual(leftDocs.EndOffset(), rightDocs.EndOffset(), info);
}
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, rightDocs.NextDoc(), info);
}
public virtual void TestRandom()
{
// token -> docID -> tokens
IDictionary <string, IDictionary <int?, IList <Token> > > actualTokens = new Dictionary <string, IDictionary <int?, IList <Token> > >();
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Iwc);
int numDocs = AtLeast(20);
//final int numDocs = AtLeast(5);
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
// TODO: randomize what IndexOptions we use; also test
// changing this up in one IW buffered segment...:
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
if (Random().NextBoolean())
{
ft.StoreTermVectors = true;
ft.StoreTermVectorOffsets = Random().NextBoolean();
ft.StoreTermVectorPositions = Random().NextBoolean();
}
for (int docCount = 0; docCount < numDocs; docCount++)
{
Document doc = new Document();
doc.Add(new IntField("id", docCount, Field.Store.NO));
IList <Token> tokens = new List <Token>();
int numTokens = AtLeast(100);
//final int numTokens = AtLeast(20);
int pos = -1;
int offset = 0;
//System.out.println("doc id=" + docCount);
for (int tokenCount = 0; tokenCount < numTokens; tokenCount++)
{
string text;
if (Random().NextBoolean())
{
text = "a";
}
else if (Random().NextBoolean())
{
text = "b";
}
else if (Random().NextBoolean())
{
text = "c";
}
else
{
text = "d";
}
int posIncr = Random().NextBoolean() ? 1 : Random().Next(5);
if (tokenCount == 0 && posIncr == 0)
{
posIncr = 1;
}
int offIncr = Random().NextBoolean() ? 0 : Random().Next(5);
int tokenOffset = Random().Next(5);
Token token = MakeToken(text, posIncr, offset + offIncr, offset + offIncr + tokenOffset);
if (!actualTokens.ContainsKey(text))
{
actualTokens[text] = new Dictionary <int?, IList <Token> >();
}
IDictionary <int?, IList <Token> > postingsByDoc = actualTokens[text];
if (!postingsByDoc.ContainsKey(docCount))
{
postingsByDoc[docCount] = new List <Token>();
}
postingsByDoc[docCount].Add(token);
tokens.Add(token);
pos += posIncr;
// stuff abs position into type:
token.Type = "" + pos;
offset += offIncr + tokenOffset;
//System.out.println(" " + token + " posIncr=" + token.getPositionIncrement() + " pos=" + pos + " off=" + token.StartOffset() + "/" + token.EndOffset() + " (freq=" + postingsByDoc.Get(docCount).Size() + ")");
}
doc.Add(new Field("content", new CannedTokenStream(tokens.ToArray()), ft));
w.AddDocument(doc);
}
DirectoryReader r = w.Reader;
w.Dispose();
string[] terms = new string[] { "a", "b", "c", "d" };
foreach (AtomicReaderContext ctx in r.Leaves)
{
// TODO: improve this
AtomicReader sub = (AtomicReader)ctx.Reader;
//System.out.println("\nsub=" + sub);
TermsEnum termsEnum = sub.Fields.Terms("content").Iterator(null);
DocsEnum docs = null;
DocsAndPositionsEnum docsAndPositions = null;
DocsAndPositionsEnum docsAndPositionsAndOffsets = null;
FieldCache.Ints docIDToID = FieldCache.DEFAULT.GetInts(sub, "id", false);
foreach (string term in terms)
{
//System.out.println(" term=" + term);
if (termsEnum.SeekExact(new BytesRef(term)))
{
docs = termsEnum.Docs(null, docs);
Assert.IsNotNull(docs);
int doc;
//System.out.println(" doc/freq");
while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
IList <Token> expected = actualTokens[term][docIDToID.Get(doc)];
//System.out.println(" doc=" + docIDToID.Get(doc) + " docID=" + doc + " " + expected.Size() + " freq");
Assert.IsNotNull(expected);
Assert.AreEqual(expected.Count, docs.Freq());
}
// explicitly exclude offsets here
docsAndPositions = termsEnum.DocsAndPositions(null, docsAndPositions, DocsAndPositionsEnum.FLAG_PAYLOADS);
Assert.IsNotNull(docsAndPositions);
//System.out.println(" doc/freq/pos");
while ((doc = docsAndPositions.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
IList <Token> expected = actualTokens[term][docIDToID.Get(doc)];
//System.out.println(" doc=" + docIDToID.Get(doc) + " " + expected.Size() + " freq");
Assert.IsNotNull(expected);
Assert.AreEqual(expected.Count, docsAndPositions.Freq());
foreach (Token token in expected)
{
int pos = Convert.ToInt32(token.Type);
//System.out.println(" pos=" + pos);
Assert.AreEqual(pos, docsAndPositions.NextPosition());
}
}
docsAndPositionsAndOffsets = termsEnum.DocsAndPositions(null, docsAndPositions);
Assert.IsNotNull(docsAndPositionsAndOffsets);
//System.out.println(" doc/freq/pos/offs");
while ((doc = docsAndPositionsAndOffsets.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
IList <Token> expected = actualTokens[term][docIDToID.Get(doc)];
//System.out.println(" doc=" + docIDToID.Get(doc) + " " + expected.Size() + " freq");
Assert.IsNotNull(expected);
Assert.AreEqual(expected.Count, docsAndPositionsAndOffsets.Freq());
foreach (Token token in expected)
{
int pos = Convert.ToInt32(token.Type);
//System.out.println(" pos=" + pos);
Assert.AreEqual(pos, docsAndPositionsAndOffsets.NextPosition());
Assert.AreEqual(token.StartOffset(), docsAndPositionsAndOffsets.StartOffset());
Assert.AreEqual(token.EndOffset(), docsAndPositionsAndOffsets.EndOffset());
}
}
}
}
// TODO: test advance:
}
r.Dispose();
dir.Dispose();
}
public virtual void DoTestNumbers(bool withPayloads)
{
Directory dir = NewDirectory();
Analyzer analyzer = withPayloads ? (Analyzer) new MockPayloadAnalyzer() : new MockAnalyzer(Random());
Iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
Iwc.SetMergePolicy(NewLogMergePolicy()); // will rely on docids a bit for skipping
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Iwc);
FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
if (Random().NextBoolean())
{
ft.StoreTermVectors = true;
ft.StoreTermVectorOffsets = Random().NextBoolean();
ft.StoreTermVectorPositions = Random().NextBoolean();
}
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new Field("numbers", English.IntToEnglish(i), ft));
doc.Add(new Field("oddeven", (i % 2) == 0 ? "even" : "odd", ft));
doc.Add(new StringField("id", "" + i, Field.Store.NO));
w.AddDocument(doc);
}
IndexReader reader = w.Reader;
w.Dispose();
string[] terms = new string[] { "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "hundred" };
foreach (string term in terms)
{
DocsAndPositionsEnum dp = MultiFields.GetTermPositionsEnum(reader, null, "numbers", new BytesRef(term));
int doc;
while ((doc = dp.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
string storedNumbers = reader.Document(doc).Get("numbers");
int freq = dp.Freq();
for (int i = 0; i < freq; i++)
{
dp.NextPosition();
int start = dp.StartOffset();
Debug.Assert(start >= 0);
int end = dp.EndOffset();
Debug.Assert(end >= 0 && end >= start);
// check that the offsets correspond to the term in the src text
Assert.IsTrue(storedNumbers.Substring(start, end - start).Equals(term));
if (withPayloads)
{
// check that we have a payload and it starts with "pos"
Assert.IsNotNull(dp.Payload);
BytesRef payload = dp.Payload;
Assert.IsTrue(payload.Utf8ToString().StartsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
}
}
}
// check we can skip correctly
int numSkippingTests = AtLeast(50);
for (int j = 0; j < numSkippingTests; j++)
{
int num = TestUtil.NextInt(Random(), 100, Math.Min(numDocs - 1, 999));
DocsAndPositionsEnum dp = MultiFields.GetTermPositionsEnum(reader, null, "numbers", new BytesRef("hundred"));
int doc = dp.Advance(num);
Assert.AreEqual(num, doc);
int freq = dp.Freq();
for (int i = 0; i < freq; i++)
{
string storedNumbers = reader.Document(doc).Get("numbers");
dp.NextPosition();
int start = dp.StartOffset();
Debug.Assert(start >= 0);
int end = dp.EndOffset();
Debug.Assert(end >= 0 && end >= start);
// check that the offsets correspond to the term in the src text
Assert.IsTrue(storedNumbers.Substring(start, end - start).Equals("hundred"));
if (withPayloads)
{
// check that we have a payload and it starts with "pos"
Assert.IsNotNull(dp.Payload);
BytesRef payload = dp.Payload;
Assert.IsTrue(payload.Utf8ToString().StartsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
}
}
// check that other fields (without offsets) work correctly
for (int i = 0; i < numDocs; i++)
{
DocsEnum dp = MultiFields.GetTermDocsEnum(reader, null, "id", new BytesRef("" + i), 0);
Assert.AreEqual(i, dp.NextDoc());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dp.NextDoc());
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestDoubleOffsetCounting()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
Field f = NewField("field", "abcd", customType);
doc.Add(f);
doc.Add(f);
Field f2 = NewField("field", "", customType);
doc.Add(f2);
doc.Add(f);
w.AddDocument(doc);
w.Dispose();
IndexReader r = DirectoryReader.Open(dir);
Terms vector = r.GetTermVectors(0).Terms("field");
Assert.IsNotNull(vector);
TermsEnum termsEnum = vector.Iterator(null);
Assert.IsNotNull(termsEnum.Next());
Assert.AreEqual("", termsEnum.Term().Utf8ToString());
// Token "" occurred once
Assert.AreEqual(1, termsEnum.TotalTermFreq());
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(8, dpEnum.StartOffset());
Assert.AreEqual(8, dpEnum.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
// Token "abcd" occurred three times
Assert.AreEqual(new BytesRef("abcd"), termsEnum.Next());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.AreEqual(3, termsEnum.TotalTermFreq());
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(0, dpEnum.StartOffset());
Assert.AreEqual(4, dpEnum.EndOffset());
dpEnum.NextPosition();
Assert.AreEqual(4, dpEnum.StartOffset());
Assert.AreEqual(8, dpEnum.EndOffset());
dpEnum.NextPosition();
Assert.AreEqual(8, dpEnum.StartOffset());
Assert.AreEqual(12, dpEnum.EndOffset());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
Assert.IsNull(termsEnum.Next());
r.Dispose();
dir.Dispose();
}
public override int StartOffset()
{
return(Current.StartOffset());
}
public override int StartOffset()
{
return(@in.StartOffset());
}