private void AddPositions(DocsAndPositionsEnum @in, IndexOutput @out)
{
int freq = @in.Freq;
@out.WriteVInt32(freq);
int previousPosition = 0;
int previousEndOffset = 0;
for (int i = 0; i < freq; i++)
{
int pos = @in.NextPosition();
BytesRef payload = @in.GetPayload();
// The low-order bit of token is set only if there is a payload, the
// previous bits are the delta-encoded position.
int token = (pos - previousPosition) << 1 | (payload is null ? 0 : 1);
@out.WriteVInt32(token);
previousPosition = pos;
if (storeOffsets) // don't encode offsets if they are not stored
{
int startOffset = @in.StartOffset;
int endOffset = @in.EndOffset;
@out.WriteVInt32(startOffset - previousEndOffset);
@out.WriteVInt32(endOffset - startOffset);
previousEndOffset = endOffset;
}
if (payload != null)
{
@out.WriteVInt32(payload.Length);
@out.WriteBytes(payload.Bytes, payload.Offset, payload.Length);
}
}
}
protected virtual void LoadPayload(Term term)
{
DocsAndPositionsEnum dp = m_reader.GetTermPositionsEnum(term);
int docID = -1;
while ((docID = dp.NextDoc()) != DocsEnum.NO_MORE_DOCS)
{
if (dp.Freq > 0)
{
dp.NextPosition();
BytesRef payload = dp.GetPayload();
if (payload != null)
{
Add(docID, payload.Bytes, payload.Length);
}
}
}
// save the last page
while (m_curSlot < MAX_SLOTS)
{
m_curPage[m_curSlot++] = MISSING;
}
m_list[m_curPageNo] = CopyPage(new int[m_curData]); // optimize the page to make getNumItems work
m_curPage = null;
}
public override void Load()
{
DocsAndPositionsEnum docPosEnum = m_reader.GetTermPositionsEnum(m_sizeTerm);
if (docPosEnum == null)
{
return;
}
int docID = -1;
while ((docID = docPosEnum.NextDoc()) != DocsEnum.NO_MORE_DOCS)
{
if (docPosEnum.Freq > 0)
{
docPosEnum.NextPosition();
int len = BytesToInt32(docPosEnum.GetPayload().Bytes);
Allocate(docID, Math.Min(len, m_maxItems), true);
}
}
}
protected internal virtual void ProcessPayload(Similarity similarity)
{
if (termSpans.IsPayloadAvailable)
{
DocsAndPositionsEnum postings = termSpans.Postings;
m_payload = postings.GetPayload();
if (m_payload != null)
{
m_payloadScore = outerInstance.outerInstance.m_function.CurrentScore(m_doc, outerInstance.outerInstance.Term.Field, m_spans.Start, m_spans.End, m_payloadsSeen, m_payloadScore, m_docScorer.ComputePayloadFactor(m_doc, m_spans.Start, m_spans.End, m_payload));
}
else
{
m_payloadScore = outerInstance.outerInstance.m_function.CurrentScore(m_doc, outerInstance.outerInstance.Term.Field, m_spans.Start, m_spans.End, m_payloadsSeen, m_payloadScore, 1F);
}
m_payloadsSeen++;
}
else
{
// zero out the payload?
}
}
protected void CompareTermVectors(Terms terms, Terms memTerms, string field_name)
{
TermsEnum termEnum = terms.GetEnumerator();
TermsEnum memTermEnum = memTerms.GetEnumerator();
while (termEnum.MoveNext())
{
assertTrue(memTermEnum.MoveNext());
assertEquals(termEnum.TotalTermFreq, memTermEnum.TotalTermFreq);
DocsAndPositionsEnum docsPosEnum = termEnum.DocsAndPositions(null, null, 0);
DocsAndPositionsEnum memDocsPosEnum = memTermEnum.DocsAndPositions(null, null, 0);
String currentTerm = termEnum.Term.Utf8ToString();
assertEquals("Token mismatch for field: " + field_name, currentTerm, memTermEnum.Term.Utf8ToString());
docsPosEnum.NextDoc();
memDocsPosEnum.NextDoc();
int freq = docsPosEnum.Freq;
assertEquals(freq, memDocsPosEnum.Freq);
for (int i = 0; i < freq; i++)
{
string failDesc = " (field:" + field_name + " term:" + currentTerm + ")";
int memPos = memDocsPosEnum.NextPosition();
int pos = docsPosEnum.NextPosition();
assertEquals("Position test failed" + failDesc, memPos, pos);
assertEquals("Start offset test failed" + failDesc, memDocsPosEnum.StartOffset, docsPosEnum.StartOffset);
assertEquals("End offset test failed" + failDesc, memDocsPosEnum.EndOffset, docsPosEnum.EndOffset);
assertEquals("Missing payload test failed" + failDesc, docsPosEnum.GetPayload(), null);
}
}
assertFalse("Still some tokens not processed", memTermEnum.MoveNext());
}
/// <summary>
/// Low level api. Returns a token stream generated from a <see cref="Terms"/>. This
/// can be used to feed the highlighter with a pre-parsed token
/// stream. The <see cref="Terms"/> must have offsets available.
/// <para/>
/// In my tests the speeds to recreate 1000 token streams using this method are:
/// <list type="bullet">
/// <item><description>
/// with TermVector offset only data stored - 420 milliseconds
/// </description></item>
/// <item><description>
/// with TermVector offset AND position data stored - 271 milliseconds
/// (nb timings for TermVector with position data are based on a tokenizer with contiguous
/// positions - no overlaps or gaps)
/// </description></item>
/// <item><description>
/// The cost of not using TermPositionVector to store
/// pre-parsed content and using an analyzer to re-parse the original content:
/// - reanalyzing the original content - 980 milliseconds
/// </description></item>
/// </list>
///
/// The re-analyze timings will typically vary depending on -
/// <list type="number">
/// <item><description>
/// The complexity of the analyzer code (timings above were using a
/// stemmer/lowercaser/stopword combo)
/// </description></item>
/// <item><description>
/// The number of other fields (Lucene reads ALL fields off the disk
/// when accessing just one document field - can cost dear!)
/// </description></item>
/// <item><description>
/// Use of compression on field storage - could be faster due to compression (less disk IO)
/// or slower (more CPU burn) depending on the content.
/// </description></item>
/// </list>
/// </summary>
/// <param name="tpv"></param>
/// <param name="tokenPositionsGuaranteedContiguous">true if the token position numbers have no overlaps or gaps. If looking
/// to eek out the last drops of performance, set to true. If in doubt, set to false.</param>
/// <exception cref="ArgumentException">if no offsets are available</exception>
public static TokenStream GetTokenStream(Terms tpv,
bool tokenPositionsGuaranteedContiguous)
{
if (!tpv.HasOffsets)
{
throw new ArgumentException("Cannot create TokenStream from Terms without offsets");
}
if (!tokenPositionsGuaranteedContiguous && tpv.HasPositions)
{
return(new TokenStreamFromTermPositionVector(tpv));
}
bool hasPayloads = tpv.HasPayloads;
// code to reconstruct the original sequence of Tokens
TermsEnum termsEnum = tpv.GetEnumerator();
int totalTokens = 0;
while (termsEnum.MoveNext())
{
totalTokens += (int)termsEnum.TotalTermFreq;
}
Token[] tokensInOriginalOrder = new Token[totalTokens];
JCG.List <Token> unsortedTokens = null;
termsEnum = tpv.GetEnumerator();
DocsAndPositionsEnum dpEnum = null;
while (termsEnum.MoveNext())
{
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
if (dpEnum == null)
{
throw new ArgumentException("Required TermVector Offset information was not found");
}
string term = termsEnum.Term.Utf8ToString();
dpEnum.NextDoc();
int freq = dpEnum.Freq;
for (int posUpto = 0; posUpto < freq; posUpto++)
{
int pos = dpEnum.NextPosition();
if (dpEnum.StartOffset < 0)
{
throw new ArgumentException("Required TermVector Offset information was not found");
}
Token token = new Token(term, dpEnum.StartOffset, dpEnum.EndOffset);
if (hasPayloads)
{
// Must make a deep copy of the returned payload,
// since D&PEnum API is allowed to re-use on every
// call:
token.Payload = BytesRef.DeepCopyOf(dpEnum.GetPayload());
}
if (tokenPositionsGuaranteedContiguous && pos != -1)
{
// We have positions stored and a guarantee that the token position
// information is contiguous
// This may be fast BUT wont work if Tokenizers used which create >1
// token in same position or
// creates jumps in position numbers - this code would fail under those
// circumstances
// tokens stored with positions - can use this to index straight into
// sorted array
tokensInOriginalOrder[pos] = token;
}
else
{
// tokens NOT stored with positions or not guaranteed contiguous - must
// add to list and sort later
if (unsortedTokens == null)
{
unsortedTokens = new JCG.List <Token>();
}
unsortedTokens.Add(token);
}
}
}
// If the field has been stored without position data we must perform a sort
if (unsortedTokens != null)
{
tokensInOriginalOrder = unsortedTokens.ToArray();
ArrayUtil.TimSort(tokensInOriginalOrder, new TokenComparer());
//tokensInOriginalOrder = tokensInOriginalOrder
// .OrderBy(t => t, new TokenComparer() )
// .ToArray();
}
return(new StoredTokenStream(tokensInOriginalOrder));
}
public virtual void TestDocsAndPositionsEnum()
{
TermsEnum termsEnum = reader.GetTerms(DOC_POSITIONS_FIELD).GetIterator(null);
assertEquals(TermsEnum.SeekStatus.FOUND, termsEnum.SeekCeil(new BytesRef(DOC_POSITIONS_TERM)));
DocsAndPositionsEnum sortedPositions = termsEnum.DocsAndPositions(null, null);
int doc;
// test nextDoc()
while ((doc = sortedPositions.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
int freq = sortedPositions.Freq;
assertEquals("incorrect freq for doc=" + doc, sortedValues[doc] / 10 + 1, freq);
for (int i = 0; i < freq; i++)
{
assertEquals("incorrect position for doc=" + doc, i, sortedPositions.NextPosition());
if (!DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat(DOC_POSITIONS_FIELD)))
{
assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.StartOffset);
assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.EndOffset);
}
assertEquals("incorrect payload for doc=" + doc, freq - i, int.Parse(sortedPositions.GetPayload().Utf8ToString(), CultureInfo.InvariantCulture));
}
}
// test advance()
DocsAndPositionsEnum reuse = sortedPositions;
sortedPositions = termsEnum.DocsAndPositions(null, reuse);
if (sortedPositions is SortingAtomicReader.SortingDocsAndPositionsEnum)
{
assertTrue(((SortingAtomicReader.SortingDocsAndPositionsEnum)sortedPositions).Reused(reuse)); // make sure reuse worked
}
doc = 0;
while ((doc = sortedPositions.Advance(doc + TestUtil.NextInt32(Random, 1, 5))) != DocIdSetIterator.NO_MORE_DOCS)
{
int freq = sortedPositions.Freq;
assertEquals("incorrect freq for doc=" + doc, sortedValues[doc] / 10 + 1, freq);
for (int i = 0; i < freq; i++)
{
assertEquals("incorrect position for doc=" + doc, i, sortedPositions.NextPosition());
if (!DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat(DOC_POSITIONS_FIELD)))
{
assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.StartOffset);
assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.EndOffset);
}
assertEquals("incorrect payload for doc=" + doc, freq - i, int.Parse(sortedPositions.GetPayload().Utf8ToString(), CultureInfo.InvariantCulture));
}
}
}
private readonly IPayloadAttribute payloadAttribute; // LUCENENET: marked readonly
///<summary>Constructor</summary>
/// <param name="vector">
/// Terms that contains the data for
/// creating the <see cref="TokenStream"/>. Must have positions and offsets.
/// </param>
public TokenStreamFromTermPositionVector(Terms vector)
{
termAttribute = AddAttribute <ICharTermAttribute>();
positionIncrementAttribute = AddAttribute <IPositionIncrementAttribute>();
offsetAttribute = AddAttribute <IOffsetAttribute>();
payloadAttribute = AddAttribute <IPayloadAttribute>();
bool hasOffsets = vector.HasOffsets;
bool hasPayloads = vector.HasPayloads;
TermsEnum termsEnum = vector.GetEnumerator();
BytesRef text;
DocsAndPositionsEnum dpEnum = null;
while (termsEnum.MoveNext())
{
text = termsEnum.Term;
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
dpEnum.NextDoc();
int freq = dpEnum.Freq;
for (int j = 0; j < freq; j++)
{
int pos = dpEnum.NextPosition();
Token token;
if (hasOffsets)
{
token = new Token(text.Utf8ToString(),
dpEnum.StartOffset,
dpEnum.EndOffset);
}
else
{
token = new Token();
token.SetEmpty().Append(text.Utf8ToString());
}
if (hasPayloads)
{
// Must make a deep copy of the returned payload,
// since D&PEnum API is allowed to re-use on every
// call:
token.Payload = BytesRef.DeepCopyOf(dpEnum.GetPayload());
}
// Yes - this is the position, not the increment! This is for
// sorting. This value
// will be corrected before use.
token.PositionIncrement = pos;
this.positionedTokens.Add(token);
}
}
CollectionUtil.TimSort(this.positionedTokens, tokenComparer);
int lastPosition = -1;
foreach (Token token in this.positionedTokens)
{
int thisPosition = token.PositionIncrement;
token.PositionIncrement = thisPosition - lastPosition;
lastPosition = thisPosition;
}
this.tokensAtCurrentPosition = this.positionedTokens.GetEnumerator();
}
/// <summary>
/// Safe (but, slowish) default method to write every
/// vector field in the document.
/// </summary>
protected void AddAllDocVectors(Fields vectors, MergeState mergeState)
{
if (vectors == null)
{
StartDocument(0);
FinishDocument();
return;
}
int numFields = vectors.Count;
if (numFields == -1)
{
// count manually! TODO: Maybe enforce that Fields.size() returns something valid?
numFields = 0;
//for (IEnumerator<string> it = vectors.Iterator(); it.hasNext();)
foreach (string it in vectors)
{
numFields++;
}
}
StartDocument(numFields);
string lastFieldName = null;
TermsEnum termsEnum = null;
DocsAndPositionsEnum docsAndPositionsEnum = null;
int fieldCount = 0;
foreach (string fieldName in vectors)
{
fieldCount++;
FieldInfo fieldInfo = mergeState.FieldInfos.FieldInfo(fieldName);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(lastFieldName == null || fieldName.CompareToOrdinal(lastFieldName) > 0, "lastFieldName={0} fieldName={1}", lastFieldName, fieldName);
}
lastFieldName = fieldName;
Terms terms = vectors.GetTerms(fieldName);
if (terms == null)
{
// FieldsEnum shouldn't lie...
continue;
}
bool hasPositions = terms.HasPositions;
bool hasOffsets = terms.HasOffsets;
bool hasPayloads = terms.HasPayloads;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!hasPayloads || hasPositions);
}
int numTerms = (int)terms.Count;
if (numTerms == -1)
{
// count manually. It is stupid, but needed, as Terms.size() is not a mandatory statistics function
numTerms = 0;
termsEnum = terms.GetEnumerator(termsEnum);
while (termsEnum.MoveNext())
{
numTerms++;
}
}
StartField(fieldInfo, numTerms, hasPositions, hasOffsets, hasPayloads);
termsEnum = terms.GetEnumerator(termsEnum);
int termCount = 0;
while (termsEnum.MoveNext())
{
termCount++;
int freq = (int)termsEnum.TotalTermFreq;
StartTerm(termsEnum.Term, freq);
if (hasPositions || hasOffsets)
{
docsAndPositionsEnum = termsEnum.DocsAndPositions(null, docsAndPositionsEnum);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(docsAndPositionsEnum != null);
}
int docID = docsAndPositionsEnum.NextDoc();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(docID != DocIdSetIterator.NO_MORE_DOCS);
Debugging.Assert(docsAndPositionsEnum.Freq == freq);
}
for (int posUpto = 0; posUpto < freq; posUpto++)
{
int pos = docsAndPositionsEnum.NextPosition();
int startOffset = docsAndPositionsEnum.StartOffset;
int endOffset = docsAndPositionsEnum.EndOffset;
BytesRef payload = docsAndPositionsEnum.GetPayload();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!hasPositions || pos >= 0);
}
AddPosition(pos, startOffset, endOffset, payload);
}
}
FinishTerm();
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(termCount == numTerms);
}
FinishField();
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(fieldCount == numFields);
}
FinishDocument();
}
public override BytesRef GetPayload()
{
return(current.GetPayload());
}
