/// <summary>Process postings from multiple segments all positioned on the
/// same term. Writes out merged entries into freqOutput and
/// the proxOutput streams.
///
/// </summary>
/// <param name="smis">array of segments
/// </param>
/// <param name="n">number of cells in the array actually occupied
/// </param>
/// <returns> number of documents across all segments where this term was found
/// </returns>
private int AppendPostings(SegmentMergeInfo[] smis, int n)
{
int lastDoc = 0;
int df = 0; // number of docs w/ term
ResetSkip();
for (int i = 0; i < n; i++)
{
SegmentMergeInfo smi = smis[i];
TermPositions postings = smi.GetPositions();
int base_Renamed = smi.base_Renamed;
int[] docMap = smi.GetDocMap();
postings.Seek(smi.termEnum);
while (postings.Next())
{
int doc = postings.Doc();
if (docMap != null)
{
doc = docMap[doc]; // map around deletions
}
doc += base_Renamed; // convert to merged space
if (doc < lastDoc)
{
throw new System.SystemException("docs out of order (" + doc + " < " + lastDoc + " )");
}
df++;
if ((df % skipInterval) == 0)
{
BufferSkip(lastDoc);
}
int docCode = (doc - lastDoc) << 1; // use low bit to flag freq=1
lastDoc = doc;
int freq = postings.Freq();
if (freq == 1)
{
freqOutput.WriteVInt(docCode | 1); // write doc & freq=1
}
else
{
freqOutput.WriteVInt(docCode); // write doc
freqOutput.WriteVInt(freq); // write frequency in doc
}
int lastPosition = 0; // write position deltas
for (int j = 0; j < freq; j++)
{
int position = postings.NextPosition();
proxOutput.WriteVInt(position - lastPosition);
lastPosition = position;
}
}
}
return(df);
}
/// <summary>Process postings from multiple segments all positioned on the
/// same term. Writes out merged entries into freqOutput and
/// the proxOutput streams.
///
/// </summary>
/// <param name="smis">array of segments
/// </param>
/// <param name="n">number of cells in the array actually occupied
/// </param>
/// <returns> number of documents across all segments where this term was found
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int AppendPostings(FormatPostingsTermsConsumer termsConsumer, SegmentMergeInfo[] smis, int n)
{
FormatPostingsDocsConsumer docConsumer = termsConsumer.AddTerm(smis[0].term.Text);
int df = 0;
for (int i = 0; i < n; i++)
{
SegmentMergeInfo smi = smis[i];
TermPositions postings = smi.GetPositions();
System.Diagnostics.Debug.Assert(postings != null);
int base_Renamed = smi.base_Renamed;
int[] docMap = smi.GetDocMap();
postings.Seek(smi.termEnum);
while (postings.Next())
{
df++;
int doc = postings.Doc;
if (docMap != null)
{
doc = docMap[doc]; // map around deletions
}
doc += base_Renamed; // convert to merged space
int freq = postings.Freq;
FormatPostingsPositionsConsumer posConsumer = docConsumer.AddDoc(doc, freq);
if (!omitTermFreqAndPositions)
{
for (int j = 0; j < freq; j++)
{
int position = postings.NextPosition();
int payloadLength = postings.PayloadLength;
if (payloadLength > 0)
{
if (payloadBuffer == null || payloadBuffer.Length < payloadLength)
{
payloadBuffer = new byte[payloadLength];
}
postings.GetPayload(payloadBuffer, 0);
}
posConsumer.AddPosition(position, payloadBuffer, 0, payloadLength);
}
posConsumer.Finish();
}
}
}
docConsumer.Finish();
return(df);
}
/// <summary>
/// Process postings from multiple segments without tf, all positioned on the same term.
/// Writes out merged entries only into freqOutput, proxOut is not written.
/// </summary>
/// <param name="smis">smis array of segments</param>
/// <param name="n">number of cells in the array actually occupied</param>
/// <returns></returns>
private int AppendPostingsNoTf(SegmentMergeInfo[] smis, int n)
{
int lastDoc = 0;
int df = 0; // number of docs w/ term
skipListWriter.ResetSkip();
int lastPayloadLength = -1; // ensures that we write the first length
for (int i = 0; i < n; i++)
{
SegmentMergeInfo smi = smis[i];
TermPositions postings = smi.GetPositions();
System.Diagnostics.Debug.Assert(postings != null);
int base_Renamed = smi.base_Renamed;
int[] docMap = smi.GetDocMap();
postings.Seek(smi.termEnum);
while (postings.Next())
{
int doc = postings.Doc();
if (docMap != null)
{
doc = docMap[doc]; // map around deletions
}
doc += base_Renamed; // convert to merged space
if (doc < 0 || (df > 0 && doc <= lastDoc))
{
throw new CorruptIndexException("docs out of order (" + doc +
" <= " + lastDoc + " )");
}
df++;
if ((df % skipInterval) == 0)
{
skipListWriter.SetSkipData(lastDoc, false, lastPayloadLength);
skipListWriter.BufferSkip(df);
}
int docCode = (doc - lastDoc);
lastDoc = doc;
freqOutput.WriteVInt(docCode); // write doc & freq=1
}
}
return(df);
}
/// <summary>Process postings from multiple segments all positioned on the
/// same term. Writes out merged entries into freqOutput and
/// the proxOutput streams.
///
/// </summary>
/// <param name="smis">array of segments
/// </param>
/// <param name="n">number of cells in the array actually occupied
/// </param>
/// <returns> number of documents across all segments where this term was found
/// </returns>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
private int AppendPostings(SegmentMergeInfo[] smis, int n)
{
int lastDoc = 0;
int df = 0; // number of docs w/ term
skipListWriter.ResetSkip();
bool storePayloads = fieldInfos.FieldInfo(smis[0].term.field).storePayloads;
int lastPayloadLength = -1; // ensures that we write the first length
for (int i = 0; i < n; i++)
{
SegmentMergeInfo smi = smis[i];
TermPositions postings = smi.GetPositions();
System.Diagnostics.Debug.Assert(postings != null);
int base_Renamed = smi.base_Renamed;
int[] docMap = smi.GetDocMap();
postings.Seek(smi.termEnum);
while (postings.Next())
{
int doc = postings.Doc();
if (docMap != null)
{
doc = docMap[doc]; // map around deletions
}
doc += base_Renamed; // convert to merged space
if (doc < 0 || (df > 0 && doc <= lastDoc))
{
throw new CorruptIndexException("docs out of order (" + doc + " <= " + lastDoc + " )");
}
df++;
if ((df % skipInterval) == 0)
{
skipListWriter.SetSkipData(lastDoc, storePayloads, lastPayloadLength);
skipListWriter.BufferSkip(df);
}
int docCode = (doc - lastDoc) << 1; // use low bit to flag freq=1
lastDoc = doc;
int freq = postings.Freq();
if (freq == 1)
{
freqOutput.WriteVInt(docCode | 1); // write doc & freq=1
}
else
{
freqOutput.WriteVInt(docCode); // write doc
freqOutput.WriteVInt(freq); // write frequency in doc
}
/** See {@link DocumentWriter#writePostings(Posting[], String) for
* documentation about the encoding of positions and payloads
*/
int lastPosition = 0; // write position deltas
for (int j = 0; j < freq; j++)
{
int position = postings.NextPosition();
int delta = position - lastPosition;
if (storePayloads)
{
int payloadLength = postings.GetPayloadLength();
if (payloadLength == lastPayloadLength)
{
proxOutput.WriteVInt(delta * 2);
}
else
{
proxOutput.WriteVInt(delta * 2 + 1);
proxOutput.WriteVInt(payloadLength);
lastPayloadLength = payloadLength;
}
if (payloadLength > 0)
{
if (payloadBuffer == null || payloadBuffer.Length < payloadLength)
{
payloadBuffer = new byte[payloadLength];
}
postings.GetPayload(payloadBuffer, 0);
proxOutput.WriteBytes(payloadBuffer, 0, payloadLength);
}
}
else
{
proxOutput.WriteVInt(delta);
}
lastPosition = position;
}
}
}
return(df);
}
