internal override void FinishDocument(TermsHash termsHash)
{
Debug.Assert(docWriter.TestPoint("TermVectorsTermsWriter.finishDocument start"));
if (!hasVectors)
{
return;
}
InitTermVectorsWriter();
Fill(docState.docID);
// Append term vectors to the real outputs:
writer.StartDocument(numVectorFields);
for (int i = 0; i < numVectorFields; i++)
{
perFields[i].FinishDocument();
}
writer.FinishDocument();
Debug.Assert(lastDocID == docState.docID, "lastDocID=" + lastDocID + " docState.docID=" + docState.docID);
lastDocID++;
termsHash.Reset();
Reset();
Debug.Assert(docWriter.TestPoint("TermVectorsTermsWriter.finishDocument end"));
}
public TermsHashPerField(DocInverterPerField docInverterPerField, TermsHash termsHash, TermsHash nextTermsHash, FieldInfo fieldInfo)
{
intPool = termsHash.intPool;
bytePool = termsHash.bytePool;
termBytePool = termsHash.termBytePool;
docState = termsHash.docState;
this.termsHash = termsHash;
bytesUsed = termsHash.bytesUsed;
fieldState = docInverterPerField.fieldState;
this.consumer = termsHash.consumer.AddField(this, fieldInfo);
PostingsBytesStartArray byteStarts = new PostingsBytesStartArray(this, bytesUsed);
bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
streamCount = consumer.StreamCount;
numPostingInt = 2 * streamCount;
this.fieldInfo = fieldInfo;
if (nextTermsHash != null)
{
nextPerField = (TermsHashPerField)nextTermsHash.AddField(docInverterPerField, fieldInfo);
}
else
{
nextPerField = null;
}
}
public TermsHash(DocumentsWriterPerThread docWriter, TermsHashConsumer consumer, bool trackAllocations, TermsHash nextTermsHash)
{
this.docState = docWriter.docState;
this.consumer = consumer;
this.trackAllocations = trackAllocations;
this.nextTermsHash = nextTermsHash;
this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.NewCounter();
intPool = new Int32BlockPool(docWriter.intBlockAllocator);
bytePool = new ByteBlockPool(docWriter.byteBlockAllocator);
if (nextTermsHash != null)
{
// We are primary
primary = true;
termBytePool = bytePool;
nextTermsHash.termBytePool = bytePool;
}
else
{
primary = false;
}
}
internal override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
{
/*
* this is the current indexing chain:
*
* DocConsumer / DocConsumerPerThread
* --> code: DocFieldProcessor
* --> DocFieldConsumer / DocFieldConsumerPerField
* --> code: DocFieldConsumers / DocFieldConsumersPerField
* --> code: DocInverter / DocInverterPerField
* --> InvertedDocConsumer / InvertedDocConsumerPerField
* --> code: TermsHash / TermsHashPerField
* --> TermsHashConsumer / TermsHashConsumerPerField
* --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
* --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
* --> InvertedDocEndConsumer / InvertedDocConsumerPerField
* --> code: NormsConsumer / NormsConsumerPerField
* --> StoredFieldsConsumer
* --> TwoStoredFieldConsumers
* -> code: StoredFieldsProcessor
* -> code: DocValuesProcessor
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true,
new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
NormsConsumer normsWriter = new NormsConsumer();
DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(
new StoredFieldsProcessor(documentsWriterPerThread),
new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
return(new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields));
}
internal override void FinishDocument(TermsHash termsHash)
{
// LUCENENET: .NET doesn't support asserts in release mode
if (Lucene.Net.Diagnostics.Debugging.AssertsEnabled)
{
docWriter.TestPoint("TermVectorsTermsWriter.finishDocument start");
}
if (!hasVectors)
{
return;
}
InitTermVectorsWriter();
Fill(docState.docID);
// Append term vectors to the real outputs:
writer.StartDocument(numVectorFields);
for (int i = 0; i < numVectorFields; i++)
{
perFields[i].FinishDocument();
}
writer.FinishDocument();
Debug.Assert(lastDocID == docState.docID, "lastDocID=" + lastDocID + " docState.docID=" + docState.docID);
lastDocID++;
termsHash.Reset();
Reset();
// LUCENENET: .NET doesn't support asserts in release mode
if (Lucene.Net.Diagnostics.Debugging.AssertsEnabled)
{
docWriter.TestPoint("TermVectorsTermsWriter.finishDocument end");
}
}
public override void Flush(IDictionary <string, TermsHashConsumerPerField> fieldsToFlush, SegmentWriteState state)
{
// Gather all FieldData's that have postings, across all
// ThreadStates
IList <FreqProxTermsWriterPerField> allFields = new List <FreqProxTermsWriterPerField>();
foreach (TermsHashConsumerPerField f in fieldsToFlush.Values)
{
FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField)f;
if (perField.termsHashPerField.bytesHash.Count > 0)
{
allFields.Add(perField);
}
}
int numAllFields = allFields.Count;
// Sort by field name
CollectionUtil.IntroSort(allFields);
FieldsConsumer consumer = state.SegmentInfo.Codec.PostingsFormat.FieldsConsumer(state);
bool success = false;
try
{
TermsHash termsHash = null;
/*
* Current writer chain:
* FieldsConsumer
* -> IMPL: FormatPostingsTermsDictWriter
* -> TermsConsumer
* -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
* -> DocsConsumer
* -> IMPL: FormatPostingsDocsWriter
* -> PositionsConsumer
* -> IMPL: FormatPostingsPositionsWriter
*/
for (int fieldNumber = 0; fieldNumber < numAllFields; fieldNumber++)
{
FieldInfo fieldInfo = allFields[fieldNumber].fieldInfo;
FreqProxTermsWriterPerField fieldWriter = allFields[fieldNumber];
// If this field has postings then add them to the
// segment
fieldWriter.Flush(fieldInfo.Name, consumer, state);
TermsHashPerField perField = fieldWriter.termsHashPerField;
Debug.Assert(termsHash == null || termsHash == perField.termsHash);
termsHash = perField.termsHash;
int numPostings = perField.bytesHash.Count;
perField.Reset();
perField.ShrinkHash(numPostings);
fieldWriter.Reset();
}
if (termsHash != null)
{
termsHash.Reset();
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(consumer);
}
else
{
IOUtils.DisposeWhileHandlingException(consumer);
}
}
}
internal override void FinishDocument(TermsHash termsHash)
{
}
internal abstract void FinishDocument(TermsHash termsHash);
