public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
{
/*
* this is the current indexing chain:
*
* DocConsumer / DocConsumerPerThread
* --> code: DocFieldProcessor
* --> DocFieldConsumer / DocFieldConsumerPerField
* --> code: DocFieldConsumers / DocFieldConsumersPerField
* --> code: DocInverter / DocInverterPerField
* --> InvertedDocConsumer / InvertedDocConsumerPerField
* --> code: TermsHash / TermsHashPerField
* --> TermsHashConsumer / TermsHashConsumerPerField
* --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
* --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
* --> InvertedDocEndConsumer / InvertedDocConsumerPerField
* --> code: NormsConsumer / NormsConsumerPerField
* --> StoredFieldsConsumer
* --> TwoStoredFieldConsumers
* -> code: StoredFieldsProcessor
* -> code: DocValuesProcessor
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
NormsConsumer normsWriter = new NormsConsumer();
DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(new StoredFieldsProcessor(documentsWriterPerThread), new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
return(new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields));
}
public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo)
{
this.TermsHashPerField = termsHashPerField;
this.Parent = parent;
this.fieldInfo = fieldInfo;
DocState = termsHashPerField.DocState;
FieldState = termsHashPerField.FieldState;
IndexOptions = fieldInfo.FieldIndexOptions;
}
public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo)
{
this.termsHashPerField = termsHashPerField;
this.parent = parent;
this.fieldInfo = fieldInfo;
docState = termsHashPerField.docState;
fieldState = termsHashPerField.fieldState;
SetIndexOptions(fieldInfo.IndexOptions);
}
internal void WriteProx(FreqProxTermsWriter.PostingList p, int proxCode)
{
Payload payload;
if (payloadAttribute == null)
{
payload = null;
}
else
{
payload = payloadAttribute.GetPayload();
}
if (payload != null && payload.length > 0)
{
termsHashPerField.WriteVInt(1, (proxCode << 1) | 1);
termsHashPerField.WriteVInt(1, payload.length);
termsHashPerField.WriteBytes(1, payload.data, payload.offset, payload.length);
hasPayloads = true;
}
else
termsHashPerField.WriteVInt(1, proxCode << 1);
p.lastPosition = fieldState.position;
}
internal override DocConsumer GetChain(DocumentsWriter documentsWriter)
{
/*
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor / DocFieldProcessorPerThread
--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerThread / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerThread / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
NormsWriter normsWriter = new NormsWriter();
DocInverter docInverter = new DocInverter(termsHash, normsWriter);
return new DocFieldProcessor(documentsWriter, docInverter);
}
internal DocumentsWriter(Directory directory, IndexWriter writer)
{
this.directory = directory;
this.writer = writer;
this.similarity = writer.GetSimilarity();
flushedDocCount = writer.MaxDoc();
byteBlockAllocator = new ByteBlockAllocator(this);
waitQueue = new WaitQueue(this);
/*
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor / DocFieldProcessorPerThread
--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerThread / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerThread / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
*/
// TODO FI: this should be something the user can pass in
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(this);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(this, true, freqProxWriter,
new TermsHash(this, false, termVectorsWriter, null));
NormsWriter normsWriter = new NormsWriter();
DocInverter docInverter = new DocInverter(termsHash, normsWriter);
StoredFieldsWriter fieldsWriter = new StoredFieldsWriter(this);
DocFieldConsumers docFieldConsumers = new DocFieldConsumers(docInverter, fieldsWriter);
consumer = docFieldProcessor = new DocFieldProcessor(this, docFieldConsumers);
}
static DocumentsWriter()
{
DefaultIndexingChain = new IndexingChain()
{ GetChain = (documentsWriter) =>
{
/*
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor / DocFieldProcessorPerThread
--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerThread / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerThread / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
NormsWriter normsWriter = new NormsWriter();
DocInverter docInverter = new DocInverter(termsHash, normsWriter);
return new DocFieldProcessor(documentsWriter, docInverter);
}
};
POINTER_NUM_BYTE = Constants.JRE_IS_64BIT?8:4;
}
public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
{
/*
this is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor
--> DocFieldConsumer / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerField
--> code: NormsConsumer / NormsConsumerPerField
--> StoredFieldsConsumer
--> TwoStoredFieldConsumers
-> code: StoredFieldsProcessor
-> code: DocValuesProcessor
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
NormsConsumer normsWriter = new NormsConsumer();
DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(new StoredFieldsProcessor(documentsWriterPerThread), new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
}
internal void writeProx(Token t, FreqProxTermsWriter.PostingList p, int proxCode)
{
Payload payload = t.GetPayload();
if (payload != null && payload.length > 0)
{
termsHashPerField.writeVInt(1, (proxCode << 1) | 1);
termsHashPerField.writeVInt(1, payload.length);
termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
hasPayloads = true;
}
else
termsHashPerField.writeVInt(1, proxCode << 1);
p.lastPosition = fieldState.position;
}