예제 #1
0
            public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
            {
                /*
                 * this is the current indexing chain:
                 *
                 * DocConsumer / DocConsumerPerThread
                 * --> code: DocFieldProcessor
                 *  --> DocFieldConsumer / DocFieldConsumerPerField
                 *    --> code: DocFieldConsumers / DocFieldConsumersPerField
                 *      --> code: DocInverter / DocInverterPerField
                 *        --> InvertedDocConsumer / InvertedDocConsumerPerField
                 *          --> code: TermsHash / TermsHashPerField
                 *            --> TermsHashConsumer / TermsHashConsumerPerField
                 *              --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
                 *              --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
                 *        --> InvertedDocEndConsumer / InvertedDocConsumerPerField
                 *          --> code: NormsConsumer / NormsConsumerPerField
                 *  --> StoredFieldsConsumer
                 *    --> TwoStoredFieldConsumers
                 *      -> code: StoredFieldsProcessor
                 *      -> code: DocValuesProcessor
                 */

                // Build up indexing chain:

                TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
                TermsHashConsumer freqProxWriter    = new FreqProxTermsWriter();

                InvertedDocConsumer  termsHash    = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
                NormsConsumer        normsWriter  = new NormsConsumer();
                DocInverter          docInverter  = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
                StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(new StoredFieldsProcessor(documentsWriterPerThread), new DocValuesProcessor(documentsWriterPerThread.bytesUsed));

                return(new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields));
            }
예제 #2
0
 public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo)
 {
     this.TermsHashPerField = termsHashPerField;
     this.Parent            = parent;
     this.fieldInfo         = fieldInfo;
     DocState     = termsHashPerField.DocState;
     FieldState   = termsHashPerField.FieldState;
     IndexOptions = fieldInfo.FieldIndexOptions;
 }
 public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo)
 {
     this.termsHashPerField = termsHashPerField;
     this.parent            = parent;
     this.fieldInfo         = fieldInfo;
     docState   = termsHashPerField.docState;
     fieldState = termsHashPerField.fieldState;
     SetIndexOptions(fieldInfo.IndexOptions);
 }
        internal void WriteProx(FreqProxTermsWriter.PostingList p, int proxCode)
        {
            Payload payload;
            if (payloadAttribute == null)
            {
                payload = null;
            }
            else
            {
                payload = payloadAttribute.GetPayload();
            }

            if (payload != null && payload.length > 0)
            {
                termsHashPerField.WriteVInt(1, (proxCode << 1) | 1);
                termsHashPerField.WriteVInt(1, payload.length);
                termsHashPerField.WriteBytes(1, payload.data, payload.offset, payload.length);
                hasPayloads = true;
            }
            else
                termsHashPerField.WriteVInt(1, proxCode << 1);
            p.lastPosition = fieldState.position;
        }
예제 #5
0
			internal override DocConsumer GetChain(DocumentsWriter documentsWriter)
			{
				/*
				This is the current indexing chain:
				
				DocConsumer / DocConsumerPerThread
				--> code: DocFieldProcessor / DocFieldProcessorPerThread
				--> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
				--> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
				--> code: DocInverter / DocInverterPerThread / DocInverterPerField
				--> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
				--> code: TermsHash / TermsHashPerThread / TermsHashPerField
				--> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
				--> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
				--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
				--> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
				--> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
				--> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
				*/
				
				// Build up indexing chain:
				
				TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
				TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
				
				InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
				NormsWriter normsWriter = new NormsWriter();
				DocInverter docInverter = new DocInverter(termsHash, normsWriter);
				return new DocFieldProcessor(documentsWriter, docInverter);
			}
예제 #6
0
        internal DocumentsWriter(Directory directory, IndexWriter writer)
        {
            this.directory = directory;
            this.writer = writer;
            this.similarity = writer.GetSimilarity();
            flushedDocCount = writer.MaxDoc();

            byteBlockAllocator = new ByteBlockAllocator(this);
            waitQueue = new WaitQueue(this);

            /*
              This is the current indexing chain:

              DocConsumer / DocConsumerPerThread
                --> code: DocFieldProcessor / DocFieldProcessorPerThread
                  --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
                    --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
                      --> code: DocInverter / DocInverterPerThread / DocInverterPerField
                        --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
                          --> code: TermsHash / TermsHashPerThread / TermsHashPerField
                            --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
                              --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
                              --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
                        --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
                          --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
                      --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
            */

            // TODO FI: this should be something the user can pass in
            // Build up indexing chain:
            TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(this);
            TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();

            InvertedDocConsumer termsHash = new TermsHash(this, true, freqProxWriter,
                                                                 new TermsHash(this, false, termVectorsWriter, null));
            NormsWriter normsWriter = new NormsWriter();
            DocInverter docInverter = new DocInverter(termsHash, normsWriter);
            StoredFieldsWriter fieldsWriter = new StoredFieldsWriter(this);
            DocFieldConsumers docFieldConsumers = new DocFieldConsumers(docInverter, fieldsWriter);
            consumer = docFieldProcessor = new DocFieldProcessor(this, docFieldConsumers);
        }
예제 #7
0
		static DocumentsWriter()
		{
			DefaultIndexingChain = new IndexingChain()
            { GetChain = (documentsWriter) =>
			    {
				    /*
				    This is the current indexing chain:
				
				    DocConsumer / DocConsumerPerThread
				    --> code: DocFieldProcessor / DocFieldProcessorPerThread
				    --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
				    --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
				    --> code: DocInverter / DocInverterPerThread / DocInverterPerField
				    --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
				    --> code: TermsHash / TermsHashPerThread / TermsHashPerField
				    --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
				    --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
				    --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
				    --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
				    --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
				    --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
				    */
				
				    // Build up indexing chain:
				
				    TermsHashConsumer termVectorsWriter = new TermVectorsTermsWriter(documentsWriter);
				    TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
				
				    InvertedDocConsumer termsHash = new TermsHash(documentsWriter, true, freqProxWriter, new TermsHash(documentsWriter, false, termVectorsWriter, null));
				    NormsWriter normsWriter = new NormsWriter();
				    DocInverter docInverter = new DocInverter(termsHash, normsWriter);
				    return new DocFieldProcessor(documentsWriter, docInverter);
			    }
            };
			POINTER_NUM_BYTE = Constants.JRE_IS_64BIT?8:4;
		}
            public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
            {
                /*
                this is the current indexing chain:

                DocConsumer / DocConsumerPerThread
                  --> code: DocFieldProcessor
                    --> DocFieldConsumer / DocFieldConsumerPerField
                      --> code: DocFieldConsumers / DocFieldConsumersPerField
                        --> code: DocInverter / DocInverterPerField
                          --> InvertedDocConsumer / InvertedDocConsumerPerField
                            --> code: TermsHash / TermsHashPerField
                              --> TermsHashConsumer / TermsHashConsumerPerField
                                --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
                                --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
                          --> InvertedDocEndConsumer / InvertedDocConsumerPerField
                            --> code: NormsConsumer / NormsConsumerPerField
                    --> StoredFieldsConsumer
                      --> TwoStoredFieldConsumers
                        -> code: StoredFieldsProcessor
                        -> code: DocValuesProcessor
                  */

                // Build up indexing chain:

                TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
                TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();

                InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
                NormsConsumer normsWriter = new NormsConsumer();
                DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
                StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(new StoredFieldsProcessor(documentsWriterPerThread), new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
                return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
            }
 internal void writeProx(Token t, FreqProxTermsWriter.PostingList p, int proxCode)
 {
     Payload payload = t.GetPayload();
     if (payload != null && payload.length > 0)
     {
         termsHashPerField.writeVInt(1, (proxCode << 1) | 1);
         termsHashPerField.writeVInt(1, payload.length);
         termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
         hasPayloads = true;
     }
     else
         termsHashPerField.writeVInt(1, proxCode << 1);
     p.lastPosition = fieldState.position;
 }