Example #1
0
        public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter,
            SegmentWriteState state, PostingsWriterBase postingsWriter)
        {
            var termsFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
                TERMS_EXTENSION);
            _termsIndexWriter = termsIndexWriter;
            _output = state.Directory.CreateOutput(termsFileName, state.Context);
            var success = false;

            try
            {
                FieldInfos = state.FieldInfos;
                WriteHeader(_output);
                CurrentField = null;
                PostingsWriter = postingsWriter;

                postingsWriter.Init(_output); // have consumer write its format/header
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(_output);
                }
            }
        }
        public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
        {
            PostingsWriterBase docsWriter = null;

            // Terms that have <= freqCutoff number of docs are
            // "pulsed" (inlined):
            PostingsWriterBase pulsingWriter = null;

            // Terms dict
            bool success = false;
            try
            {
                docsWriter = _wrappedPostingsBaseFormat.PostingsWriterBase(state);

                // Terms that have <= freqCutoff number of docs are
                // "pulsed" (inlined):
                pulsingWriter = new PulsingPostingsWriter(state, _freqCutoff, docsWriter);
                FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, _minBlockSize, _maxBlockSize);
                success = true;
                return ret;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(docsWriter, pulsingWriter);
                }
            }
        }
 public FixedGapTermsIndexWriter(SegmentWriteState state)
 {
     String indexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
         TERMS_INDEX_EXTENSION);
     _termIndexInterval = state.TermIndexInterval;
     Output = state.Directory.CreateOutput(indexFileName, state.Context);
     bool success = false;
     try
     {
         _fieldInfos = state.FieldInfos;
         WriteHeader(Output);
         Output.WriteInt(_termIndexInterval);
         success = true;
     }
     finally
     {
         if (!success)
         {
             IOUtils.CloseWhileHandlingException(Output);
         }
     }
 }
 internal Lucene42NormsConsumer(SegmentWriteState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension, float acceptableOverheadRatio)
 {
     this.AcceptableOverheadRatio = acceptableOverheadRatio;
     MaxDoc = state.SegmentInfo.DocCount;
     bool success = false;
     try
     {
         string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
         Data = state.Directory.CreateOutput(dataName, state.Context);
         CodecUtil.WriteHeader(Data, dataCodec, Lucene42DocValuesProducer.VERSION_CURRENT);
         string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
         Meta = state.Directory.CreateOutput(metaName, state.Context);
         CodecUtil.WriteHeader(Meta, metaCodec, Lucene42DocValuesProducer.VERSION_CURRENT);
         success = true;
     }
     finally
     {
         if (!success)
         {
             IOUtils.CloseWhileHandlingException(this);
         }
     }
 }
Example #5
0
 /// <summary>
 /// Returns a <see cref="DocValuesConsumer"/> to write docvalues to the
 /// index.
 /// </summary>
 public abstract DocValuesConsumer FieldsConsumer(SegmentWriteState state);
Example #6
0
        public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory, int skipInterval)
        {
            FREQ_OUT = null;
            FREQ_INDEX = null;
            POS_OUT = null;
            POS_INDEX = null;
            PAYLOAD_OUT = null;
            var success = false;
            try
            {
                SKIP_INTERVAL = skipInterval;
                SKIP_MINIMUM = skipInterval; // set to the same for now
                var docFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, DOC_EXTENSION);

                DOC_OUT = factory.CreateOutput(state.Directory, docFileName, state.Context);
                DOC_INDEX = DOC_OUT.Index();

                if (state.FieldInfos.HasFreq())
                {
                    var frqFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FREQ_EXTENSION);
                    FREQ_OUT = factory.CreateOutput(state.Directory, frqFileName, state.Context);
                    FREQ_INDEX = FREQ_OUT.Index();
                }

                if (state.FieldInfos.HasProx())
                {
                    var posFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, POS_EXTENSION);
                    POS_OUT = factory.CreateOutput(state.Directory, posFileName, state.Context);
                    POS_INDEX = POS_OUT.Index();

                    // TODO: -- only if at least one field stores payloads?
                    var payloadFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,PAYLOAD_EXTENSION);
                    PAYLOAD_OUT = state.Directory.CreateOutput(payloadFileName, state.Context);
                }

                var skipFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, SKIP_EXTENSION);
                SKIP_OUT = state.Directory.CreateOutput(skipFileName, state.Context);

                TOTAL_NUM_DOCS = state.SegmentInfo.DocCount;

                SKIP_LIST_WRITER = new SepSkipListWriter(skipInterval, MAX_SKIP_LEVELS, TOTAL_NUM_DOCS, FREQ_OUT, DOC_OUT,
                    POS_OUT, PAYLOAD_OUT);

                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(DOC_OUT, SKIP_OUT, FREQ_OUT, POS_OUT, PAYLOAD_OUT);
                }
            }
        }
Example #7
0
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return(@delegate.FieldsConsumer(state));
 }
        /// <summary>
        /// Creates a postings writer with the specified PackedInts overhead ratio </summary>
        // TODO: does this ctor even make sense?
        public Lucene41PostingsWriter(SegmentWriteState state, float acceptableOverheadRatio)
            : base()
        {
            DocOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), state.Context);
            IndexOutput posOut = null;
            IndexOutput payOut = null;
            bool success = false;
            try
            {
                CodecUtil.WriteHeader(DocOut, DOC_CODEC, VERSION_CURRENT);
                ForUtil = new ForUtil(acceptableOverheadRatio, DocOut);
                if (state.FieldInfos.HasProx())
                {
                    PosDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
                    posOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), state.Context);
                    CodecUtil.WriteHeader(posOut, POS_CODEC, VERSION_CURRENT);

                    if (state.FieldInfos.HasPayloads())
                    {
                        PayloadBytes = new sbyte[128];
                        PayloadLengthBuffer = new int[ForUtil.MAX_DATA_SIZE];
                    }
                    else
                    {
                        PayloadBytes = null;
                        PayloadLengthBuffer = null;
                    }

                    if (state.FieldInfos.HasOffsets())
                    {
                        OffsetStartDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
                        OffsetLengthBuffer = new int[ForUtil.MAX_DATA_SIZE];
                    }
                    else
                    {
                        OffsetStartDeltaBuffer = null;
                        OffsetLengthBuffer = null;
                    }

                    if (state.FieldInfos.HasPayloads() || state.FieldInfos.HasOffsets())
                    {
                        payOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), state.Context);
                        CodecUtil.WriteHeader(payOut, PAY_CODEC, VERSION_CURRENT);
                    }
                }
                else
                {
                    PosDeltaBuffer = null;
                    PayloadLengthBuffer = null;
                    OffsetStartDeltaBuffer = null;
                    OffsetLengthBuffer = null;
                    PayloadBytes = null;
                }
                this.PayOut = payOut;
                this.PosOut = posOut;
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(DocOut, posOut, payOut);
                }
            }

            DocDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
            FreqBuffer = new int[ForUtil.MAX_DATA_SIZE];

            // TODO: should we try skipping every 2/4 blocks...?
            SkipWriter = new Lucene41SkipWriter(MaxSkipLevels, Lucene41PostingsFormat.BLOCK_SIZE, state.SegmentInfo.DocCount, DocOut, posOut, payOut);

            Encoded = new sbyte[ForUtil.MAX_ENCODED_SIZE];
        }
        public FSTOrdTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter)
        {
            var termsIndexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
                TERMS_INDEX_EXTENSION);
            var termsBlockFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
                TERMS_BLOCK_EXTENSION);

            this.postingsWriter = postingsWriter;
            fieldInfos = state.FieldInfos;

            var success = false;
            try
            {
                indexOut = state.Directory.CreateOutput(termsIndexFileName, state.Context);
                blockOut = state.Directory.CreateOutput(termsBlockFileName, state.Context);
                WriteHeader(indexOut);
                WriteHeader(blockOut);
                this.postingsWriter.Init(blockOut);
                success = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(indexOut, blockOut);
                }
            }
        }
        /// <summary>
        /// If the total number of positions (summed across all docs
        /// for this term) is less than or equal maxPositions, then the postings are
        /// inlined into terms dict
        /// </summary>
        public PulsingPostingsWriter(SegmentWriteState state, int maxPositions, PostingsWriterBase wrappedPostingsWriter)
        {
            _pending = new Position[maxPositions];
            for (var i = 0; i < maxPositions; i++)
            {
                _pending[i] = new Position();
            }
            _fields = new List<FieldMetaData>();

            // We simply wrap another postings writer, but only call
            // on it when tot positions is >= the cutoff:
            _wrappedPostingsWriter = wrappedPostingsWriter;
            _segmentState = state;
        }
 public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
     SegmentWriteState state, BloomFilteringPostingsFormat bfpf)
 {
     _delegateFieldsConsumer = fieldsConsumer;
     _state = state;
     _bfpf = bfpf;
 }
Example #12
0
 public Lucene45DocValuesConsumerAnonymousHelper(DiskDocValuesFormat outerInstance, SegmentWriteState state)
     : base(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION)
 {
     this.outerInstance = outerInstance;
 }
 public override FuzzySet GetSetForField(SegmentWriteState state, FieldInfo info)
 {
     //Assume all of the docs have a unique term (e.g. a primary key) and we hope to maintain a set with 10% of bits set
     return FuzzySet.CreateSetBasedOnQuality(state.SegmentInfo.DocCount, 0.10f);
 }
Example #14
0
 public SepPostingsWriter(SegmentWriteState state, IntStreamFactory factory)
     : this(state, factory, DEFAULT_SKIP_INTERVAL)
 {
 }
Example #15
0
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     return(new Lucene45DocValuesConsumerAnonymousHelper(this, state));
 }
Example #16
0
 internal FormatPostingsTermsWriter(SegmentWriteState state, FormatPostingsFieldsWriter parent) : base()
 {
     this.parent = parent;
     termsOut    = parent.termsOut;
     docsWriter  = new FormatPostingsDocsWriter(state, this);
 }
Example #17
0
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new System.NotSupportedException("this codec cannot write docvalues");
 }
 public SimpleTextFieldsWriter(SegmentWriteState state)
 {
     var fileName = SimpleTextPostingsFormat.GetPostingsFileName(state.SegmentInfo.Name, state.SegmentSuffix);
     _output = state.Directory.CreateOutput(fileName, state.Context);
 }
        // TODO: it'd be nice to let the FST builder prune based
        // on term count of each node (the prune1/prune2 that it
        // accepts), and build the index based on that.  This
        // should result in a more compact terms index, more like
        // a prefix trie than the other selectors, because it
        // only stores enough leading bytes to get down to N
        // terms that may complete that prefix.  It becomes
        // "deeper" when terms are dense, and "shallow" when they
        // are less dense.
        //
        // However, it's not easy to make that work this this
        // API, because that pruning doesn't immediately know on
        // seeing each term whether that term will be a seek point
        // or not.  It requires some non-causality in the API, ie
        // only on seeing some number of future terms will the
        // builder decide which past terms are seek points.
        // Somehow the API'd need to be able to return a "I don't
        // know" value, eg like a Future, which only later on is
        // flipped (frozen) to true or false.
        //
        // We could solve this with a 2-pass approach, where the
        // first pass would build an FSA (no outputs) solely to
        // determine which prefixes are the 'leaves' in the
        // pruning. The 2nd pass would then look at this prefix
        // trie to mark the seek points and build the FST mapping
        // to the true output.
        //
        // But, one downside to this approach is that it'd result
        // in uneven index term selection.  EG with prune1=10, the
        // resulting index terms could be as frequent as every 10
        // terms or as rare as every <maxArcCount> * 10 (eg 2560),
        // in the extremes.
        public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy)
        {
            string indexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
                TERMS_INDEX_EXTENSION);
            Output = state.Directory.CreateOutput(indexFileName, state.Context);
            bool success = false;

            try
            {
                _policy = policy;
                WriteHeader(Output);
                success = true;
            }
            finally
            {
                if (!success)
                    IOUtils.CloseWhileHandlingException(Output);
            }
        }
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return new AssertingFieldsConsumer(@in.FieldsConsumer(state));
 }
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return(new PreFlexRWFieldsWriter(state));
 }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public codecs.FieldsConsumer fieldsConsumer(index.SegmentWriteState state) throws java.io.IOException
	  public override FieldsConsumer fieldsConsumer(SegmentWriteState state)
	  {

//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String fileName = index.IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
		string fileName = IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final store.IndexOutput out = state.directory.createOutput(fileName, state.context);
		IndexOutput @out = state.directory.createOutput(fileName, state.context);
		bool success = false;
		try
		{
		  CodecUtil.WriteHeader(@out, CODEC_NAME, VERSION_CURRENT);
		  success = true;
		}
		finally
		{
		  if (!success)
		  {
			IOUtils.CloseWhileHandlingException(@out);
		  }
		}

		return new FieldsConsumerAnonymousInnerClassHelper(this, @out);
	  }
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     return new Lucene45DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
 }
        public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
        {
            PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);

            bool success = false;
            try
            {
                FieldsConsumer ret = new BlockTreeTermsWriter(state, postingsWriter, MinTermBlockSize, MaxTermBlockSize);
                success = true;
                return ret;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.CloseWhileHandlingException(postingsWriter);
                }
            }
        }
 public Lucene45DocValuesConsumerAnonymousHelper(DiskDocValuesFormat outerInstance, SegmentWriteState state)
     : base(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION)
 {
     this.outerInstance = outerInstance;
 }
 public override FuzzySet GetSetForField(SegmentWriteState state, FieldInfo info)
 {
     //Assume all of the docs have a unique term (e.g. a primary key) and we hope to maintain a set with 10% of bits set
     return(FuzzySet.CreateSetBasedOnQuality(state.SegmentInfo.DocCount, 0.10f));
 }
 public FieldsWriter(PerFieldDocValuesFormat outerInstance, SegmentWriteState state)
 {
     this.OuterInstance = outerInstance;
     SegmentWriteState = state;
 }
Example #28
0
 /// <summary>
 /// 
 /// </summary>
 /// <param name="state">The content to be indexed</param>
 /// <param name="info">The field requiring a BloomFilter</param>
 /// <returns>An appropriately sized set or null if no BloomFiltering required</returns>
 public abstract FuzzySet GetSetForField(SegmentWriteState state, FieldInfo info);
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new System.NotSupportedException("this codec can only be used for reading");
 }
Example #30
0
 // note: intentionally ignores seg suffix
 internal Lucene40DocValuesWriter(SegmentWriteState state, string filename, string legacyKey)
 {
     this.state     = state;
     this.legacyKey = legacyKey;
     this.dir       = new CompoundFileDirectory(state.Directory, filename, state.Context, true);
 }
Example #31
0
 public override FuzzySet GetSetForField(SegmentWriteState state, FieldInfo info)
 {
     return(FuzzySet.CreateSetBasedOnMaxMemory(1024));
 }
Example #32
0
 public SepPostingsWriter(SegmentWriteState state, Int32StreamFactory factory)
     : this(state, factory, DEFAULT_SKIP_INTERVAL)
 {
 }
        private readonly ISet <string> _fieldsSeen = new JCG.HashSet <string>(); // for asserting

        // LUCENENET NOTE: Changed from public to internal because the class had to be made public, but is not for public use.
        internal SimpleTextDocValuesWriter(SegmentWriteState state, string ext)
        {
            data = state.Directory.CreateOutput(
                IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, ext), state.Context);
            numDocs = state.SegmentInfo.DocCount;
        }
Example #34
0
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new NotImplementedException("This codec can only be used for reading");
 }
Example #35
0
 public override DocValuesConsumer NormsConsumer(SegmentWriteState state)
 {
     throw new System.NotSupportedException("this codec can only be used for reading");
 }
Example #36
0
 public abstract void  CloseDocStore(SegmentWriteState state);
Example #37
0
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new System.NotSupportedException("this codec cannot write docvalues");
 }
Example #38
0
 public override sealed FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return(new FieldsWriter(this, state));
 }
 public override PostingsWriterBase PostingsWriterBase(SegmentWriteState state)
 {
     throw new NotSupportedException("this codec can only be used for reading");
 }
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new NotImplementedException("This codec can only be used for reading");
 }
Example #41
0
 public abstract void  Flush(System.Collections.Generic.ICollection <DocConsumerPerThread> threads, SegmentWriteState state);
Example #42
0
 public override PostingsWriterBase PostingsWriterBase(SegmentWriteState state)
 {
     throw UnsupportedOperationException.Create("this codec can only be used for reading");
 }
Example #43
0
 public FieldsWriter(PerFieldPostingsFormat outerInstance, SegmentWriteState state)
 {
     this.outerInstance = outerInstance;
     segmentWriteState  = state;
 }
Example #44
0
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return(new AssertingFieldsConsumer(@in.FieldsConsumer(state)));
 }
 public abstract void  Flush(System.Collections.IDictionary threadsAndFields, SegmentWriteState state);
Example #46
0
        public SimpleTextFieldsWriter(SegmentWriteState state)
        {
            var fileName = SimpleTextPostingsFormat.GetPostingsFileName(state.SegmentInfo.Name, state.SegmentSuffix);

            _output = state.Directory.CreateOutput(fileName, state.Context);
        }
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     if (_delegatePostingsFormat == null)
     {
         throw new InvalidOperationException("Error - constructed without a choice of PostingsFormat");
     }
     return new BloomFilteredFieldsConsumer(
         _delegatePostingsFormat.FieldsConsumer(state), state, this);
 }
Example #48
0
        // private String segment;

        /// <summary>
        /// Creates a <seealso cref="Lucene40PostingsWriter"/>, with the
        ///  <seealso cref="#DEFAULT_SKIP_INTERVAL"/>.
        /// </summary>
        public Lucene40PostingsWriter(SegmentWriteState state)
            : this(state, DEFAULT_SKIP_INTERVAL)
        {
        }
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     return new Lucene45DocValuesConsumerAnonymousHelper(this, state);
 }
Example #50
0
        /// <summary>
        /// Creates a postings writer with the specified PackedInts overhead ratio </summary>
        // TODO: does this ctor even make sense?
        public Lucene41PostingsWriter(SegmentWriteState state, float acceptableOverheadRatio)
            : base()
        {
            docOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION), state.Context);
            IndexOutput posOut  = null;
            IndexOutput payOut  = null;
            bool        success = false;

            try
            {
                CodecUtil.WriteHeader(docOut, DOC_CODEC, VERSION_CURRENT);
                forUtil = new ForUtil(acceptableOverheadRatio, docOut);
                if (state.FieldInfos.HasProx)
                {
                    posDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
                    posOut         = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.POS_EXTENSION), state.Context);
                    CodecUtil.WriteHeader(posOut, POS_CODEC, VERSION_CURRENT);

                    if (state.FieldInfos.HasPayloads)
                    {
                        payloadBytes        = new byte[128];
                        payloadLengthBuffer = new int[ForUtil.MAX_DATA_SIZE];
                    }
                    else
                    {
                        payloadBytes        = null;
                        payloadLengthBuffer = null;
                    }

                    if (state.FieldInfos.HasOffsets)
                    {
                        offsetStartDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
                        offsetLengthBuffer     = new int[ForUtil.MAX_DATA_SIZE];
                    }
                    else
                    {
                        offsetStartDeltaBuffer = null;
                        offsetLengthBuffer     = null;
                    }

                    if (state.FieldInfos.HasPayloads || state.FieldInfos.HasOffsets)
                    {
                        payOut = state.Directory.CreateOutput(IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION), state.Context);
                        CodecUtil.WriteHeader(payOut, PAY_CODEC, VERSION_CURRENT);
                    }
                }
                else
                {
                    posDeltaBuffer         = null;
                    payloadLengthBuffer    = null;
                    offsetStartDeltaBuffer = null;
                    offsetLengthBuffer     = null;
                    payloadBytes           = null;
                }
                this.payOut = payOut;
                this.posOut = posOut;
                success     = true;
            }
            finally
            {
                if (!success)
                {
                    IOUtils.DisposeWhileHandlingException(docOut, posOut, payOut);
                }
            }

            docDeltaBuffer = new int[ForUtil.MAX_DATA_SIZE];
            freqBuffer     = new int[ForUtil.MAX_DATA_SIZE];

            // TODO: should we try skipping every 2/4 blocks...?
            skipWriter = new Lucene41SkipWriter(maxSkipLevels, Lucene41PostingsFormat.BLOCK_SIZE, state.SegmentInfo.DocCount, docOut, posOut, payOut);

            encoded = new byte[ForUtil.MAX_ENCODED_SIZE];
        }
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     return ForName("Lucene41").FieldsConsumer(state);
 }
Example #52
0
 /// <summary>
 /// Creates a postings writer with <code>PackedInts.COMPACT</code> </summary>
 public Lucene41PostingsWriter(SegmentWriteState state)
     : this(state, PackedInt32s.COMPACT)
 {
 }
 /// <summary>
 /// Creates a postings writer with <code>PackedInts.COMPACT</code> </summary>
 public Lucene41PostingsWriter(SegmentWriteState state)
     : this(state, PackedInts.COMPACT)
 {
 }
Example #54
0
 internal abstract void  CloseDocStore(SegmentWriteState state);
 public override sealed DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     return new FieldsWriter(this, state);
 }
Example #56
0
 public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw UnsupportedOperationException.Create("this codec can only be used for reading");
 }
 public abstract void Flush(SegmentWriteState state);
 public override DocValuesConsumer FieldsConsumer(SegmentWriteState state)
 {
     throw new NotImplementedException();
 }
 /// <summary>
 /// expert: Creates a new writer </summary>
 public Lucene45DocValuesConsumer(SegmentWriteState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
 {
     bool success = false;
     try
     {
         string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
         Data = state.Directory.CreateOutput(dataName, state.Context);
         CodecUtil.WriteHeader(Data, dataCodec, Lucene45DocValuesFormat.VERSION_CURRENT);
         string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
         Meta = state.Directory.CreateOutput(metaName, state.Context);
         CodecUtil.WriteHeader(Meta, metaCodec, Lucene45DocValuesFormat.VERSION_CURRENT);
         MaxDoc = state.SegmentInfo.DocCount;
         success = true;
     }
     finally
     {
         if (!success)
         {
             IOUtils.CloseWhileHandlingException(this);
         }
     }
 }
Example #60
0
        public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
        {
            int minSkipInterval;

            if (state.SegmentInfo.DocCount > 1000000)
            {
                // Test2BPostings can OOME otherwise:
                minSkipInterval = 3;
            }
            else
            {
                minSkipInterval = 2;
            }

            // we pull this before the seed intentionally: because its not consumed at runtime
            // (the skipInterval is written into postings header)
            int skipInterval = TestUtil.NextInt32(seedRandom, minSkipInterval, 10);

            if (LuceneTestCase.Verbose)
            {
                Console.WriteLine("MockRandomCodec: skipInterval=" + skipInterval);
            }

            long seed = seedRandom.nextLong();

            if (LuceneTestCase.Verbose)
            {
                Console.WriteLine("MockRandomCodec: writing to seg=" + state.SegmentInfo.Name + " formatID=" + state.SegmentSuffix + " seed=" + seed);
            }

            string      seedFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, SEED_EXT);
            IndexOutput @out         = state.Directory.CreateOutput(seedFileName, state.Context);

            try
            {
                @out.WriteInt64(seed);
            }
            finally
            {
                @out.Dispose();
            }

            Random random = new Random((int)seed);

            random.nextInt(); // consume a random for buffersize

            PostingsWriterBase postingsWriter;

            if (random.nextBoolean())
            {
                postingsWriter = new SepPostingsWriter(state, new MockInt32StreamFactory(random), skipInterval);
            }
            else
            {
                if (LuceneTestCase.Verbose)
                {
                    Console.WriteLine("MockRandomCodec: writing Standard postings");
                }
                // TODO: randomize variables like acceptibleOverHead?!
                postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
            }

            if (random.nextBoolean())
            {
                int totTFCutoff = TestUtil.NextInt32(random, 1, 20);
                if (LuceneTestCase.Verbose)
                {
                    Console.WriteLine("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff);
                }
                postingsWriter = new PulsingPostingsWriter(state, totTFCutoff, postingsWriter);
            }

            FieldsConsumer fields;
            int            t1 = random.nextInt(4);

            if (t1 == 0)
            {
                bool success = false;
                try
                {
                    fields  = new FSTTermsWriter(state, postingsWriter);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        postingsWriter.Dispose();
                    }
                }
            }
            else if (t1 == 1)
            {
                bool success = false;
                try
                {
                    fields  = new FSTOrdTermsWriter(state, postingsWriter);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        postingsWriter.Dispose();
                    }
                }
            }
            else if (t1 == 2)
            {
                // Use BlockTree terms dict

                if (LuceneTestCase.Verbose)
                {
                    Console.WriteLine("MockRandomCodec: writing BlockTree terms dict");
                }

                // TODO: would be nice to allow 1 but this is very
                // slow to write
                int minTermsInBlock = TestUtil.NextInt32(random, 2, 100);
                int maxTermsInBlock = Math.Max(2, (minTermsInBlock - 1) * 2 + random.nextInt(100));

                bool success = false;
                try
                {
                    fields  = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        postingsWriter.Dispose();
                    }
                }
            }
            else
            {
                if (LuceneTestCase.Verbose)
                {
                    Console.WriteLine("MockRandomCodec: writing Block terms dict");
                }

                bool success = false;

                TermsIndexWriterBase indexWriter;
                try
                {
                    if (random.nextBoolean())
                    {
                        state.TermIndexInterval = TestUtil.NextInt32(random, 1, 100);
                        if (LuceneTestCase.Verbose)
                        {
                            Console.WriteLine("MockRandomCodec: fixed-gap terms index (tii=" + state.TermIndexInterval + ")");
                        }
                        indexWriter = new FixedGapTermsIndexWriter(state);
                    }
                    else
                    {
                        VariableGapTermsIndexWriter.IndexTermSelector selector;
                        int n2 = random.nextInt(3);
                        if (n2 == 0)
                        {
                            int tii = TestUtil.NextInt32(random, 1, 100);
                            selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii);
                            if (LuceneTestCase.Verbose)
                            {
                                Console.WriteLine("MockRandomCodec: variable-gap terms index (tii=" + tii + ")");
                            }
                        }
                        else if (n2 == 1)
                        {
                            int docFreqThresh = TestUtil.NextInt32(random, 2, 100);
                            int tii           = TestUtil.NextInt32(random, 1, 100);
                            selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii);
                        }
                        else
                        {
                            int seed2 = random.Next();
                            int gap   = TestUtil.NextInt32(random, 2, 40);
                            if (LuceneTestCase.Verbose)
                            {
                                Console.WriteLine("MockRandomCodec: random-gap terms index (max gap=" + gap + ")");
                            }
                            selector = new IndexTermSelectorAnonymousHelper(seed2, gap);

                            //           selector = new VariableGapTermsIndexWriter.IndexTermSelector() {
                            //                Random rand = new Random(seed2);

                            //@Override
                            //                public bool isIndexTerm(BytesRef term, TermStats stats)
                            //{
                            //    return rand.nextInt(gap) == gap / 2;
                            //}

                            //@Override
                            //                  public void newField(FieldInfo fieldInfo)
                            //{
                            //}
                            //              };
                        }
                        indexWriter = new VariableGapTermsIndexWriter(state, selector);
                    }
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        postingsWriter.Dispose();
                    }
                }

                success = false;
                try
                {
                    fields  = new BlockTermsWriter(indexWriter, state, postingsWriter);
                    success = true;
                }
                finally
                {
                    if (!success)
                    {
                        try
                        {
                            postingsWriter.Dispose();
                        }
                        finally
                        {
                            indexWriter.Dispose();
                        }
                    }
                }
            }

            return(fields);
        }