Beispiel #1
0
            internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
            {
                _liveDocs      = liveDocs;
                _storePayloads = fieldInfo.HasPayloads();

                // TODO: can't we only do this if consumer skipped consuming the previous docs?
                _docIndex.CopyFrom(termState.DOC_INDEX);
                _docIndex.Seek(_docReader);

                _freqIndex.CopyFrom(termState.FREQ_INDEX);
                _freqIndex.Seek(_freqReader);

                _posIndex.CopyFrom(termState.POS_INDEX);
                _posSeekPending = true;
                _payloadPending = false;

                _payloadFp = termState.PAYLOAD_FP;
                _skipFp    = termState.SKIP_FP;

                _docFreq             = termState.DocFreq;
                _count               = 0;
                _doc                 = -1;
                _accum               = 0;
                _pendingPosCount     = 0;
                _pendingPayloadBytes = 0;
                _skipped             = false;

                return(this);
            }
Beispiel #2
0
            internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, IBits liveDocs)
            {
                this.liveDocs = liveDocs;
                storePayloads = fieldInfo.HasPayloads;
                //System.out.println("Sep D&P init");

                // TODO: can't we only do this if consumer
                // skipped consuming the previous docs?
                docIndex.CopyFrom(termState.docIndex);
                docIndex.Seek(docReader);
                //System.out.println("  docIndex=" + docIndex);

                freqIndex.CopyFrom(termState.freqIndex);
                freqIndex.Seek(freqReader);
                //System.out.println("  freqIndex=" + freqIndex);

                posIndex.CopyFrom(termState.posIndex);
                //System.out.println("  posIndex=" + posIndex);
                posSeekPending = true;
                payloadPending = false;

                payloadFP = termState.payloadFP;
                skipFP    = termState.skipFP;
                //System.out.println("  skipFP=" + skipFP);

                docFreq             = termState.DocFreq;
                count               = 0;
                doc                 = -1;
                accum               = 0;
                pendingPosCount     = 0;
                pendingPayloadBytes = 0;
                skipped             = false;

                return(this);
            }
Beispiel #3
0
            public override object Clone()
            {
                var other = new SepTermState();

                other.CopyFrom(this);
                return(other);
            }
Beispiel #4
0
        public override DocsAndPositionsEnum DocsAndPositions(FieldInfo fieldInfo, BlockTermState termState,
                                                              IBits liveDocs, DocsAndPositionsEnum reuse, DocsAndPositionsFlags flags)
        {
            Debug.Assert(fieldInfo.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
            SepTermState            termState_ = (SepTermState)termState;
            SepDocsAndPositionsEnum postingsEnum;

            if (reuse == null || !(reuse is SepDocsAndPositionsEnum))
            {
                postingsEnum = new SepDocsAndPositionsEnum(this);
            }
            else
            {
                postingsEnum = (SepDocsAndPositionsEnum)reuse;
                if (postingsEnum.startDocIn != docIn)
                {
                    // If you are using ParellelReader, and pass in a
                    // reused DocsAndPositionsEnum, it could have come
                    // from another reader also using sep codec
                    postingsEnum = new SepDocsAndPositionsEnum(this);
                }
            }

            return(postingsEnum.Init(fieldInfo, termState_, liveDocs));
        }
Beispiel #5
0
            internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, IBits liveDocs)
            {
                this.liveDocs     = liveDocs;
                this.indexOptions = fieldInfo.IndexOptions;
                omitTF            = indexOptions == IndexOptions.DOCS_ONLY;
                storePayloads     = fieldInfo.HasPayloads;

                // TODO: can't we only do this if consumer
                // skipped consuming the previous docs?
                docIndex.CopyFrom(termState.docIndex);
                docIndex.Seek(docReader);

                if (!omitTF)
                {
                    freqIndex.CopyFrom(termState.freqIndex);
                    freqIndex.Seek(freqReader);
                }

                docFreq = termState.DocFreq;
                // NOTE: unused if docFreq < skipMinimum:
                skipFP  = termState.skipFP;
                count   = 0;
                doc     = -1;
                accum   = 0;
                freq    = 1;
                skipped = false;

                return(this);
            }
Beispiel #6
0
            internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
            {
                _liveDocs = liveDocs;
                if (fieldInfo.FieldIndexOptions.HasValue)
                {
                    _indexOptions = fieldInfo.FieldIndexOptions.Value;
                }

                _omitTf        = _indexOptions == FieldInfo.IndexOptions.DOCS_ONLY;
                _storePayloads = fieldInfo.HasPayloads();

                // TODO: can't we only do this if consumer
                // skipped consuming the previous docs?
                _docIndex.CopyFrom(termState.DOC_INDEX);
                _docIndex.Seek(_docReader);

                if (!_omitTf)
                {
                    _freqIndex.CopyFrom(termState.FREQ_INDEX);
                    _freqIndex.Seek(_freqReader);
                }

                _docFreq = termState.DocFreq;
                // NOTE: unused if docFreq < skipMinimum:
                _skipFp  = termState.SKIP_FP;
                _count   = 0;
                _doc     = -1;
                _accum   = 0;
                _freq    = 1;
                _skipped = false;

                return(this);
            }
Beispiel #7
0
        public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState bstate, bool absolute)
        {
            var state = (SepTermState)bstate;

            if (absolute)
            {
                LAST_SKIP_FP    = 0;
                LAST_PAYLOAD_FP = 0;
                _lastState      = state;
            }
            _lastState.DocIndex.CopyFrom(state.DocIndex, false);
            _lastState.DocIndex.Write(output, absolute);
            if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
            {
                _lastState.FreqIndex.CopyFrom(state.FreqIndex, false);
                _lastState.FreqIndex.Write(output, absolute);
                if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    _lastState.PosIndex.CopyFrom(state.PosIndex, false);
                    _lastState.PosIndex.Write(output, absolute);
                    if (STORE_PAYLOADS)
                    {
                        if (absolute)
                        {
                            output.WriteVLong(state.PayloadFp);
                        }
                        else
                        {
                            output.WriteVLong(state.PayloadFp - LAST_PAYLOAD_FP);
                        }
                        LAST_PAYLOAD_FP = state.PayloadFp;
                    }
                }
            }
            if (state.SkipFp == -1)
            {
                return;
            }

            if (absolute)
            {
                output.WriteVLong(state.SkipFp);
            }
            else
            {
                output.WriteVLong(state.SkipFp - LAST_SKIP_FP);
            }
            LAST_SKIP_FP = state.SkipFp;
        }
        /// <summary>Called when we are done adding docs to this term. </summary>
        public override void FinishTerm(BlockTermState state)
        {
            SepTermState state_ = (SepTermState)state;

            // TODO: -- wasteful we are counting this in two places?
            if (Debugging.AssertsEnabled)
            {
                Debugging.Assert(state_.DocFreq > 0);
                Debugging.Assert(state_.DocFreq == df);
            }

            state_.DocIndex = docOut.GetIndex();
            state_.DocIndex.CopyFrom(docIndex, false);
            if (indexOptions != IndexOptions.DOCS_ONLY)
            {
                state_.FreqIndex = freqOut.GetIndex();
                state_.FreqIndex.CopyFrom(freqIndex, false);
                if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    state_.PosIndex = posOut.GetIndex();
                    state_.PosIndex.CopyFrom(posIndex, false);
                }
                else
                {
                    state_.PosIndex = null;
                }
            }
            else
            {
                state_.FreqIndex = null;
                state_.PosIndex  = null;
            }

            if (df >= skipMinimum)
            {
                state_.SkipFP = skipOut.GetFilePointer();
                //System.out.println("  skipFP=" + skipFP);
                skipListWriter.WriteSkip(skipOut);
                //System.out.println("    numBytes=" + (skipOut.getFilePointer()-skipFP));
            }
            else
            {
                state_.SkipFP = -1;
            }
            state_.PayloadFP = payloadStart;

            lastDocID = 0;
            df        = 0;
        }
Beispiel #9
0
 // Currently, this instance is re-used across fields, so
 // our parent calls setField whenever the field changes
 public override int SetField(FieldInfo fieldInfo)
 {
     this.fieldInfo    = fieldInfo;
     this.indexOptions = fieldInfo.IndexOptions;
     if (indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0)
     {
         throw new System.NotSupportedException("this codec cannot index offsets");
     }
     skipListWriter.SetIndexOptions(indexOptions);
     storePayloads = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && fieldInfo.HasPayloads;
     lastPayloadFP = 0;
     lastSkipFP    = 0;
     lastState     = SetEmptyState();
     return(0);
 }
Beispiel #10
0
        public override BlockTermState NewTermState()
        {
            SepTermState state = new SepTermState();

            state.docIndex = docIn.GetIndex();
            if (freqIn != null)
            {
                state.freqIndex = freqIn.GetIndex();
            }
            if (posIn != null)
            {
                state.posIndex = posIn.GetIndex();
            }
            return(state);
        }
Beispiel #11
0
        public override DocsEnum Docs(FieldInfo fieldInfo, BlockTermState termState, IBits liveDocs, DocsEnum reuse,
                                      DocsFlags flags)
        {
            SepTermState termState_ = (SepTermState)termState;

            // If you are using ParellelReader, and pass in a
            // reused DocsAndPositionsEnum, it could have come
            // from another reader also using sep codec
            if (reuse is null || !(reuse is SepDocsEnum docsEnum) || docsEnum.startDocIn != docIn)
            {
                docsEnum = new SepDocsEnum(this);
            }

            return(docsEnum.Init(fieldInfo, termState_, liveDocs));
        }
 // Currently, this instance is re-used across fields, so
 // our parent calls setField whenever the field changes
 public override int SetField(FieldInfo fieldInfo)
 {
     this.fieldInfo    = fieldInfo;
     this.indexOptions = fieldInfo.IndexOptions;
     // LUCENENET specific - to avoid boxing, changed from CompareTo() to IndexOptionsComparer.Compare()
     if (IndexOptionsComparer.Default.Compare(indexOptions, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0)
     {
         throw new NotSupportedException("this codec cannot index offsets");
     }
     skipListWriter.SetIndexOptions(indexOptions);
     storePayloads = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && fieldInfo.HasPayloads;
     lastPayloadFP = 0;
     lastSkipFP    = 0;
     lastState     = SetEmptyState();
     return(0);
 }
Beispiel #13
0
        public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState state, bool absolute)
        {
            SepTermState state_ = (SepTermState)state;

            if (absolute)
            {
                lastSkipFP    = 0;
                lastPayloadFP = 0;
                lastState     = state_;
            }
            lastState.DocIndex.CopyFrom(state_.DocIndex, false);
            lastState.DocIndex.Write(output, absolute);
            if (indexOptions != IndexOptions.DOCS_ONLY)
            {
                lastState.FreqIndex.CopyFrom(state_.FreqIndex, false);
                lastState.FreqIndex.Write(output, absolute);
                if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    lastState.PosIndex.CopyFrom(state_.PosIndex, false);
                    lastState.PosIndex.Write(output, absolute);
                    if (storePayloads)
                    {
                        if (absolute)
                        {
                            output.WriteVInt64(state_.PayloadFP);
                        }
                        else
                        {
                            output.WriteVInt64(state_.PayloadFP - lastPayloadFP);
                        }
                        lastPayloadFP = state_.PayloadFP;
                    }
                }
            }
            if (state_.SkipFP != -1)
            {
                if (absolute)
                {
                    output.WriteVInt64(state_.SkipFP);
                }
                else
                {
                    output.WriteVInt64(state_.SkipFP - lastSkipFP);
                }
                lastSkipFP = state_.SkipFP;
            }
        }
Beispiel #14
0
        public override void DecodeTerm(long[] empty, DataInput input, FieldInfo fieldInfo, BlockTermState termState,
                                        bool absolute)
        {
            SepTermState termState_ = (SepTermState)termState;

            termState_.docIndex.Read(input, absolute);
            if (fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY)
            {
                termState_.freqIndex.Read(input, absolute);
                if (fieldInfo.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    //System.out.println("  freqIndex=" + termState.freqIndex);
                    termState_.posIndex.Read(input, absolute);
                    //System.out.println("  posIndex=" + termState.posIndex);
                    if (fieldInfo.HasPayloads)
                    {
                        if (absolute)
                        {
                            termState_.payloadFP = input.ReadVInt64();
                        }
                        else
                        {
                            termState_.payloadFP += input.ReadVInt64();
                        }
                        //System.out.println("  payloadFP=" + termState.payloadFP);
                    }
                }
            }

            if (termState_.DocFreq >= skipMinimum)
            {
                //System.out.println("   readSkip @ " + in.getPosition());
                if (absolute)
                {
                    termState_.skipFP = input.ReadVInt64();
                }
                else
                {
                    termState_.skipFP += input.ReadVInt64();
                }
                //System.out.println("  skipFP=" + termState.skipFP);
            }
            else if (absolute)
            {
                termState_.skipFP = 0;
            }
        }
Beispiel #15
0
            public override void CopyFrom(TermState other)
            {
                base.CopyFrom(other);
                SepTermState other_ = (SepTermState)other;

                if (docIndex == null)
                {
                    docIndex = (Int32IndexInput.Index)other_.docIndex.Clone();
                }
                else
                {
                    docIndex.CopyFrom(other_.docIndex);
                }
                if (other_.freqIndex != null)
                {
                    if (freqIndex == null)
                    {
                        freqIndex = (Int32IndexInput.Index)other_.freqIndex.Clone();
                    }
                    else
                    {
                        freqIndex.CopyFrom(other_.freqIndex);
                    }
                }
                else
                {
                    freqIndex = null;
                }
                if (other_.posIndex != null)
                {
                    if (posIndex == null)
                    {
                        posIndex = (Int32IndexInput.Index)other_.posIndex.Clone();
                    }
                    else
                    {
                        posIndex.CopyFrom(other_.posIndex);
                    }
                }
                else
                {
                    posIndex = null;
                }
                payloadFP = other_.payloadFP;
                skipFP    = other_.skipFP;
            }
Beispiel #16
0
        private SepTermState SetEmptyState()
        {
            SepTermState emptyState = new SepTermState();

            emptyState.DocIndex = docOut.GetIndex();
            if (indexOptions != IndexOptions.DOCS_ONLY)
            {
                emptyState.FreqIndex = freqOut.GetIndex();
                if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    emptyState.PosIndex = posOut.GetIndex();
                }
            }
            emptyState.PayloadFP = 0;
            emptyState.SkipFP    = 0;
            return(emptyState);
        }
Beispiel #17
0
        private SepTermState SetEmptyState()
        {
            var emptyState = new SepTermState {
                DocIndex = DOC_OUT.Index()
            };

            if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
            {
                emptyState.FreqIndex = FREQ_OUT.Index();
                if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    emptyState.PosIndex = POS_OUT.Index();
                }
            }
            emptyState.PayloadFp = 0;
            emptyState.SkipFp    = 0;
            return(emptyState);
        }
Beispiel #18
0
        public override BlockTermState NewTermState()
        {
            var state = new SepTermState {
                DOC_INDEX = _docIn.Index()
            };

            if (_freqIn != null)
            {
                state.FREQ_INDEX = _freqIn.Index();
            }

            if (_posIn != null)
            {
                state.POS_INDEX = _posIn.Index();
            }

            return(state);
        }
Beispiel #19
0
        // Currently, this instance is re-used across fields, so
        // our parent calls setField whenever the field changes
        public override int SetField(FieldInfo fi)
        {
            FIELD_INFO = fi;

            if (FIELD_INFO.FieldIndexOptions.HasValue)
            {
                INDEX_OPTIONS = FIELD_INFO.FieldIndexOptions.Value;
            }

            if (INDEX_OPTIONS >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
            {
                throw new System.NotSupportedException("this codec cannot index offsets");
            }
            SKIP_LIST_WRITER.IndexOptions = INDEX_OPTIONS;
            STORE_PAYLOADS = INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS &&
                             FIELD_INFO.HasPayloads();
            LAST_PAYLOAD_FP = 0;
            LAST_SKIP_FP    = 0;
            _lastState      = SetEmptyState();
            return(0);
        }
 public override object Clone()
 {
     var other = new SepTermState();
     other.CopyFrom(this);
     return other;
 }
        public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState bstate, bool absolute)
        {
            var state = (SepTermState) bstate;
            if (absolute)
            {
                LAST_SKIP_FP = 0;
                LAST_PAYLOAD_FP = 0;
                _lastState = state;
            }
            _lastState.DocIndex.CopyFrom(state.DocIndex, false);
            _lastState.DocIndex.Write(output, absolute);
            if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
            {
                _lastState.FreqIndex.CopyFrom(state.FreqIndex, false);
                _lastState.FreqIndex.Write(output, absolute);
                if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
                {
                    _lastState.PosIndex.CopyFrom(state.PosIndex, false);
                    _lastState.PosIndex.Write(output, absolute);
                    if (STORE_PAYLOADS)
                    {
                        if (absolute)
                        {
                            output.WriteVLong(state.PayloadFp);
                        }
                        else
                        {
                            output.WriteVLong(state.PayloadFp - LAST_PAYLOAD_FP);
                        }
                        LAST_PAYLOAD_FP = state.PayloadFp;
                    }
                }
            }
            if (state.SkipFp == -1) return;

            if (absolute)
            {
                output.WriteVLong(state.SkipFp);
            }
            else
            {
                output.WriteVLong(state.SkipFp - LAST_SKIP_FP);
            }
            LAST_SKIP_FP = state.SkipFp;
        }
        // Currently, this instance is re-used across fields, so
        // our parent calls setField whenever the field changes
        public override int SetField(FieldInfo fi)
        {
            FIELD_INFO = fi;

            if (FIELD_INFO.FieldIndexOptions.HasValue)
                INDEX_OPTIONS = FIELD_INFO.FieldIndexOptions.Value;

            if (INDEX_OPTIONS >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
            {
                throw new System.NotSupportedException("this codec cannot index offsets");
            }
            SKIP_LIST_WRITER.IndexOptions = INDEX_OPTIONS;
            STORE_PAYLOADS = INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS &&
                            FIELD_INFO.HasPayloads();
            LAST_PAYLOAD_FP = 0;
            LAST_SKIP_FP = 0;
            _lastState = SetEmptyState();
            return 0;
        }
 private SepTermState SetEmptyState()
 {
     var emptyState = new SepTermState {DocIndex = DOC_OUT.Index()};
     if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY)
     {
         emptyState.FreqIndex = FREQ_OUT.Index();
         if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
         {
             emptyState.PosIndex = POS_OUT.Index();
         }
     }
     emptyState.PayloadFp = 0;
     emptyState.SkipFp = 0;
     return emptyState;
 }
            internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
            {
                _liveDocs = liveDocs;
                _storePayloads = fieldInfo.HasPayloads();

                // TODO: can't we only do this if consumer skipped consuming the previous docs?
                _docIndex.CopyFrom(termState.DOC_INDEX);
                _docIndex.Seek(_docReader);

                _freqIndex.CopyFrom(termState.FREQ_INDEX);
                _freqIndex.Seek(_freqReader);

                _posIndex.CopyFrom(termState.POS_INDEX);
                _posSeekPending = true;
                _payloadPending = false;

                _payloadFp = termState.PAYLOAD_FP;
                _skipFp = termState.SKIP_FP;

                _docFreq = termState.DocFreq;
                _count = 0;
                _doc = -1;
                _accum = 0;
                _pendingPosCount = 0;
                _pendingPayloadBytes = 0;
                _skipped = false;

                return this;
            }
        public override BlockTermState NewTermState()
        {
            var state = new SepTermState {DOC_INDEX = _docIn.Index()};

            if (_freqIn != null)
                state.FREQ_INDEX = _freqIn.Index();

            if (_posIn != null)
                state.POS_INDEX = _posIn.Index();

            return state;
        }
            internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
            {
                _liveDocs = liveDocs;
                if (fieldInfo.FieldIndexOptions.HasValue)
                    _indexOptions = fieldInfo.FieldIndexOptions.Value;

                _omitTf = _indexOptions == FieldInfo.IndexOptions.DOCS_ONLY;
                _storePayloads = fieldInfo.HasPayloads();

                // TODO: can't we only do this if consumer
                // skipped consuming the previous docs?
                _docIndex.CopyFrom(termState.DOC_INDEX);
                _docIndex.Seek(_docReader);

                if (!_omitTf)
                {
                    _freqIndex.CopyFrom(termState.FREQ_INDEX);
                    _freqIndex.Seek(_freqReader);
                }

                _docFreq = termState.DocFreq;
                // NOTE: unused if docFreq < skipMinimum:
                _skipFp = termState.SKIP_FP;
                _count = 0;
                _doc = -1;
                _accum = 0;
                _freq = 1;
                _skipped = false;

                return this;
            }