internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) { _liveDocs = liveDocs; _storePayloads = fieldInfo.HasPayloads(); // TODO: can't we only do this if consumer skipped consuming the previous docs? _docIndex.CopyFrom(termState.DOC_INDEX); _docIndex.Seek(_docReader); _freqIndex.CopyFrom(termState.FREQ_INDEX); _freqIndex.Seek(_freqReader); _posIndex.CopyFrom(termState.POS_INDEX); _posSeekPending = true; _payloadPending = false; _payloadFp = termState.PAYLOAD_FP; _skipFp = termState.SKIP_FP; _docFreq = termState.DocFreq; _count = 0; _doc = -1; _accum = 0; _pendingPosCount = 0; _pendingPayloadBytes = 0; _skipped = false; return(this); }
internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, IBits liveDocs) { this.liveDocs = liveDocs; storePayloads = fieldInfo.HasPayloads; //System.out.println("Sep D&P init"); // TODO: can't we only do this if consumer // skipped consuming the previous docs? docIndex.CopyFrom(termState.docIndex); docIndex.Seek(docReader); //System.out.println(" docIndex=" + docIndex); freqIndex.CopyFrom(termState.freqIndex); freqIndex.Seek(freqReader); //System.out.println(" freqIndex=" + freqIndex); posIndex.CopyFrom(termState.posIndex); //System.out.println(" posIndex=" + posIndex); posSeekPending = true; payloadPending = false; payloadFP = termState.payloadFP; skipFP = termState.skipFP; //System.out.println(" skipFP=" + skipFP); docFreq = termState.DocFreq; count = 0; doc = -1; accum = 0; pendingPosCount = 0; pendingPayloadBytes = 0; skipped = false; return(this); }
public override object Clone() { var other = new SepTermState(); other.CopyFrom(this); return(other); }
public override DocsAndPositionsEnum DocsAndPositions(FieldInfo fieldInfo, BlockTermState termState, IBits liveDocs, DocsAndPositionsEnum reuse, DocsAndPositionsFlags flags) { Debug.Assert(fieldInfo.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); SepTermState termState_ = (SepTermState)termState; SepDocsAndPositionsEnum postingsEnum; if (reuse == null || !(reuse is SepDocsAndPositionsEnum)) { postingsEnum = new SepDocsAndPositionsEnum(this); } else { postingsEnum = (SepDocsAndPositionsEnum)reuse; if (postingsEnum.startDocIn != docIn) { // If you are using ParellelReader, and pass in a // reused DocsAndPositionsEnum, it could have come // from another reader also using sep codec postingsEnum = new SepDocsAndPositionsEnum(this); } } return(postingsEnum.Init(fieldInfo, termState_, liveDocs)); }
internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, IBits liveDocs) { this.liveDocs = liveDocs; this.indexOptions = fieldInfo.IndexOptions; omitTF = indexOptions == IndexOptions.DOCS_ONLY; storePayloads = fieldInfo.HasPayloads; // TODO: can't we only do this if consumer // skipped consuming the previous docs? docIndex.CopyFrom(termState.docIndex); docIndex.Seek(docReader); if (!omitTF) { freqIndex.CopyFrom(termState.freqIndex); freqIndex.Seek(freqReader); } docFreq = termState.DocFreq; // NOTE: unused if docFreq < skipMinimum: skipFP = termState.skipFP; count = 0; doc = -1; accum = 0; freq = 1; skipped = false; return(this); }
internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) { _liveDocs = liveDocs; if (fieldInfo.FieldIndexOptions.HasValue) { _indexOptions = fieldInfo.FieldIndexOptions.Value; } _omitTf = _indexOptions == FieldInfo.IndexOptions.DOCS_ONLY; _storePayloads = fieldInfo.HasPayloads(); // TODO: can't we only do this if consumer // skipped consuming the previous docs? _docIndex.CopyFrom(termState.DOC_INDEX); _docIndex.Seek(_docReader); if (!_omitTf) { _freqIndex.CopyFrom(termState.FREQ_INDEX); _freqIndex.Seek(_freqReader); } _docFreq = termState.DocFreq; // NOTE: unused if docFreq < skipMinimum: _skipFp = termState.SKIP_FP; _count = 0; _doc = -1; _accum = 0; _freq = 1; _skipped = false; return(this); }
public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState bstate, bool absolute) { var state = (SepTermState)bstate; if (absolute) { LAST_SKIP_FP = 0; LAST_PAYLOAD_FP = 0; _lastState = state; } _lastState.DocIndex.CopyFrom(state.DocIndex, false); _lastState.DocIndex.Write(output, absolute); if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY) { _lastState.FreqIndex.CopyFrom(state.FreqIndex, false); _lastState.FreqIndex.Write(output, absolute); if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { _lastState.PosIndex.CopyFrom(state.PosIndex, false); _lastState.PosIndex.Write(output, absolute); if (STORE_PAYLOADS) { if (absolute) { output.WriteVLong(state.PayloadFp); } else { output.WriteVLong(state.PayloadFp - LAST_PAYLOAD_FP); } LAST_PAYLOAD_FP = state.PayloadFp; } } } if (state.SkipFp == -1) { return; } if (absolute) { output.WriteVLong(state.SkipFp); } else { output.WriteVLong(state.SkipFp - LAST_SKIP_FP); } LAST_SKIP_FP = state.SkipFp; }
/// <summary>Called when we are done adding docs to this term. </summary> public override void FinishTerm(BlockTermState state) { SepTermState state_ = (SepTermState)state; // TODO: -- wasteful we are counting this in two places? if (Debugging.AssertsEnabled) { Debugging.Assert(state_.DocFreq > 0); Debugging.Assert(state_.DocFreq == df); } state_.DocIndex = docOut.GetIndex(); state_.DocIndex.CopyFrom(docIndex, false); if (indexOptions != IndexOptions.DOCS_ONLY) { state_.FreqIndex = freqOut.GetIndex(); state_.FreqIndex.CopyFrom(freqIndex, false); if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { state_.PosIndex = posOut.GetIndex(); state_.PosIndex.CopyFrom(posIndex, false); } else { state_.PosIndex = null; } } else { state_.FreqIndex = null; state_.PosIndex = null; } if (df >= skipMinimum) { state_.SkipFP = skipOut.GetFilePointer(); //System.out.println(" skipFP=" + skipFP); skipListWriter.WriteSkip(skipOut); //System.out.println(" numBytes=" + (skipOut.getFilePointer()-skipFP)); } else { state_.SkipFP = -1; } state_.PayloadFP = payloadStart; lastDocID = 0; df = 0; }
// Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes public override int SetField(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; this.indexOptions = fieldInfo.IndexOptions; if (indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) { throw new System.NotSupportedException("this codec cannot index offsets"); } skipListWriter.SetIndexOptions(indexOptions); storePayloads = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && fieldInfo.HasPayloads; lastPayloadFP = 0; lastSkipFP = 0; lastState = SetEmptyState(); return(0); }
public override BlockTermState NewTermState() { SepTermState state = new SepTermState(); state.docIndex = docIn.GetIndex(); if (freqIn != null) { state.freqIndex = freqIn.GetIndex(); } if (posIn != null) { state.posIndex = posIn.GetIndex(); } return(state); }
public override DocsEnum Docs(FieldInfo fieldInfo, BlockTermState termState, IBits liveDocs, DocsEnum reuse, DocsFlags flags) { SepTermState termState_ = (SepTermState)termState; // If you are using ParellelReader, and pass in a // reused DocsAndPositionsEnum, it could have come // from another reader also using sep codec if (reuse is null || !(reuse is SepDocsEnum docsEnum) || docsEnum.startDocIn != docIn) { docsEnum = new SepDocsEnum(this); } return(docsEnum.Init(fieldInfo, termState_, liveDocs)); }
// Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes public override int SetField(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; this.indexOptions = fieldInfo.IndexOptions; // LUCENENET specific - to avoid boxing, changed from CompareTo() to IndexOptionsComparer.Compare() if (IndexOptionsComparer.Default.Compare(indexOptions, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) { throw new NotSupportedException("this codec cannot index offsets"); } skipListWriter.SetIndexOptions(indexOptions); storePayloads = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && fieldInfo.HasPayloads; lastPayloadFP = 0; lastSkipFP = 0; lastState = SetEmptyState(); return(0); }
public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState state, bool absolute) { SepTermState state_ = (SepTermState)state; if (absolute) { lastSkipFP = 0; lastPayloadFP = 0; lastState = state_; } lastState.DocIndex.CopyFrom(state_.DocIndex, false); lastState.DocIndex.Write(output, absolute); if (indexOptions != IndexOptions.DOCS_ONLY) { lastState.FreqIndex.CopyFrom(state_.FreqIndex, false); lastState.FreqIndex.Write(output, absolute); if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { lastState.PosIndex.CopyFrom(state_.PosIndex, false); lastState.PosIndex.Write(output, absolute); if (storePayloads) { if (absolute) { output.WriteVInt64(state_.PayloadFP); } else { output.WriteVInt64(state_.PayloadFP - lastPayloadFP); } lastPayloadFP = state_.PayloadFP; } } } if (state_.SkipFP != -1) { if (absolute) { output.WriteVInt64(state_.SkipFP); } else { output.WriteVInt64(state_.SkipFP - lastSkipFP); } lastSkipFP = state_.SkipFP; } }
public override void DecodeTerm(long[] empty, DataInput input, FieldInfo fieldInfo, BlockTermState termState, bool absolute) { SepTermState termState_ = (SepTermState)termState; termState_.docIndex.Read(input, absolute); if (fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY) { termState_.freqIndex.Read(input, absolute); if (fieldInfo.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { //System.out.println(" freqIndex=" + termState.freqIndex); termState_.posIndex.Read(input, absolute); //System.out.println(" posIndex=" + termState.posIndex); if (fieldInfo.HasPayloads) { if (absolute) { termState_.payloadFP = input.ReadVInt64(); } else { termState_.payloadFP += input.ReadVInt64(); } //System.out.println(" payloadFP=" + termState.payloadFP); } } } if (termState_.DocFreq >= skipMinimum) { //System.out.println(" readSkip @ " + in.getPosition()); if (absolute) { termState_.skipFP = input.ReadVInt64(); } else { termState_.skipFP += input.ReadVInt64(); } //System.out.println(" skipFP=" + termState.skipFP); } else if (absolute) { termState_.skipFP = 0; } }
public override void CopyFrom(TermState other) { base.CopyFrom(other); SepTermState other_ = (SepTermState)other; if (docIndex == null) { docIndex = (Int32IndexInput.Index)other_.docIndex.Clone(); } else { docIndex.CopyFrom(other_.docIndex); } if (other_.freqIndex != null) { if (freqIndex == null) { freqIndex = (Int32IndexInput.Index)other_.freqIndex.Clone(); } else { freqIndex.CopyFrom(other_.freqIndex); } } else { freqIndex = null; } if (other_.posIndex != null) { if (posIndex == null) { posIndex = (Int32IndexInput.Index)other_.posIndex.Clone(); } else { posIndex.CopyFrom(other_.posIndex); } } else { posIndex = null; } payloadFP = other_.payloadFP; skipFP = other_.skipFP; }
private SepTermState SetEmptyState() { SepTermState emptyState = new SepTermState(); emptyState.DocIndex = docOut.GetIndex(); if (indexOptions != IndexOptions.DOCS_ONLY) { emptyState.FreqIndex = freqOut.GetIndex(); if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { emptyState.PosIndex = posOut.GetIndex(); } } emptyState.PayloadFP = 0; emptyState.SkipFP = 0; return(emptyState); }
private SepTermState SetEmptyState() { var emptyState = new SepTermState { DocIndex = DOC_OUT.Index() }; if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY) { emptyState.FreqIndex = FREQ_OUT.Index(); if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { emptyState.PosIndex = POS_OUT.Index(); } } emptyState.PayloadFp = 0; emptyState.SkipFp = 0; return(emptyState); }
public override BlockTermState NewTermState() { var state = new SepTermState { DOC_INDEX = _docIn.Index() }; if (_freqIn != null) { state.FREQ_INDEX = _freqIn.Index(); } if (_posIn != null) { state.POS_INDEX = _posIn.Index(); } return(state); }
// Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes public override int SetField(FieldInfo fi) { FIELD_INFO = fi; if (FIELD_INFO.FieldIndexOptions.HasValue) { INDEX_OPTIONS = FIELD_INFO.FieldIndexOptions.Value; } if (INDEX_OPTIONS >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { throw new System.NotSupportedException("this codec cannot index offsets"); } SKIP_LIST_WRITER.IndexOptions = INDEX_OPTIONS; STORE_PAYLOADS = INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && FIELD_INFO.HasPayloads(); LAST_PAYLOAD_FP = 0; LAST_SKIP_FP = 0; _lastState = SetEmptyState(); return(0); }
public override object Clone() { var other = new SepTermState(); other.CopyFrom(this); return other; }
public override void EncodeTerm(long[] longs, DataOutput output, FieldInfo fi, BlockTermState bstate, bool absolute) { var state = (SepTermState) bstate; if (absolute) { LAST_SKIP_FP = 0; LAST_PAYLOAD_FP = 0; _lastState = state; } _lastState.DocIndex.CopyFrom(state.DocIndex, false); _lastState.DocIndex.Write(output, absolute); if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY) { _lastState.FreqIndex.CopyFrom(state.FreqIndex, false); _lastState.FreqIndex.Write(output, absolute); if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { _lastState.PosIndex.CopyFrom(state.PosIndex, false); _lastState.PosIndex.Write(output, absolute); if (STORE_PAYLOADS) { if (absolute) { output.WriteVLong(state.PayloadFp); } else { output.WriteVLong(state.PayloadFp - LAST_PAYLOAD_FP); } LAST_PAYLOAD_FP = state.PayloadFp; } } } if (state.SkipFp == -1) return; if (absolute) { output.WriteVLong(state.SkipFp); } else { output.WriteVLong(state.SkipFp - LAST_SKIP_FP); } LAST_SKIP_FP = state.SkipFp; }
// Currently, this instance is re-used across fields, so // our parent calls setField whenever the field changes public override int SetField(FieldInfo fi) { FIELD_INFO = fi; if (FIELD_INFO.FieldIndexOptions.HasValue) INDEX_OPTIONS = FIELD_INFO.FieldIndexOptions.Value; if (INDEX_OPTIONS >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { throw new System.NotSupportedException("this codec cannot index offsets"); } SKIP_LIST_WRITER.IndexOptions = INDEX_OPTIONS; STORE_PAYLOADS = INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && FIELD_INFO.HasPayloads(); LAST_PAYLOAD_FP = 0; LAST_SKIP_FP = 0; _lastState = SetEmptyState(); return 0; }
private SepTermState SetEmptyState() { var emptyState = new SepTermState {DocIndex = DOC_OUT.Index()}; if (INDEX_OPTIONS != FieldInfo.IndexOptions.DOCS_ONLY) { emptyState.FreqIndex = FREQ_OUT.Index(); if (INDEX_OPTIONS == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { emptyState.PosIndex = POS_OUT.Index(); } } emptyState.PayloadFp = 0; emptyState.SkipFp = 0; return emptyState; }
internal virtual SepDocsAndPositionsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) { _liveDocs = liveDocs; _storePayloads = fieldInfo.HasPayloads(); // TODO: can't we only do this if consumer skipped consuming the previous docs? _docIndex.CopyFrom(termState.DOC_INDEX); _docIndex.Seek(_docReader); _freqIndex.CopyFrom(termState.FREQ_INDEX); _freqIndex.Seek(_freqReader); _posIndex.CopyFrom(termState.POS_INDEX); _posSeekPending = true; _payloadPending = false; _payloadFp = termState.PAYLOAD_FP; _skipFp = termState.SKIP_FP; _docFreq = termState.DocFreq; _count = 0; _doc = -1; _accum = 0; _pendingPosCount = 0; _pendingPayloadBytes = 0; _skipped = false; return this; }
public override BlockTermState NewTermState() { var state = new SepTermState {DOC_INDEX = _docIn.Index()}; if (_freqIn != null) state.FREQ_INDEX = _freqIn.Index(); if (_posIn != null) state.POS_INDEX = _posIn.Index(); return state; }
internal virtual SepDocsEnum Init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs) { _liveDocs = liveDocs; if (fieldInfo.FieldIndexOptions.HasValue) _indexOptions = fieldInfo.FieldIndexOptions.Value; _omitTf = _indexOptions == FieldInfo.IndexOptions.DOCS_ONLY; _storePayloads = fieldInfo.HasPayloads(); // TODO: can't we only do this if consumer // skipped consuming the previous docs? _docIndex.CopyFrom(termState.DOC_INDEX); _docIndex.Seek(_docReader); if (!_omitTf) { _freqIndex.CopyFrom(termState.FREQ_INDEX); _freqIndex.Seek(_freqReader); } _docFreq = termState.DocFreq; // NOTE: unused if docFreq < skipMinimum: _skipFp = termState.SKIP_FP; _count = 0; _doc = -1; _accum = 0; _freq = 1; _skipped = false; return this; }