//private readonly string segment; // LUCENENET: Not used public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, string segment, int indexDivisor, string segmentSuffix, IOContext context) { input = dir.OpenInput(IndexFileNames.SegmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); //this.segment = segment; // LUCENENET: Not used bool success = false; if (Debugging.AssertsEnabled) { Debugging.Assert(indexDivisor == -1 || indexDivisor > 0); } try { version = ReadHeader(input); this.indexDivisor = indexDivisor; if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) { CodecUtil.ChecksumEntireFile(input); } SeekDir(input, dirOffset); // Read directory int numFields = input.ReadVInt32(); if (numFields < 0) { throw new CorruptIndexException("invalid numFields: " + numFields + " (resource=" + input + ")"); } for (int i = 0; i < numFields; i++) { int field = input.ReadVInt32(); long indexStart = input.ReadVInt64(); FieldInfo fieldInfo = fieldInfos.FieldInfo(field); FieldIndexData previous = fields.Put(fieldInfo, new FieldIndexData(this, /* fieldInfo, // LUCENENET: Not referenced */ indexStart)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.Name + " (resource=" + input + ")"); } } success = true; } finally { if (indexDivisor > 0) { input.Dispose(); input = null; if (success) { indexLoaded = true; } } } }
public IndexEnum(FieldIndexData.CoreFieldIndex fieldIndex, FixedGapTermsIndexReader fgtir) { Term = new BytesRef(); _fieldIndex = fieldIndex; _fgtir = fgtir; }
public CoreFieldIndex(FieldIndexData outerInstance, long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart, int numIndexTerms) { this.termsStart = termsStart; termBytesStart = outerInstance.outerInstance.termBytes.GetPointer(); IndexInput clone = (IndexInput)outerInstance.outerInstance.input.Clone(); clone.Seek(indexStart); // -1 is passed to mean "don't load term index", but // if we are then later loaded it's overwritten with // a real value if (Debugging.AssertsEnabled) { Debugging.Assert(outerInstance.outerInstance.indexDivisor > 0); } this.numIndexTerms = 1 + (numIndexTerms - 1) / outerInstance.outerInstance.indexDivisor; if (Debugging.AssertsEnabled) { Debugging.Assert(this.numIndexTerms > 0, "numIndexTerms={0} indexDivisor={1}", numIndexTerms, outerInstance.outerInstance.indexDivisor); } if (outerInstance.outerInstance.indexDivisor == 1) { // Default (load all index terms) is fast -- slurp in the images from disk: try { long numTermBytes = packedIndexStart - indexStart; outerInstance.outerInstance.termBytes.Copy(clone, numTermBytes); // records offsets into main terms dict file termsDictOffsets = PackedInt32s.GetReader(clone); if (Debugging.AssertsEnabled) { Debugging.Assert(termsDictOffsets.Count == numIndexTerms); } // records offsets into byte[] term data termOffsets = PackedInt32s.GetReader(clone); if (Debugging.AssertsEnabled) { Debugging.Assert(termOffsets.Count == 1 + numIndexTerms); } } finally { clone.Dispose(); } } else { // Get packed iterators IndexInput clone1 = (IndexInput)outerInstance.outerInstance.input.Clone(); IndexInput clone2 = (IndexInput)outerInstance.outerInstance.input.Clone(); try { // Subsample the index terms clone1.Seek(packedIndexStart); PackedInt32s.IReaderIterator termsDictOffsetsIter = PackedInt32s.GetReaderIterator(clone1, PackedInt32s.DEFAULT_BUFFER_SIZE); clone2.Seek(packedOffsetsStart); PackedInt32s.IReaderIterator termOffsetsIter = PackedInt32s.GetReaderIterator(clone2, PackedInt32s.DEFAULT_BUFFER_SIZE); // TODO: often we can get by w/ fewer bits per // value, below.. .but this'd be more complex: // we'd have to try @ fewer bits and then grow // if we overflowed it. PackedInt32s.Mutable termsDictOffsetsM = PackedInt32s.GetMutable(this.numIndexTerms, termsDictOffsetsIter.BitsPerValue, PackedInt32s.DEFAULT); PackedInt32s.Mutable termOffsetsM = PackedInt32s.GetMutable(this.numIndexTerms + 1, termOffsetsIter.BitsPerValue, PackedInt32s.DEFAULT); termsDictOffsets = termsDictOffsetsM; termOffsets = termOffsetsM; int upto = 0; long termOffsetUpto = 0; while (upto < this.numIndexTerms) { // main file offset copies straight over termsDictOffsetsM.Set(upto, termsDictOffsetsIter.Next()); termOffsetsM.Set(upto, termOffsetUpto); long termOffset = termOffsetsIter.Next(); long nextTermOffset = termOffsetsIter.Next(); int numTermBytes = (int)(nextTermOffset - termOffset); clone.Seek(indexStart + termOffset); if (Debugging.AssertsEnabled) { Debugging.Assert(indexStart + termOffset < clone.Length, "indexStart={0} termOffset={1} len={2}", indexStart, termOffset, clone.Length); Debugging.Assert(indexStart + termOffset + numTermBytes < clone.Length); } outerInstance.outerInstance.termBytes.Copy(clone, numTermBytes); termOffsetUpto += numTermBytes; upto++; if (upto == this.numIndexTerms) { break; } // skip terms: termsDictOffsetsIter.Next(); for (int i = 0; i < outerInstance.outerInstance.indexDivisor - 2; i++) { termOffsetsIter.Next(); termsDictOffsetsIter.Next(); } } termOffsetsM.Set(upto, termOffsetUpto); } finally { clone1.Dispose(); clone2.Dispose(); clone.Dispose(); } } }
public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, string segment, int indexDivisor, IComparer <BytesRef> termComp, string segmentSuffix, IOContext context) { this.termComp = termComp; if (Debugging.AssertsEnabled) { Debugging.Assert(indexDivisor == -1 || indexDivisor > 0); } input = dir.OpenInput(IndexFileNames.SegmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); bool success = false; try { version = ReadHeader(input); if (version >= FixedGapTermsIndexWriter.VERSION_CHECKSUM) { CodecUtil.ChecksumEntireFile(input); } indexInterval = input.ReadInt32(); if (indexInterval < 1) { throw new CorruptIndexException("invalid indexInterval: " + indexInterval + " (resource=" + input + ")"); } this.indexDivisor = indexDivisor; if (indexDivisor < 0) { totalIndexInterval = indexInterval; } else { // In case terms index gets loaded, later, on demand totalIndexInterval = indexInterval * indexDivisor; } if (Debugging.AssertsEnabled) { Debugging.Assert(totalIndexInterval > 0); } SeekDir(input, dirOffset); // Read directory int numFields = input.ReadVInt32(); if (numFields < 0) { throw new CorruptIndexException("invalid numFields: " + numFields + " (resource=" + input + ")"); } //System.out.println("FGR: init seg=" + segment + " div=" + indexDivisor + " nF=" + numFields); for (int i = 0; i < numFields; i++) { int field = input.ReadVInt32(); int numIndexTerms = input.ReadVInt32(); if (numIndexTerms < 0) { throw new CorruptIndexException("invalid numIndexTerms: " + numIndexTerms + " (resource=" + input + ")"); } long termsStart = input.ReadVInt64(); long indexStart = input.ReadVInt64(); long packedIndexStart = input.ReadVInt64(); long packedOffsetsStart = input.ReadVInt64(); if (packedIndexStart < indexStart) { throw new CorruptIndexException("invalid packedIndexStart: " + packedIndexStart + " indexStart: " + indexStart + "numIndexTerms: " + numIndexTerms + " (resource=" + input + ")"); } FieldInfo fieldInfo = fieldInfos.FieldInfo(field); FieldIndexData previous = fields.Put(fieldInfo, new FieldIndexData(this, /* fieldInfo, // LUCENENET: Not referenced */ numIndexTerms, indexStart, termsStart, packedIndexStart, packedOffsetsStart)); if (previous != null) { throw new CorruptIndexException("duplicate field: " + fieldInfo.Name + " (resource=" + input + ")"); } } success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(input); } if (indexDivisor > 0) { input.Dispose(); input = null; if (success) { indexLoaded = true; } termBytesReader = termBytes.Freeze(true); } } }
public override FieldIndexEnum GetFieldEnum(FieldInfo fieldInfo) { FieldIndexData fieldData = _fields[fieldInfo]; return(fieldData.Fst == null ? null : new IndexEnum(fieldData.Fst)); }