private int ReadHeader(IndexInput input) { int version = CodecUtil.CheckHeader(input, VariableGapTermsIndexWriter.CODEC_NAME, VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT); if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) { dirOffset = input.ReadInt64(); } return(version); }
/// <summary> /// Sole constructor. </summary> public Lucene40TermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context) { string segment = si.Name; int size = si.DocCount; bool success = false; try { string idxName = IndexFileNames.SegmentFileName(segment, "", VECTORS_INDEX_EXTENSION); tvx = d.OpenInput(idxName, context); int tvxVersion = CodecUtil.CheckHeader(tvx, CODEC_NAME_INDEX, VERSION_START, VERSION_CURRENT); string fn = IndexFileNames.SegmentFileName(segment, "", VECTORS_DOCUMENTS_EXTENSION); tvd = d.OpenInput(fn, context); int tvdVersion = CodecUtil.CheckHeader(tvd, CODEC_NAME_DOCS, VERSION_START, VERSION_CURRENT); fn = IndexFileNames.SegmentFileName(segment, "", VECTORS_FIELDS_EXTENSION); tvf = d.OpenInput(fn, context); int tvfVersion = CodecUtil.CheckHeader(tvf, CODEC_NAME_FIELDS, VERSION_START, VERSION_CURRENT); Debug.Assert(HEADER_LENGTH_INDEX == tvx.GetFilePointer()); Debug.Assert(HEADER_LENGTH_DOCS == tvd.GetFilePointer()); Debug.Assert(HEADER_LENGTH_FIELDS == tvf.GetFilePointer()); Debug.Assert(tvxVersion == tvdVersion); Debug.Assert(tvxVersion == tvfVersion); numTotalDocs = (int)(tvx.Length - HEADER_LENGTH_INDEX >> 4); this.size = numTotalDocs; Debug.Assert(size == 0 || numTotalDocs == size); this.fieldInfos = fieldInfos; success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { try { Dispose(); } // ensure we throw our original exception #pragma warning disable 168 catch (Exception t) #pragma warning restore 168 { } } } }
private int ReadHeader(DataInput input) { var version = CodecUtil.CheckHeader(input, FixedGapTermsIndexWriter.CODEC_NAME, FixedGapTermsIndexWriter.VERSION_START, FixedGapTermsIndexWriter.VERSION_CURRENT); if (version < FixedGapTermsIndexWriter.VERSION_APPEND_ONLY) { _dirOffset = input.ReadLong(); } return(version); }
private int ReadHeader(DataInput input) { int version = CodecUtil.CheckHeader(input, BlockTermsWriter.CODEC_NAME, BlockTermsWriter.VERSION_START, BlockTermsWriter.VERSION_CURRENT); if (version < BlockTermsWriter.VERSION_APPEND_ONLY) { dirOffset = input.ReadInt64(); } return(version); }
public BloomFilteredFieldsProducer(BloomFilteringPostingsFormat outerInstance, SegmentReadState state) { this.outerInstance = outerInstance; var bloomFileName = IndexFileNames.SegmentFileName( state.SegmentInfo.Name, state.SegmentSuffix, BLOOM_EXTENSION); ChecksumIndexInput bloomIn = null; var success = false; try { bloomIn = state.Directory.OpenChecksumInput(bloomFileName, state.Context); var version = CodecUtil.CheckHeader(bloomIn, /*BLOOM_CODEC_NAME*/ outerInstance.Name, VERSION_START, VERSION_CURRENT); // Load the hash function used in the BloomFilter // hashFunction = HashFunction.forName(bloomIn.readString()); // Load the delegate postings format var delegatePostingsFormat = ForName(bloomIn.ReadString()); _delegateFieldsProducer = delegatePostingsFormat .FieldsProducer(state); var numBlooms = bloomIn.ReadInt32(); for (var i = 0; i < numBlooms; i++) { var fieldNum = bloomIn.ReadInt32(); var bloom = FuzzySet.Deserialize(bloomIn); var fieldInfo = state.FieldInfos.FieldInfo(fieldNum); _bloomsByFieldName.Add(fieldInfo.Name, bloom); } if (version >= VERSION_CHECKSUM) { CodecUtil.CheckFooter(bloomIn); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(bloomIn); #pragma warning restore 612, 618 } IOUtils.Dispose(bloomIn); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(bloomIn, _delegateFieldsProducer); } } }
private CharacterDefinition() { using Stream @is = BinaryDictionary.GetTypeResource(GetType(), FILENAME_SUFFIX); DataInput @in = new InputStreamDataInput(@is); CodecUtil.CheckHeader(@in, HEADER, VERSION, VERSION); @in.ReadBytes(characterCategoryMap, 0, characterCategoryMap.Length); for (int i = 0; i < CLASS_COUNT; i++) { byte b = @in.ReadByte(); invokeMap[i] = (b & 0x01) != 0; groupMap[i] = (b & 0x02) != 0; } }
public override SegmentInfo Read(Directory dir, string segment, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION); ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context); bool success = false; try { int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT); string version = input.ReadString(); int docCount = input.ReadInt32(); if (docCount < 0) { throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); } bool isCompoundFile = input.ReadByte() == SegmentInfo.YES; IDictionary <string, string> diagnostics = input.ReadStringStringMap(); ISet <string> files = input.ReadStringSet(); if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM) { CodecUtil.CheckFooter(input); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(input); #pragma warning restore 612, 618 } SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics); si.SetFiles(files); success = true; return(si); } finally { if (!success) { IOUtils.DisposeWhileHandlingException(input); } else { input.Dispose(); } } }
/// <summary> /// Sole constructor. </summary> public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) { string segment = si.Name; bool success = false; fieldInfos = fn; try { fieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context); string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION); indexStream = d.OpenInput(indexStreamFN, context); CodecUtil.CheckHeader(indexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); CodecUtil.CheckHeader(fieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT); if (Debugging.AssertsEnabled) { Debugging.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == fieldsStream.GetFilePointer()); Debugging.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == indexStream.GetFilePointer()); } long indexSize = indexStream.Length - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX; this.size = (int)(indexSize >> 3); // Verify two sources of "maxDoc" agree: if (this.size != si.DocCount) { throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.size + " but segmentInfo shows " + si.DocCount); } numTotalDocs = (int)(indexSize >> 3); success = true; } finally { // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { try { Dispose(); } // ensure we throw our original exception catch (Exception) { } } } }
private NumericDocValues LoadByteField(FieldInfo field, IndexInput input) { CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_START, Lucene40DocValuesFormat.INTS_VERSION_CURRENT); int valueSize = input.ReadInt(); if (valueSize != 1) { throw new CorruptIndexException("invalid valueSize: " + valueSize); } int maxDoc = State.SegmentInfo.DocCount; sbyte[] values = new sbyte[maxDoc]; input.ReadBytes(values, 0, values.Length); RamBytesUsed_Renamed.AddAndGet(RamUsageEstimator.SizeOf(values)); return(new NumericDocValuesAnonymousInnerClassHelper3(this, values)); }
private NumericDocValues LoadByteField(/*FieldInfo field, // LUCENENET: Never read */ IndexInput input) { CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_START, Lucene40DocValuesFormat.INTS_VERSION_CURRENT); int valueSize = input.ReadInt32(); if (valueSize != 1) { throw new CorruptIndexException("invalid valueSize: " + valueSize); } int maxDoc = state.SegmentInfo.DocCount; var values = new byte[maxDoc]; input.ReadBytes(values, 0, values.Length); ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values)); return(new NumericDocValuesAnonymousClass3(values)); }
private SortedDocValues LoadBytesFixedSorted(/*FieldInfo field, // LUCENENET: Never read */ IndexInput data, IndexInput index) { CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT); CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT); int fixedLength = data.ReadInt32(); int valueCount = index.ReadInt32(); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, fixedLength * (long)valueCount); PagedBytes.Reader bytesReader = bytes.Freeze(true); PackedInt32s.Reader reader = PackedInt32s.GetReader(index); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed()); return(CorrectBuggyOrds(new SortedDocValuesAnonymousClass(fixedLength, valueCount, bytesReader, reader))); }
private SortedDocValues LoadBytesVarSorted(/*FieldInfo field, // LUCENENET: Never read */ IndexInput data, IndexInput index) { CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); long maxAddress = index.ReadInt64(); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, maxAddress); PagedBytes.Reader bytesReader = bytes.Freeze(true); PackedInt32s.Reader addressReader = PackedInt32s.GetReader(index); PackedInt32s.Reader ordsReader = PackedInt32s.GetReader(index); int valueCount = addressReader.Count - 1; ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addressReader.RamBytesUsed() + ordsReader.RamBytesUsed()); return(CorrectBuggyOrds(new SortedDocValuesAnonymousClass2(bytesReader, addressReader, ordsReader, valueCount))); }
/// <summary> /// NOTE: This was loadFloatField() in Lucene. /// </summary> private NumericDocValues LoadSingleField(FieldInfo field, IndexInput input) { CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.FLOATS_CODEC_NAME, Lucene40DocValuesFormat.FLOATS_VERSION_START, Lucene40DocValuesFormat.FLOATS_VERSION_CURRENT); int valueSize = input.ReadInt32(); if (valueSize != 4) { throw new CorruptIndexException("invalid valueSize: " + valueSize); } int maxDoc = state.SegmentInfo.DocCount; int[] values = new int[maxDoc]; for (int i = 0; i < values.Length; i++) { values[i] = input.ReadInt32(); } ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values)); return(new NumericDocValuesAnonymousInnerClassHelper7(values)); }
private NumericDocValues LoadDoubleField(/*FieldInfo field, // LUCENENET: Never read */ IndexInput input) { CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.FLOATS_CODEC_NAME, Lucene40DocValuesFormat.FLOATS_VERSION_START, Lucene40DocValuesFormat.FLOATS_VERSION_CURRENT); int valueSize = input.ReadInt32(); if (valueSize != 8) { throw new CorruptIndexException("invalid valueSize: " + valueSize); } int maxDoc = state.SegmentInfo.DocCount; long[] values = new long[maxDoc]; for (int i = 0; i < values.Length; i++) { values[i] = input.ReadInt64(); } ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values)); return(new NumericDocValuesAnonymousClass8(values)); }
public override void Init(IndexInput termsIn) { _version = CodecUtil.CheckHeader(termsIn, PulsingPostingsWriter.CODEC, PulsingPostingsWriter.VERSION_START, PulsingPostingsWriter.VERSION_CURRENT); _maxPositions = termsIn.ReadVInt(); _wrappedPostingsReader.Init(termsIn); if (_wrappedPostingsReader is PulsingPostingsReader || _version < PulsingPostingsWriter.VERSION_META_ARRAY) { _fields = null; } else { _fields = new SortedDictionary <int, int>(); var summaryFileName = IndexFileNames.SegmentFileName(_segmentState.SegmentInfo.Name, _segmentState.SegmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION); IndexInput input = null; try { input = _segmentState.Directory.OpenInput(summaryFileName, _segmentState.Context); CodecUtil.CheckHeader(input, PulsingPostingsWriter.CODEC, _version, PulsingPostingsWriter.VERSION_CURRENT); var numField = input.ReadVInt(); for (var i = 0; i < numField; i++) { var fieldNum = input.ReadVInt(); var longsSize = input.ReadVInt(); _fields.Add(fieldNum, longsSize); } } finally { IOUtils.CloseWhileHandlingException(input); } } }
private SortedDocValues LoadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index) { CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT); long maxAddress = index.ReadLong(); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, maxAddress); PagedBytes.Reader bytesReader = bytes.Freeze(true); PackedInts.Reader addressReader = PackedInts.GetReader(index); PackedInts.Reader ordsReader = PackedInts.GetReader(index); int valueCount = addressReader.Size() - 1; RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + addressReader.RamBytesUsed() + ordsReader.RamBytesUsed()); return(CorrectBuggyOrds(new SortedDocValuesAnonymousInnerClassHelper2(this, bytesReader, addressReader, ordsReader, valueCount))); }
private SegmentInfo ReadUpgradedSegmentInfo(string name, Directory dir, IndexInput input) { CodecUtil.CheckHeader(input, Lucene3xSegmentInfoFormat.UPGRADED_SI_CODEC_NAME, Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_START, Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_CURRENT); string version = input.ReadString(); int docCount = input.ReadInt(); IDictionary <string, string> attributes = input.ReadStringStringMap(); bool isCompoundFile = input.ReadByte() == SegmentInfo.YES; IDictionary <string, string> diagnostics = input.ReadStringStringMap(); ISet <string> files = input.ReadStringSet(); SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, null, diagnostics, CollectionsHelper.UnmodifiableMap(attributes)); info.Files = files; return(info); }
public override SegmentInfo Read(Directory dir, string segment, IOContext context) { string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene40SegmentInfoFormat.SI_EXTENSION); IndexInput input = dir.OpenInput(fileName, context); bool success = false; try { CodecUtil.CheckHeader(input, Lucene40SegmentInfoFormat.CODEC_NAME, Lucene40SegmentInfoFormat.VERSION_START, Lucene40SegmentInfoFormat.VERSION_CURRENT); string version = input.ReadString(); int docCount = input.ReadInt(); if (docCount < 0) { throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")"); } bool isCompoundFile = input.ReadByte() == SegmentInfo.YES; IDictionary <string, string> diagnostics = input.ReadStringStringMap(); input.ReadStringStringMap(); // read deprecated attributes ISet <string> files = input.ReadStringSet(); CodecUtil.CheckEOF(input); SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics); si.Files = files; success = true; return(si); } finally { if (!success) { IOUtils.CloseWhileHandlingException(input); } else { input.Dispose(); } } }
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET: //ORIGINAL LINE: @Override public codecs.FieldsProducer fieldsProducer(index.SegmentReadState state) throws java.io.IOException public override FieldsProducer fieldsProducer(SegmentReadState state) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final String fileName = index.IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); string fileName = IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final store.ChecksumIndexInput in = state.directory.openChecksumInput(fileName, store.IOContext.READONCE); ChecksumIndexInput @in = state.directory.openChecksumInput(fileName, IOContext.READONCE); //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final java.util.SortedMap<String,TermsReader> fields = new java.util.TreeMap<>(); SortedMap <string, TermsReader> fields = new SortedDictionary <string, TermsReader>(); try { CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT); while (true) { //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final int termCount = in.readVInt(); int termCount = @in.readVInt(); if (termCount == 0) { break; } //JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final': //ORIGINAL LINE: final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); TermsReader termsReader = new TermsReader(state.fieldInfos, @in, termCount); // System.out.println("load field=" + termsReader.field.name); fields.put(termsReader.field.name, termsReader); } CodecUtil.CheckFooter(@in); } finally { @in.close(); } return(new FieldsProducerAnonymousInnerClassHelper(this, fields)); }
public override bool Load(DataInput input) { CodecUtil.CheckHeader(input, CODEC_NAME, VERSION_START, VERSION_START); count = input.ReadVInt64(); var separatorOrig = (sbyte)input.ReadByte(); if (separatorOrig != separator) { throw new InvalidOperationException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig); } int gramsOrig = input.ReadVInt32(); if (gramsOrig != grams) { throw new InvalidOperationException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig); } totTokens = input.ReadVInt64(); fst = new FST <long?>(input, PositiveInt32Outputs.Singleton); return(true); }
private BinaryDocValues LoadBytesFixedDeref(FieldInfo field) { string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name + "_" + Convert.ToString(field.Number, CultureInfo.InvariantCulture), segmentSuffix, "dat"); string indexName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name + "_" + Convert.ToString(field.Number, CultureInfo.InvariantCulture), segmentSuffix, "idx"); IndexInput data = null; IndexInput index = null; bool success = false; try { data = dir.OpenInput(dataName, state.Context); CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT); index = dir.OpenInput(indexName, state.Context); CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT); int fixedLength = data.ReadInt32(); int valueCount = index.ReadInt32(); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, fixedLength * (long)valueCount); PagedBytes.Reader bytesReader = bytes.Freeze(true); PackedInt32s.Reader reader = PackedInt32s.GetReader(index); CodecUtil.CheckEOF(data); CodecUtil.CheckEOF(index); ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed()); success = true; return(new BinaryDocValuesAnonymousClass3(fixedLength, bytesReader, reader)); } finally { if (success) { IOUtils.Dispose(data, index); } else { IOUtils.DisposeWhileHandlingException(data, index); } } }
// private String segment; /// <summary> /// Sole constructor. </summary> public Lucene40PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix) { bool success = false; IndexInput freqIn = null; IndexInput proxIn = null; try { freqIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), ioContext); CodecUtil.CheckHeader(freqIn, FRQ_CODEC, VERSION_START, VERSION_CURRENT); // TODO: hasProx should (somehow!) become codec private, // but it's tricky because 1) FIS.hasProx is global (it // could be all fields that have prox are written by a // different codec), 2) the field may have had prox in // the past but all docs w/ that field were deleted. // Really we'd need to init prxOut lazily on write, and // then somewhere record that we actually wrote it so we // know whether to open on read: if (fieldInfos.HasProx) { proxIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), ioContext); CodecUtil.CheckHeader(proxIn, PRX_CODEC, VERSION_START, VERSION_CURRENT); } else { proxIn = null; } this.freqIn = freqIn; this.proxIn = proxIn; success = true; } finally { if (!success) { IOUtils.CloseWhileHandlingException(freqIn, proxIn); } } }
private BinaryDocValues LoadBytesVarDeref(FieldInfo field) { string dataName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "dat"); string indexName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "idx"); IndexInput data = null; IndexInput index = null; bool success = false; try { data = Dir.OpenInput(dataName, State.Context); CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); index = Dir.OpenInput(indexName, State.Context); CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT); long totalBytes = index.ReadLong(); PagedBytes bytes = new PagedBytes(16); bytes.Copy(data, totalBytes); PagedBytes.Reader bytesReader = bytes.Freeze(true); PackedInts.Reader reader = PackedInts.GetReader(index); CodecUtil.CheckEOF(data); CodecUtil.CheckEOF(index); RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed()); success = true; return(new BinaryDocValuesAnonymousInnerClassHelper4(this, bytesReader, reader)); } finally { if (success) { IOUtils.Close(data, index); } else { IOUtils.CloseWhileHandlingException(data, index); } } }
public override FieldsProducer FieldsProducer(SegmentReadState readState) { // Load our ID: string idFileName = IndexFileNames.SegmentFileName(readState.SegmentInfo.Name, readState.SegmentSuffix, ID_EXTENSION); IndexInput @in = readState.Directory.OpenInput(idFileName, readState.Context); bool success = false; int id; try { CodecUtil.CheckHeader(@in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST); id = @in.ReadVInt32(); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(@in); } else { IOUtils.Dispose(@in); } } UninterruptableMonitor.Enter(state); try { return(state.TryGetValue(id, out RAMPostings value) ? value : null); } finally { UninterruptableMonitor.Exit(state); } }
private int ReadHeader(IndexInput @in) { return(CodecUtil.CheckHeader(@in, FSTTermsWriter.TERMS_CODEC_NAME, FSTTermsWriter.TERMS_VERSION_START, FSTTermsWriter.TERMS_VERSION_CURRENT)); }
/// <summary> /// Helper method that reads CFS entries from an input stream </summary> private static IDictionary <string, FileEntry> ReadEntries(IndexInputSlicer handle, Directory dir, string name) { IOException priorE = null; IndexInput stream = null; ChecksumIndexInput entriesStream = null; // read the first VInt. If it is negative, it's the version number // otherwise it's the count (pre-3.1 indexes) try { IDictionary <string, FileEntry> mapping; #pragma warning disable 612, 618 stream = handle.OpenFullSlice(); #pragma warning restore 612, 618 int firstInt = stream.ReadVInt32(); // impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible // and separate norms/etc are outside of cfs. if (firstInt == CODEC_MAGIC_BYTE1) { sbyte secondByte = (sbyte)stream.ReadByte(); sbyte thirdByte = (sbyte)stream.ReadByte(); sbyte fourthByte = (sbyte)stream.ReadByte(); if (secondByte != CODEC_MAGIC_BYTE2 || thirdByte != CODEC_MAGIC_BYTE3 || fourthByte != CODEC_MAGIC_BYTE4) { throw new CorruptIndexException("Illegal/impossible header for CFS file: " + secondByte + "," + thirdByte + "," + fourthByte); } int version = CodecUtil.CheckHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT); string entriesFileName = IndexFileNames.SegmentFileName( IndexFileNames.StripExtension(name), "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); entriesStream = dir.OpenChecksumInput(entriesFileName, IOContext.READ_ONCE); CodecUtil.CheckHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT); int numEntries = entriesStream.ReadVInt32(); mapping = new Dictionary <string, FileEntry>(numEntries); for (int i = 0; i < numEntries; i++) { FileEntry fileEntry = new FileEntry(); string id = entriesStream.ReadString(); FileEntry previous = mapping.Put(id, fileEntry); if (previous != null) { throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS: " + entriesStream); } fileEntry.Offset = entriesStream.ReadInt64(); fileEntry.Length = entriesStream.ReadInt64(); } if (version >= CompoundFileWriter.VERSION_CHECKSUM) { CodecUtil.CheckFooter(entriesStream); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(entriesStream); #pragma warning restore 612, 618 } } else { // TODO remove once 3.x is not supported anymore mapping = ReadLegacyEntries(stream, firstInt); } return(mapping); } catch (IOException ioe) { priorE = ioe; } finally { IOUtils.DisposeWhileHandlingException(priorE, stream, entriesStream); } // this is needed until Java 7's real try-with-resources: throw new InvalidOperationException("impossible to get here"); }
internal Lucene42DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension) { maxDoc = state.SegmentInfo.DocCount; string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension); // read in the entries from the metadata file. ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context); bool success = false; ramBytesUsed = new AtomicInt64(RamUsageEstimator.ShallowSizeOfInstance(this.GetType())); try { version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT); numerics = new Dictionary <int, NumericEntry>(); binaries = new Dictionary <int, BinaryEntry>(); fsts = new Dictionary <int, FSTEntry>(); ReadFields(@in, state.FieldInfos); if (version >= VERSION_CHECKSUM) { CodecUtil.CheckFooter(@in); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(@in); #pragma warning restore 612, 618 } success = true; } finally { if (success) { IOUtils.Dispose(@in); } else { IOUtils.DisposeWhileHandlingException(@in); } } success = false; try { string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension); data = state.Directory.OpenInput(dataName, state.Context); int version2 = CodecUtil.CheckHeader(data, dataCodec, VERSION_START, VERSION_CURRENT); if (version != version2) { throw new CorruptIndexException("Format versions mismatch"); } success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(this.data); } } }
protected BinaryDictionary() { int[] targetMapOffsets = null, targetMap = null; string[] posDict = null; string[] inflFormDict = null; string[] inflTypeDict = null; ByteBuffer buffer; // LUCENENET: IDE0059: Remove unnecessary value assignment using (Stream mapIS = GetResource(TARGETMAP_FILENAME_SUFFIX)) { DataInput @in = new InputStreamDataInput(mapIS); CodecUtil.CheckHeader(@in, TARGETMAP_HEADER, VERSION, VERSION); targetMap = new int[@in.ReadVInt32()]; targetMapOffsets = new int[@in.ReadVInt32()]; int accum = 0, sourceId = 0; for (int ofs = 0; ofs < targetMap.Length; ofs++) { int val = @in.ReadVInt32(); if ((val & 0x01) != 0) { targetMapOffsets[sourceId] = ofs; sourceId++; } accum += val.TripleShift(1); targetMap[ofs] = accum; } if (sourceId + 1 != targetMapOffsets.Length) { throw new IOException("targetMap file format broken"); } targetMapOffsets[sourceId] = targetMap.Length; } using (Stream posIS = GetResource(POSDICT_FILENAME_SUFFIX)) { DataInput @in = new InputStreamDataInput(posIS); CodecUtil.CheckHeader(@in, POSDICT_HEADER, VERSION, VERSION); int posSize = @in.ReadVInt32(); posDict = new string[posSize]; inflTypeDict = new string[posSize]; inflFormDict = new string[posSize]; for (int j = 0; j < posSize; j++) { posDict[j] = @in.ReadString(); inflTypeDict[j] = @in.ReadString(); inflFormDict[j] = @in.ReadString(); // this is how we encode null inflections if (inflTypeDict[j].Length == 0) { inflTypeDict[j] = null; } if (inflFormDict[j].Length == 0) { inflFormDict[j] = null; } } } ByteBuffer tmpBuffer; using (Stream dictIS = GetResource(DICT_FILENAME_SUFFIX)) { // no buffering here, as we load in one large buffer DataInput @in = new InputStreamDataInput(dictIS); CodecUtil.CheckHeader(@in, DICT_HEADER, VERSION, VERSION); int size = @in.ReadVInt32(); tmpBuffer = ByteBuffer.Allocate(size); // AllocateDirect..? int read = dictIS.Read(tmpBuffer.Array, 0, size); if (read != size) { throw EOFException.Create("Cannot read whole dictionary"); } } buffer = tmpBuffer.AsReadOnlyBuffer(); this.targetMap = targetMap; this.targetMapOffsets = targetMapOffsets; this.posDict = posDict; this.inflTypeDict = inflTypeDict; this.inflFormDict = inflFormDict; this.buffer = buffer; }
/// <summary> /// Sole constructor. </summary> public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode) { this.compressionMode = compressionMode; string segment = si.Name; bool success = false; fieldInfos = fn; numDocs = si.DocCount; ChecksumIndexInput indexStream = null; try { // Load the index into memory string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION); indexStream = d.OpenChecksumInput(indexStreamFN, context); string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX; version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT); if (Debugging.AssertsEnabled) { Debugging.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer()); } indexReader = new CompressingStoredFieldsIndexReader(indexStream, si); if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM) { indexStream.ReadVInt64(); // the end of the data file CodecUtil.CheckFooter(indexStream); } else { #pragma warning disable 612, 618 CodecUtil.CheckEOF(indexStream); #pragma warning restore 612, 618 } indexStream.Dispose(); indexStream = null; // Open the data file and read metadata string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION); vectorsStream = d.OpenInput(vectorsStreamFN, context); string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT; int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT); if (version != version2) { throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2); } if (Debugging.AssertsEnabled) { Debugging.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.GetFilePointer()); } packedIntsVersion = vectorsStream.ReadVInt32(); chunkSize = vectorsStream.ReadVInt32(); decompressor = compressionMode.NewDecompressor(); this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0); success = true; } finally { if (!success) { IOUtils.DisposeWhileHandlingException(this, indexStream); } } }
/// <summary> /// Reads the snapshots information from the given <seealso cref="Directory"/>. this /// method can be used if the snapshots information is needed, however you /// cannot instantiate the deletion policy (because e.g., some other process /// keeps a lock on the snapshots directory). /// </summary> private void LoadPriorSnapshots() { lock (this) { long genLoaded = -1; IOException ioe = null; IList <string> snapshotFiles = new List <string>(); foreach (string file in dir.ListAll()) { if (file.StartsWith(SNAPSHOTS_PREFIX, StringComparison.Ordinal)) { long gen = Convert.ToInt64(file.Substring(SNAPSHOTS_PREFIX.Length)); if (genLoaded == -1 || gen > genLoaded) { snapshotFiles.Add(file); IDictionary <long, int> m = new Dictionary <long, int>(); IndexInput @in = dir.OpenInput(file, IOContext.DEFAULT); try { CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_START); int count = @in.ReadVInt32(); for (int i = 0; i < count; i++) { long commitGen = @in.ReadVInt64(); int refCount = @in.ReadVInt32(); m[commitGen] = refCount; } } catch (IOException ioe2) { // Save first exception & throw in the end if (ioe == null) { ioe = ioe2; } } finally { @in.Dispose(); } genLoaded = gen; m_refCounts.Clear(); m_refCounts.PutAll(m); } } } if (genLoaded == -1) { // Nothing was loaded... if (ioe != null) { // ... not for lack of trying: throw ioe; } } else { if (snapshotFiles.Count > 1) { // Remove any broken / old snapshot files: string curFileName = SNAPSHOTS_PREFIX + genLoaded; foreach (string file in snapshotFiles) { if (!curFileName.Equals(file, StringComparison.Ordinal)) { dir.DeleteFile(file); } } } nextWriteGen = 1 + genLoaded; } } }