/// <summary>
/// Helper method that reads CFS entries from an input stream </summary>
private static IDictionary <string, FileEntry> ReadEntries(IndexInputSlicer handle, Directory dir, string name)
{
IOException priorE = null;
IndexInput stream = null;
ChecksumIndexInput entriesStream = null;
// read the first VInt. If it is negative, it's the version number
// otherwise it's the count (pre-3.1 indexes)
try
{
IDictionary <string, FileEntry> mapping;
#pragma warning disable 612, 618
stream = handle.OpenFullSlice();
#pragma warning restore 612, 618
int firstInt = stream.ReadVInt32();
// impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible
// and separate norms/etc are outside of cfs.
if (firstInt == CODEC_MAGIC_BYTE1)
{
sbyte secondByte = (sbyte)stream.ReadByte();
sbyte thirdByte = (sbyte)stream.ReadByte();
sbyte fourthByte = (sbyte)stream.ReadByte();
if (secondByte != CODEC_MAGIC_BYTE2 || thirdByte != CODEC_MAGIC_BYTE3 || fourthByte != CODEC_MAGIC_BYTE4)
{
throw new CorruptIndexException("Illegal/impossible header for CFS file: " + secondByte + "," + thirdByte + "," + fourthByte);
}
int version = CodecUtil.CheckHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT);
string entriesFileName = IndexFileNames.SegmentFileName(
IndexFileNames.StripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
entriesStream = dir.OpenChecksumInput(entriesFileName, IOContext.READ_ONCE);
CodecUtil.CheckHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT);
int numEntries = entriesStream.ReadVInt32();
mapping = new Dictionary <string, FileEntry>(numEntries);
for (int i = 0; i < numEntries; i++)
{
FileEntry fileEntry = new FileEntry();
string id = entriesStream.ReadString();
FileEntry previous = mapping.Put(id, fileEntry);
if (previous != null)
{
throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS: " + entriesStream);
}
fileEntry.Offset = entriesStream.ReadInt64();
fileEntry.Length = entriesStream.ReadInt64();
}
if (version >= CompoundFileWriter.VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(entriesStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(entriesStream);
#pragma warning restore 612, 618
}
}
else
{
// TODO remove once 3.x is not supported anymore
mapping = ReadLegacyEntries(stream, firstInt);
}
return(mapping);
}
catch (IOException ioe)
{
priorE = ioe;
}
finally
{
IOUtils.DisposeWhileHandlingException(priorE, stream, entriesStream);
}
// this is needed until Java 7's real try-with-resources:
throw new InvalidOperationException("impossible to get here");
}
internal MemoryDocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension,
string metaCodec, string metaExtension)
{
maxDoc = state.SegmentInfo.DocCount;
var metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
// read in the entries from the metadata file.
var @in = state.Directory.OpenChecksumInput(metaName, state.Context);
bool success = false;
try
{
version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
numerics = new Dictionary <int?, NumericEntry>();
binaries = new Dictionary <int?, BinaryEntry>();
fsts = new Dictionary <int?, FSTEntry>();
ReadFields(@in /*, state.FieldInfos // LUCENENET: Not referenced */);
if (version >= VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(@in);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(@in);
#pragma warning restore 612, 618
}
ramBytesUsed = new AtomicInt64(RamUsageEstimator.ShallowSizeOfInstance(this.GetType()));
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(@in);
}
else
{
IOUtils.DisposeWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix,
dataExtension);
data = state.Directory.OpenInput(dataName, state.Context);
int version2 = CodecUtil.CheckHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
if (version != version2)
{
throw new CorruptIndexException("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this.data);
}
}
}
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
ChecksumIndexInput input = directory.OpenChecksumInput(fileName, context);
bool success = false;
try
{
int codecVersion = CodecUtil.CheckHeader(input, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_START, Lucene46FieldInfosFormat.FORMAT_CURRENT);
int size = input.ReadVInt32(); //read in the size
FieldInfo[] infos = new FieldInfo[size];
for (int i = 0; i < size; i++)
{
string name = input.ReadString();
int fieldNumber = input.ReadVInt32();
byte bits = input.ReadByte();
bool isIndexed = (bits & Lucene46FieldInfosFormat.IS_INDEXED) != 0;
bool storeTermVector = (bits & Lucene46FieldInfosFormat.STORE_TERMVECTOR) != 0;
bool omitNorms = (bits & Lucene46FieldInfosFormat.OMIT_NORMS) != 0;
bool storePayloads = (bits & Lucene46FieldInfosFormat.STORE_PAYLOADS) != 0;
IndexOptions indexOptions;
if (!isIndexed)
{
indexOptions = IndexOptions.NONE;
}
else if ((bits & Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_ONLY;
}
else if ((bits & Lucene46FieldInfosFormat.OMIT_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS;
}
else if ((bits & Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
else
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
// DV Types are packed in one byte
byte val = input.ReadByte();
DocValuesType docValuesType = GetDocValuesType(input, (sbyte)(val & 0x0F));
DocValuesType normsType = GetDocValuesType(input, (sbyte)(((int)((uint)val >> 4)) & 0x0F));
long dvGen = input.ReadInt64();
IDictionary <string, string> attributes = input.ReadStringStringMap();
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, normsType, Collections.UnmodifiableMap(attributes));
infos[i].DocValuesGen = dvGen;
}
if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM)
{
CodecUtil.CheckFooter(input);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(input);
#pragma warning restore 612, 618
}
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
return(fieldInfos);
}
finally
{
if (success)
{
input.Dispose();
}
else
{
IOUtils.DisposeWhileHandlingException(input);
}
}
}
public override NumericDocValues GetNumeric(FieldInfo field)
{
lock (this)
{
if (!numericInstances.TryGetValue(field.Number, out NumericDocValues instance))
{
string fileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name + "_" + Convert.ToString(field.Number, CultureInfo.InvariantCulture), segmentSuffix, "dat");
IndexInput input = dir.OpenInput(fileName, state.Context);
bool success = false;
try
{
var type = field.GetAttribute(legacyKey).ToLegacyDocValuesType();
//switch (Enum.Parse(typeof(LegacyDocValuesType), field.GetAttribute(LegacyKey)))
//{
if (type == LegacyDocValuesType.VAR_INTS)
{
instance = LoadVarInt32sField(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_8)
{
instance = LoadByteField(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_16)
{
instance = LoadInt16Field(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_32)
{
instance = LoadInt32Field(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_64)
{
instance = LoadInt64Field(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FLOAT_32)
{
instance = LoadSingleField(/* field, // LUCENENET: Never read */ input);
}
else if (type == LegacyDocValuesType.FLOAT_64)
{
instance = LoadDoubleField(/* field, // LUCENENET: Never read */ input);
}
else
{
throw new InvalidOperationException();
}
CodecUtil.CheckEOF(input);
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(input);
}
else
{
IOUtils.DisposeWhileHandlingException(input);
}
}
numericInstances[field.Number] = instance;
}
return(instance);
}
}
//static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
{
string termsIndexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
string termsBlockFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
this.postingsReader = postingsReader;
ChecksumIndexInput indexIn = null;
IndexInput blockIn = null;
bool success = false;
try
{
indexIn = state.Directory.OpenChecksumInput(termsIndexFileName, state.Context);
blockIn = state.Directory.OpenInput(termsBlockFileName, state.Context);
version = ReadHeader(indexIn);
ReadHeader(blockIn);
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
{
CodecUtil.ChecksumEntireFile(blockIn);
}
this.postingsReader.Init(blockIn);
SeekDir(blockIn);
FieldInfos fieldInfos = state.FieldInfos;
int numFields = blockIn.ReadVInt32();
for (int i = 0; i < numFields; i++)
{
FieldInfo fieldInfo = fieldInfos.FieldInfo(blockIn.ReadVInt32());
bool hasFreq = fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY;
long numTerms = blockIn.ReadVInt64();
long sumTotalTermFreq = hasFreq ? blockIn.ReadVInt64() : -1;
long sumDocFreq = blockIn.ReadVInt64();
int docCount = blockIn.ReadVInt32();
int longsSize = blockIn.ReadVInt32();
var index = new FST <long?>(indexIn, PositiveInt32Outputs.Singleton);
var current = new TermsReader(this, fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
TermsReader previous;
// LUCENENET NOTE: This simulates a put operation in Java,
// getting the prior value first before setting it.
fields.TryGetValue(fieldInfo.Name, out previous);
fields[fieldInfo.Name] = current;
CheckFieldSummary(state.SegmentInfo, indexIn, blockIn, current, previous);
}
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(indexIn);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexIn);
#pragma warning restore 612, 618
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(indexIn, blockIn);
}
else
{
IOUtils.DisposeWhileHandlingException(indexIn, blockIn);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
// Load the index into memory
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
}
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
long maxPointer = -1;
if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
maxPointer = indexStream.ReadVInt64();
CodecUtil.CheckFooter(indexStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexStream);
#pragma warning restore 612, 618
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
fieldsStream = d.OpenInput(fieldsStreamFN, context);
if (version >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
if (maxPointer + CodecUtil.FooterLength() != fieldsStream.Length)
{
throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.Length);
}
}
else
{
maxPointer = fieldsStream.Length;
}
this.maxPointer = maxPointer;
string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
int fieldsVersion = CodecUtil.CheckHeader(fieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (version != fieldsVersion)
{
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameDat) == fieldsStream.GetFilePointer());
}
if (version >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
{
chunkSize = fieldsStream.ReadVInt32();
}
else
{
chunkSize = -1;
}
packedIntsVersion = fieldsStream.ReadVInt32();
decompressor = compressionMode.NewDecompressor();
this.bytes = new BytesRef();
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this, indexStream);
}
}
}
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene40FieldInfosFormat.FIELD_INFOS_EXTENSION);
IndexInput input = directory.OpenInput(fileName, iocontext);
bool success = false;
try
{
CodecUtil.CheckHeader(input, Lucene40FieldInfosFormat.CODEC_NAME, Lucene40FieldInfosFormat.FORMAT_START, Lucene40FieldInfosFormat.FORMAT_CURRENT);
int size = input.ReadVInt32(); //read in the size
FieldInfo[] infos = new FieldInfo[size];
for (int i = 0; i < size; i++)
{
string name = input.ReadString();
int fieldNumber = input.ReadVInt32();
byte bits = input.ReadByte();
bool isIndexed = (bits & Lucene40FieldInfosFormat.IS_INDEXED) != 0;
bool storeTermVector = (bits & Lucene40FieldInfosFormat.STORE_TERMVECTOR) != 0;
bool omitNorms = (bits & Lucene40FieldInfosFormat.OMIT_NORMS) != 0;
bool storePayloads = (bits & Lucene40FieldInfosFormat.STORE_PAYLOADS) != 0;
IndexOptions indexOptions;
if (!isIndexed)
{
indexOptions = IndexOptions.NONE;
}
else if ((bits & Lucene40FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_ONLY;
}
else if ((bits & Lucene40FieldInfosFormat.OMIT_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS;
}
else if ((bits & Lucene40FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
else
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
// LUCENE-3027: past indices were able to write
// storePayloads=true when omitTFAP is also true,
// which is invalid. We correct that, here:
if (isIndexed && indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0)
{
storePayloads = false;
}
// DV Types are packed in one byte
byte val = input.ReadByte();
LegacyDocValuesType oldValuesType = GetDocValuesType((sbyte)(val & 0x0F));
LegacyDocValuesType oldNormsType = GetDocValuesType((sbyte)(((int)((uint)val >> 4)) & 0x0F));
IDictionary <string, string> attributes = input.ReadStringStringMap();
if (oldValuesType.GetMapping() != DocValuesType.NONE)
{
attributes[LEGACY_DV_TYPE_KEY] = oldValuesType.ToString();
}
if (oldNormsType.GetMapping() != DocValuesType.NONE)
{
if (oldNormsType.GetMapping() != DocValuesType.NUMERIC)
{
throw new CorruptIndexException("invalid norm type: " + oldNormsType + " (resource=" + input + ")");
}
attributes[LEGACY_NORM_TYPE_KEY] = oldNormsType.ToString();
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, oldValuesType.GetMapping(), oldNormsType.GetMapping(), attributes);
}
CodecUtil.CheckEOF(input);
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
return(fieldInfos);
}
finally
{
if (success)
{
input.Dispose();
}
else
{
IOUtils.DisposeWhileHandlingException(input);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.CompressionMode_Renamed = compressionMode;
string segment = si.Name;
bool success = false;
FieldInfos = fn;
NumDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
string fieldsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, Lucene40StoredFieldsWriter.FIELDS_EXTENSION);
// Load the index into memory
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingStoredFieldsWriter.CODEC_SFX_IDX;
Version_Renamed = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
Debug.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.FilePointer);
IndexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
long maxPointer = -1;
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
maxPointer = indexStream.ReadVLong();
CodecUtil.CheckFooter(indexStream);
}
else
{
CodecUtil.CheckEOF(indexStream);
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
FieldsStream = d.OpenInput(fieldsStreamFN, context);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_CHECKSUM)
{
if (maxPointer + CodecUtil.FooterLength() != FieldsStream.Length())
{
throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + FieldsStream.Length());
}
}
else
{
maxPointer = FieldsStream.Length();
}
this.MaxPointer = maxPointer;
string codecNameDat = formatName + CompressingStoredFieldsWriter.CODEC_SFX_DAT;
int fieldsVersion = CodecUtil.CheckHeader(FieldsStream, codecNameDat, CompressingStoredFieldsWriter.VERSION_START, CompressingStoredFieldsWriter.VERSION_CURRENT);
if (Version_Renamed != fieldsVersion)
{
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + Version_Renamed + " != " + fieldsVersion);
}
Debug.Assert(CodecUtil.HeaderLength(codecNameDat) == FieldsStream.FilePointer);
if (Version_Renamed >= CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS)
{
ChunkSize_Renamed = FieldsStream.ReadVInt();
}
else
{
ChunkSize_Renamed = -1;
}
PackedIntsVersion = FieldsStream.ReadVInt();
Decompressor = compressionMode.NewDecompressor();
this.Bytes = new BytesRef();
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this, indexStream);
}
}
}
internal Lucene42DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
{
MaxDoc = state.SegmentInfo.DocCount;
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context);
bool success = false;
RamBytesUsed_Renamed = new AtomicLong(RamUsageEstimator.ShallowSizeOfInstance(this.GetType()));
try
{
Version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
Numerics = new Dictionary <int, NumericEntry>();
Binaries = new Dictionary <int, BinaryEntry>();
Fsts = new Dictionary <int, FSTEntry>();
ReadFields(@in, state.FieldInfos);
if (Version >= VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(@in);
}
else
{
CodecUtil.CheckEOF(@in);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(@in);
}
else
{
IOUtils.CloseWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
Data = state.Directory.OpenInput(dataName, state.Context);
int version2 = CodecUtil.CheckHeader(Data, dataCodec, VERSION_START, VERSION_CURRENT);
if (Version != version2)
{
throw new CorruptIndexException("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(this.Data);
}
}
}
/// <summary>
/// Sole constructor. </summary>
public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
// Load the index into memory
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION);
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
}
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM)
{
indexStream.ReadVInt64(); // the end of the data file
CodecUtil.CheckFooter(indexStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexStream);
#pragma warning restore 612, 618
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION);
vectorsStream = d.OpenInput(vectorsStreamFN, context);
string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT;
int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (version != version2)
{
throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.GetFilePointer());
}
packedIntsVersion = vectorsStream.ReadVInt32();
chunkSize = vectorsStream.ReadVInt32();
decompressor = compressionMode.NewDecompressor();
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this, indexStream);
}
}
}
public override NumericDocValues GetNumeric(FieldInfo field)
{
lock (this)
{
NumericDocValues instance;
if (!NumericInstances.TryGetValue(field.Number, out instance))
{
string fileName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "dat");
IndexInput input = Dir.OpenInput(fileName, State.Context);
bool success = false;
try
{
var type = LegacyDocValuesType.ValueOf(field.GetAttribute(LegacyKey));
//switch (Enum.Parse(typeof(LegacyDocValuesType), field.GetAttribute(LegacyKey)))
//{
if (type == LegacyDocValuesType.VAR_INTS)
{
instance = LoadVarIntsField(field, input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_8)
{
instance = LoadByteField(field, input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_16)
{
instance = LoadShortField(field, input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_32)
{
instance = LoadIntField(field, input);
}
else if (type == LegacyDocValuesType.FIXED_INTS_64)
{
instance = LoadLongField(field, input);
}
else if (type == LegacyDocValuesType.FLOAT_32)
{
instance = LoadFloatField(field, input);
}
else if (type == LegacyDocValuesType.FLOAT_64)
{
instance = LoadDoubleField(field, input);
}
else
{
throw new InvalidOperationException();
}
CodecUtil.CheckEOF(input);
success = true;
}
finally
{
if (success)
{
IOUtils.Close(input);
}
else
{
IOUtils.CloseWhileHandlingException(input);
}
}
NumericInstances[field.Number] = instance;
}
return(instance);
}
}
public override FieldInfos Read(Directory directory, string segmentName, string segmentSuffix, IOContext iocontext)
{
string fileName = IndexFileNames.SegmentFileName(segmentName, "", Lucene42FieldInfosFormat.EXTENSION);
IndexInput input = directory.OpenInput(fileName, iocontext);
bool success = false;
try
{
CodecUtil.CheckHeader(input, Lucene42FieldInfosFormat.CODEC_NAME,
Lucene42FieldInfosFormat.FORMAT_START,
Lucene42FieldInfosFormat.FORMAT_CURRENT);
int size = input.ReadVInt32(); //read in the size
FieldInfo[] infos = new FieldInfo[size];
for (int i = 0; i < size; i++)
{
string name = input.ReadString();
int fieldNumber = input.ReadVInt32();
sbyte bits = (sbyte)input.ReadByte();
bool isIndexed = (bits & Lucene42FieldInfosFormat.IS_INDEXED) != 0;
bool storeTermVector = (bits & Lucene42FieldInfosFormat.STORE_TERMVECTOR) != 0;
bool omitNorms = (bits & Lucene42FieldInfosFormat.OMIT_NORMS) != 0;
bool storePayloads = (bits & Lucene42FieldInfosFormat.STORE_PAYLOADS) != 0;
IndexOptions indexOptions;
if (!isIndexed)
{
indexOptions = IndexOptions.NONE;
}
else if ((bits & Lucene42FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_ONLY;
}
else if ((bits & Lucene42FieldInfosFormat.OMIT_POSITIONS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS;
}
else if ((bits & Lucene42FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0)
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
else
{
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
// DV Types are packed in one byte
sbyte val = (sbyte)input.ReadByte();
DocValuesType docValuesType = GetDocValuesType(input, (byte)(val & 0x0F));
DocValuesType normsType = GetDocValuesType(input, (byte)((val.TripleShift(4)) & 0x0F));
IDictionary <string, string> attributes = input.ReadStringStringMap();
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, normsType, attributes.AsReadOnly());
}
CodecUtil.CheckEOF(input);
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
return(fieldInfos);
}
finally
{
if (success)
{
input.Dispose();
}
else
{
IOUtils.DisposeWhileHandlingException(input);
}
}
}
//static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
{
string termsIndexFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTOrdTermsWriter.TERMS_INDEX_EXTENSION);
string termsBlockFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
this.postingsReader = postingsReader;
ChecksumIndexInput indexIn = null;
IndexInput blockIn = null;
bool success = false;
try
{
indexIn = state.Directory.OpenChecksumInput(termsIndexFileName, state.Context);
blockIn = state.Directory.OpenInput(termsBlockFileName, state.Context);
version = ReadHeader(indexIn);
ReadHeader(blockIn);
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
{
CodecUtil.ChecksumEntireFile(blockIn);
}
this.postingsReader.Init(blockIn);
SeekDir(blockIn);
FieldInfos fieldInfos = state.FieldInfos;
int numFields = blockIn.ReadVInt();
for (int i = 0; i < numFields; i++)
{
FieldInfo fieldInfo = fieldInfos.FieldInfo(blockIn.ReadVInt());
bool hasFreq = fieldInfo.FieldIndexOptions != FieldInfo.IndexOptions.DOCS_ONLY;
long numTerms = blockIn.ReadVLong();
long sumTotalTermFreq = hasFreq ? blockIn.ReadVLong() : -1;
long sumDocFreq = blockIn.ReadVLong();
int docCount = blockIn.ReadVInt();
int longsSize = blockIn.ReadVInt();
var index = new FST <long?>(indexIn, PositiveIntOutputs.Singleton);
var current = new TermsReader(this, fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
var previous = fields[fieldInfo.Name] = current;
CheckFieldSummary(state.SegmentInfo, indexIn, blockIn, current, previous);
}
if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(indexIn);
}
else
{
CodecUtil.CheckEOF(indexIn);
}
success = true;
}
finally
{
if (success)
{
IOUtils.Close(indexIn, blockIn);
}
else
{
IOUtils.CloseWhileHandlingException(indexIn, blockIn);
}
}
}