private int ReadHeader(IndexInput input)
{
int version = CodecUtil.CheckHeader(input, VariableGapTermsIndexWriter.CODEC_NAME,
VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT);
if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY)
{
dirOffset = input.ReadInt64();
}
return(version);
}
/// <summary>
/// Sole constructor. </summary>
public Lucene40TermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context)
{
string segment = si.Name;
int size = si.DocCount;
bool success = false;
try
{
string idxName = IndexFileNames.SegmentFileName(segment, "", VECTORS_INDEX_EXTENSION);
tvx = d.OpenInput(idxName, context);
int tvxVersion = CodecUtil.CheckHeader(tvx, CODEC_NAME_INDEX, VERSION_START, VERSION_CURRENT);
string fn = IndexFileNames.SegmentFileName(segment, "", VECTORS_DOCUMENTS_EXTENSION);
tvd = d.OpenInput(fn, context);
int tvdVersion = CodecUtil.CheckHeader(tvd, CODEC_NAME_DOCS, VERSION_START, VERSION_CURRENT);
fn = IndexFileNames.SegmentFileName(segment, "", VECTORS_FIELDS_EXTENSION);
tvf = d.OpenInput(fn, context);
int tvfVersion = CodecUtil.CheckHeader(tvf, CODEC_NAME_FIELDS, VERSION_START, VERSION_CURRENT);
Debug.Assert(HEADER_LENGTH_INDEX == tvx.GetFilePointer());
Debug.Assert(HEADER_LENGTH_DOCS == tvd.GetFilePointer());
Debug.Assert(HEADER_LENGTH_FIELDS == tvf.GetFilePointer());
Debug.Assert(tvxVersion == tvdVersion);
Debug.Assert(tvxVersion == tvfVersion);
numTotalDocs = (int)(tvx.Length - HEADER_LENGTH_INDEX >> 4);
this.size = numTotalDocs;
Debug.Assert(size == 0 || numTotalDocs == size);
this.fieldInfos = fieldInfos;
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
try
{
Dispose();
} // ensure we throw our original exception
#pragma warning disable 168
catch (Exception t)
#pragma warning restore 168
{
}
}
}
}
private int ReadHeader(DataInput input)
{
var version = CodecUtil.CheckHeader(input, FixedGapTermsIndexWriter.CODEC_NAME,
FixedGapTermsIndexWriter.VERSION_START, FixedGapTermsIndexWriter.VERSION_CURRENT);
if (version < FixedGapTermsIndexWriter.VERSION_APPEND_ONLY)
{
_dirOffset = input.ReadLong();
}
return(version);
}
private int ReadHeader(DataInput input)
{
int version = CodecUtil.CheckHeader(input, BlockTermsWriter.CODEC_NAME,
BlockTermsWriter.VERSION_START,
BlockTermsWriter.VERSION_CURRENT);
if (version < BlockTermsWriter.VERSION_APPEND_ONLY)
{
dirOffset = input.ReadInt64();
}
return(version);
}
public BloomFilteredFieldsProducer(BloomFilteringPostingsFormat outerInstance, SegmentReadState state)
{
this.outerInstance = outerInstance;
var bloomFileName = IndexFileNames.SegmentFileName(
state.SegmentInfo.Name, state.SegmentSuffix, BLOOM_EXTENSION);
ChecksumIndexInput bloomIn = null;
var success = false;
try
{
bloomIn = state.Directory.OpenChecksumInput(bloomFileName, state.Context);
var version = CodecUtil.CheckHeader(bloomIn, /*BLOOM_CODEC_NAME*/ outerInstance.Name, VERSION_START, VERSION_CURRENT);
// Load the hash function used in the BloomFilter
// hashFunction = HashFunction.forName(bloomIn.readString());
// Load the delegate postings format
var delegatePostingsFormat = ForName(bloomIn.ReadString());
_delegateFieldsProducer = delegatePostingsFormat
.FieldsProducer(state);
var numBlooms = bloomIn.ReadInt32();
for (var i = 0; i < numBlooms; i++)
{
var fieldNum = bloomIn.ReadInt32();
var bloom = FuzzySet.Deserialize(bloomIn);
var fieldInfo = state.FieldInfos.FieldInfo(fieldNum);
_bloomsByFieldName.Add(fieldInfo.Name, bloom);
}
if (version >= VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(bloomIn);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(bloomIn);
#pragma warning restore 612, 618
}
IOUtils.Dispose(bloomIn);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(bloomIn, _delegateFieldsProducer);
}
}
}
private CharacterDefinition()
{
using Stream @is = BinaryDictionary.GetTypeResource(GetType(), FILENAME_SUFFIX);
DataInput @in = new InputStreamDataInput(@is);
CodecUtil.CheckHeader(@in, HEADER, VERSION, VERSION);
@in.ReadBytes(characterCategoryMap, 0, characterCategoryMap.Length);
for (int i = 0; i < CLASS_COUNT; i++)
{
byte b = @in.ReadByte();
invokeMap[i] = (b & 0x01) != 0;
groupMap[i] = (b & 0x02) != 0;
}
}
public override SegmentInfo Read(Directory dir, string segment, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
ChecksumIndexInput input = dir.OpenChecksumInput(fileName, context);
bool success = false;
try
{
int codecVersion = CodecUtil.CheckHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME, Lucene46SegmentInfoFormat.VERSION_START, Lucene46SegmentInfoFormat.VERSION_CURRENT);
string version = input.ReadString();
int docCount = input.ReadInt32();
if (docCount < 0)
{
throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
}
bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
IDictionary <string, string> diagnostics = input.ReadStringStringMap();
ISet <string> files = input.ReadStringSet();
if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(input);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(input);
#pragma warning restore 612, 618
}
SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
si.SetFiles(files);
success = true;
return(si);
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(input);
}
else
{
input.Dispose();
}
}
}
/// <summary>
/// Sole constructor. </summary>
public Lucene40StoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context)
{
string segment = si.Name;
bool success = false;
fieldInfos = fn;
try
{
fieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_EXTENSION), context);
string indexStreamFN = IndexFileNames.SegmentFileName(segment, "", Lucene40StoredFieldsWriter.FIELDS_INDEX_EXTENSION);
indexStream = d.OpenInput(indexStreamFN, context);
CodecUtil.CheckHeader(indexStream, Lucene40StoredFieldsWriter.CODEC_NAME_IDX, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
CodecUtil.CheckHeader(fieldsStream, Lucene40StoredFieldsWriter.CODEC_NAME_DAT, Lucene40StoredFieldsWriter.VERSION_START, Lucene40StoredFieldsWriter.VERSION_CURRENT);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_DAT == fieldsStream.GetFilePointer());
Debugging.Assert(Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX == indexStream.GetFilePointer());
}
long indexSize = indexStream.Length - Lucene40StoredFieldsWriter.HEADER_LENGTH_IDX;
this.size = (int)(indexSize >> 3);
// Verify two sources of "maxDoc" agree:
if (this.size != si.DocCount)
{
throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + this.size + " but segmentInfo shows " + si.DocCount);
}
numTotalDocs = (int)(indexSize >> 3);
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
try
{
Dispose();
} // ensure we throw our original exception
catch (Exception)
{
}
}
}
}
private NumericDocValues LoadByteField(FieldInfo field, IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_START, Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
int valueSize = input.ReadInt();
if (valueSize != 1)
{
throw new CorruptIndexException("invalid valueSize: " + valueSize);
}
int maxDoc = State.SegmentInfo.DocCount;
sbyte[] values = new sbyte[maxDoc];
input.ReadBytes(values, 0, values.Length);
RamBytesUsed_Renamed.AddAndGet(RamUsageEstimator.SizeOf(values));
return(new NumericDocValuesAnonymousInnerClassHelper3(this, values));
}
private NumericDocValues LoadByteField(/*FieldInfo field, // LUCENENET: Never read */ IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.INTS_CODEC_NAME, Lucene40DocValuesFormat.INTS_VERSION_START, Lucene40DocValuesFormat.INTS_VERSION_CURRENT);
int valueSize = input.ReadInt32();
if (valueSize != 1)
{
throw new CorruptIndexException("invalid valueSize: " + valueSize);
}
int maxDoc = state.SegmentInfo.DocCount;
var values = new byte[maxDoc];
input.ReadBytes(values, 0, values.Length);
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values));
return(new NumericDocValuesAnonymousClass3(values));
}
private SortedDocValues LoadBytesFixedSorted(/*FieldInfo field, // LUCENENET: Never read */ IndexInput data, IndexInput index)
{
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
int fixedLength = data.ReadInt32();
int valueCount = index.ReadInt32();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, fixedLength * (long)valueCount);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInt32s.Reader reader = PackedInt32s.GetReader(index);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed());
return(CorrectBuggyOrds(new SortedDocValuesAnonymousClass(fixedLength, valueCount, bytesReader, reader)));
}
private SortedDocValues LoadBytesVarSorted(/*FieldInfo field, // LUCENENET: Never read */ IndexInput data, IndexInput index)
{
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
long maxAddress = index.ReadInt64();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, maxAddress);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInt32s.Reader addressReader = PackedInt32s.GetReader(index);
PackedInt32s.Reader ordsReader = PackedInt32s.GetReader(index);
int valueCount = addressReader.Count - 1;
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + addressReader.RamBytesUsed() + ordsReader.RamBytesUsed());
return(CorrectBuggyOrds(new SortedDocValuesAnonymousClass2(bytesReader, addressReader, ordsReader, valueCount)));
}
/// <summary>
/// NOTE: This was loadFloatField() in Lucene.
/// </summary>
private NumericDocValues LoadSingleField(FieldInfo field, IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.FLOATS_CODEC_NAME, Lucene40DocValuesFormat.FLOATS_VERSION_START, Lucene40DocValuesFormat.FLOATS_VERSION_CURRENT);
int valueSize = input.ReadInt32();
if (valueSize != 4)
{
throw new CorruptIndexException("invalid valueSize: " + valueSize);
}
int maxDoc = state.SegmentInfo.DocCount;
int[] values = new int[maxDoc];
for (int i = 0; i < values.Length; i++)
{
values[i] = input.ReadInt32();
}
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values));
return(new NumericDocValuesAnonymousInnerClassHelper7(values));
}
private NumericDocValues LoadDoubleField(/*FieldInfo field, // LUCENENET: Never read */ IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.FLOATS_CODEC_NAME, Lucene40DocValuesFormat.FLOATS_VERSION_START, Lucene40DocValuesFormat.FLOATS_VERSION_CURRENT);
int valueSize = input.ReadInt32();
if (valueSize != 8)
{
throw new CorruptIndexException("invalid valueSize: " + valueSize);
}
int maxDoc = state.SegmentInfo.DocCount;
long[] values = new long[maxDoc];
for (int i = 0; i < values.Length; i++)
{
values[i] = input.ReadInt64();
}
ramBytesUsed.AddAndGet(RamUsageEstimator.SizeOf(values));
return(new NumericDocValuesAnonymousClass8(values));
}
public override void Init(IndexInput termsIn)
{
_version = CodecUtil.CheckHeader(termsIn, PulsingPostingsWriter.CODEC,
PulsingPostingsWriter.VERSION_START,
PulsingPostingsWriter.VERSION_CURRENT);
_maxPositions = termsIn.ReadVInt();
_wrappedPostingsReader.Init(termsIn);
if (_wrappedPostingsReader is PulsingPostingsReader || _version < PulsingPostingsWriter.VERSION_META_ARRAY)
{
_fields = null;
}
else
{
_fields = new SortedDictionary <int, int>();
var summaryFileName = IndexFileNames.SegmentFileName(_segmentState.SegmentInfo.Name,
_segmentState.SegmentSuffix, PulsingPostingsWriter.SUMMARY_EXTENSION);
IndexInput input = null;
try
{
input =
_segmentState.Directory.OpenInput(summaryFileName, _segmentState.Context);
CodecUtil.CheckHeader(input,
PulsingPostingsWriter.CODEC,
_version,
PulsingPostingsWriter.VERSION_CURRENT);
var numField = input.ReadVInt();
for (var i = 0; i < numField; i++)
{
var fieldNum = input.ReadVInt();
var longsSize = input.ReadVInt();
_fields.Add(fieldNum, longsSize);
}
}
finally
{
IOUtils.CloseWhileHandlingException(input);
}
}
}
private SortedDocValues LoadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index)
{
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
long maxAddress = index.ReadLong();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, maxAddress);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInts.Reader addressReader = PackedInts.GetReader(index);
PackedInts.Reader ordsReader = PackedInts.GetReader(index);
int valueCount = addressReader.Size() - 1;
RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + addressReader.RamBytesUsed() + ordsReader.RamBytesUsed());
return(CorrectBuggyOrds(new SortedDocValuesAnonymousInnerClassHelper2(this, bytesReader, addressReader, ordsReader, valueCount)));
}
private SegmentInfo ReadUpgradedSegmentInfo(string name, Directory dir, IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene3xSegmentInfoFormat.UPGRADED_SI_CODEC_NAME, Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_START, Lucene3xSegmentInfoFormat.UPGRADED_SI_VERSION_CURRENT);
string version = input.ReadString();
int docCount = input.ReadInt();
IDictionary <string, string> attributes = input.ReadStringStringMap();
bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
IDictionary <string, string> diagnostics = input.ReadStringStringMap();
ISet <string> files = input.ReadStringSet();
SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, null, diagnostics, CollectionsHelper.UnmodifiableMap(attributes));
info.Files = files;
return(info);
}
public override SegmentInfo Read(Directory dir, string segment, IOContext context)
{
string fileName = IndexFileNames.SegmentFileName(segment, "", Lucene40SegmentInfoFormat.SI_EXTENSION);
IndexInput input = dir.OpenInput(fileName, context);
bool success = false;
try
{
CodecUtil.CheckHeader(input, Lucene40SegmentInfoFormat.CODEC_NAME, Lucene40SegmentInfoFormat.VERSION_START, Lucene40SegmentInfoFormat.VERSION_CURRENT);
string version = input.ReadString();
int docCount = input.ReadInt();
if (docCount < 0)
{
throw new CorruptIndexException("invalid docCount: " + docCount + " (resource=" + input + ")");
}
bool isCompoundFile = input.ReadByte() == SegmentInfo.YES;
IDictionary <string, string> diagnostics = input.ReadStringStringMap();
input.ReadStringStringMap(); // read deprecated attributes
ISet <string> files = input.ReadStringSet();
CodecUtil.CheckEOF(input);
SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
si.Files = files;
success = true;
return(si);
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(input);
}
else
{
input.Dispose();
}
}
}
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: @Override public codecs.FieldsProducer fieldsProducer(index.SegmentReadState state) throws java.io.IOException
public override FieldsProducer fieldsProducer(SegmentReadState state)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String fileName = index.IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
string fileName = IndexFileNames.SegmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final store.ChecksumIndexInput in = state.directory.openChecksumInput(fileName, store.IOContext.READONCE);
ChecksumIndexInput @in = state.directory.openChecksumInput(fileName, IOContext.READONCE);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.SortedMap<String,TermsReader> fields = new java.util.TreeMap<>();
SortedMap <string, TermsReader> fields = new SortedDictionary <string, TermsReader>();
try
{
CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
while (true)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int termCount = in.readVInt();
int termCount = @in.readVInt();
if (termCount == 0)
{
break;
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
TermsReader termsReader = new TermsReader(state.fieldInfos, @in, termCount);
// System.out.println("load field=" + termsReader.field.name);
fields.put(termsReader.field.name, termsReader);
}
CodecUtil.CheckFooter(@in);
}
finally
{
@in.close();
}
return(new FieldsProducerAnonymousInnerClassHelper(this, fields));
}
public override bool Load(DataInput input)
{
CodecUtil.CheckHeader(input, CODEC_NAME, VERSION_START, VERSION_START);
count = input.ReadVInt64();
var separatorOrig = (sbyte)input.ReadByte();
if (separatorOrig != separator)
{
throw new InvalidOperationException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig);
}
int gramsOrig = input.ReadVInt32();
if (gramsOrig != grams)
{
throw new InvalidOperationException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig);
}
totTokens = input.ReadVInt64();
fst = new FST <long?>(input, PositiveInt32Outputs.Singleton);
return(true);
}
private BinaryDocValues LoadBytesFixedDeref(FieldInfo field)
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name + "_" + Convert.ToString(field.Number, CultureInfo.InvariantCulture), segmentSuffix, "dat");
string indexName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name + "_" + Convert.ToString(field.Number, CultureInfo.InvariantCulture), segmentSuffix, "idx");
IndexInput data = null;
IndexInput index = null;
bool success = false;
try
{
data = dir.OpenInput(dataName, state.Context);
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
index = dir.OpenInput(indexName, state.Context);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
int fixedLength = data.ReadInt32();
int valueCount = index.ReadInt32();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, fixedLength * (long)valueCount);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInt32s.Reader reader = PackedInt32s.GetReader(index);
CodecUtil.CheckEOF(data);
CodecUtil.CheckEOF(index);
ramBytesUsed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed());
success = true;
return(new BinaryDocValuesAnonymousClass3(fixedLength, bytesReader, reader));
}
finally
{
if (success)
{
IOUtils.Dispose(data, index);
}
else
{
IOUtils.DisposeWhileHandlingException(data, index);
}
}
}
// private String segment;
/// <summary>
/// Sole constructor. </summary>
public Lucene40PostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, string segmentSuffix)
{
bool success = false;
IndexInput freqIn = null;
IndexInput proxIn = null;
try
{
freqIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), ioContext);
CodecUtil.CheckHeader(freqIn, FRQ_CODEC, VERSION_START, VERSION_CURRENT);
// TODO: hasProx should (somehow!) become codec private,
// but it's tricky because 1) FIS.hasProx is global (it
// could be all fields that have prox are written by a
// different codec), 2) the field may have had prox in
// the past but all docs w/ that field were deleted.
// Really we'd need to init prxOut lazily on write, and
// then somewhere record that we actually wrote it so we
// know whether to open on read:
if (fieldInfos.HasProx)
{
proxIn = dir.OpenInput(IndexFileNames.SegmentFileName(segmentInfo.Name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), ioContext);
CodecUtil.CheckHeader(proxIn, PRX_CODEC, VERSION_START, VERSION_CURRENT);
}
else
{
proxIn = null;
}
this.freqIn = freqIn;
this.proxIn = proxIn;
success = true;
}
finally
{
if (!success)
{
IOUtils.CloseWhileHandlingException(freqIn, proxIn);
}
}
}
private BinaryDocValues LoadBytesVarDeref(FieldInfo field)
{
string dataName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "dat");
string indexName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "idx");
IndexInput data = null;
IndexInput index = null;
bool success = false;
try
{
data = Dir.OpenInput(dataName, State.Context);
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
index = Dir.OpenInput(indexName, State.Context);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_DEREF_VERSION_CURRENT);
long totalBytes = index.ReadLong();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, totalBytes);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInts.Reader reader = PackedInts.GetReader(index);
CodecUtil.CheckEOF(data);
CodecUtil.CheckEOF(index);
RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed());
success = true;
return(new BinaryDocValuesAnonymousInnerClassHelper4(this, bytesReader, reader));
}
finally
{
if (success)
{
IOUtils.Close(data, index);
}
else
{
IOUtils.CloseWhileHandlingException(data, index);
}
}
}
public override FieldsProducer FieldsProducer(SegmentReadState readState)
{
// Load our ID:
string idFileName = IndexFileNames.SegmentFileName(readState.SegmentInfo.Name, readState.SegmentSuffix, ID_EXTENSION);
IndexInput @in = readState.Directory.OpenInput(idFileName, readState.Context);
bool success = false;
int id;
try
{
CodecUtil.CheckHeader(@in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST);
id = @in.ReadVInt32();
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(@in);
}
else
{
IOUtils.Dispose(@in);
}
}
UninterruptableMonitor.Enter(state);
try
{
return(state.TryGetValue(id, out RAMPostings value) ? value : null);
}
finally
{
UninterruptableMonitor.Exit(state);
}
}
private int ReadHeader(IndexInput @in)
{
return(CodecUtil.CheckHeader(@in, FSTTermsWriter.TERMS_CODEC_NAME, FSTTermsWriter.TERMS_VERSION_START, FSTTermsWriter.TERMS_VERSION_CURRENT));
}
/// <summary>
/// Helper method that reads CFS entries from an input stream </summary>
private static IDictionary <string, FileEntry> ReadEntries(IndexInputSlicer handle, Directory dir, string name)
{
IOException priorE = null;
IndexInput stream = null;
ChecksumIndexInput entriesStream = null;
// read the first VInt. If it is negative, it's the version number
// otherwise it's the count (pre-3.1 indexes)
try
{
IDictionary <string, FileEntry> mapping;
#pragma warning disable 612, 618
stream = handle.OpenFullSlice();
#pragma warning restore 612, 618
int firstInt = stream.ReadVInt32();
// impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible
// and separate norms/etc are outside of cfs.
if (firstInt == CODEC_MAGIC_BYTE1)
{
sbyte secondByte = (sbyte)stream.ReadByte();
sbyte thirdByte = (sbyte)stream.ReadByte();
sbyte fourthByte = (sbyte)stream.ReadByte();
if (secondByte != CODEC_MAGIC_BYTE2 || thirdByte != CODEC_MAGIC_BYTE3 || fourthByte != CODEC_MAGIC_BYTE4)
{
throw new CorruptIndexException("Illegal/impossible header for CFS file: " + secondByte + "," + thirdByte + "," + fourthByte);
}
int version = CodecUtil.CheckHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT);
string entriesFileName = IndexFileNames.SegmentFileName(
IndexFileNames.StripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
entriesStream = dir.OpenChecksumInput(entriesFileName, IOContext.READ_ONCE);
CodecUtil.CheckHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT);
int numEntries = entriesStream.ReadVInt32();
mapping = new Dictionary <string, FileEntry>(numEntries);
for (int i = 0; i < numEntries; i++)
{
FileEntry fileEntry = new FileEntry();
string id = entriesStream.ReadString();
FileEntry previous = mapping.Put(id, fileEntry);
if (previous != null)
{
throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS: " + entriesStream);
}
fileEntry.Offset = entriesStream.ReadInt64();
fileEntry.Length = entriesStream.ReadInt64();
}
if (version >= CompoundFileWriter.VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(entriesStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(entriesStream);
#pragma warning restore 612, 618
}
}
else
{
// TODO remove once 3.x is not supported anymore
mapping = ReadLegacyEntries(stream, firstInt);
}
return(mapping);
}
catch (IOException ioe)
{
priorE = ioe;
}
finally
{
IOUtils.DisposeWhileHandlingException(priorE, stream, entriesStream);
}
// this is needed until Java 7's real try-with-resources:
throw new InvalidOperationException("impossible to get here");
}
internal Lucene42DocValuesProducer(SegmentReadState state, string dataCodec, string dataExtension, string metaCodec, string metaExtension)
{
maxDoc = state.SegmentInfo.DocCount;
string metaName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput @in = state.Directory.OpenChecksumInput(metaName, state.Context);
bool success = false;
ramBytesUsed = new AtomicInt64(RamUsageEstimator.ShallowSizeOfInstance(this.GetType()));
try
{
version = CodecUtil.CheckHeader(@in, metaCodec, VERSION_START, VERSION_CURRENT);
numerics = new Dictionary <int, NumericEntry>();
binaries = new Dictionary <int, BinaryEntry>();
fsts = new Dictionary <int, FSTEntry>();
ReadFields(@in, state.FieldInfos);
if (version >= VERSION_CHECKSUM)
{
CodecUtil.CheckFooter(@in);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(@in);
#pragma warning restore 612, 618
}
success = true;
}
finally
{
if (success)
{
IOUtils.Dispose(@in);
}
else
{
IOUtils.DisposeWhileHandlingException(@in);
}
}
success = false;
try
{
string dataName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, dataExtension);
data = state.Directory.OpenInput(dataName, state.Context);
int version2 = CodecUtil.CheckHeader(data, dataCodec, VERSION_START, VERSION_CURRENT);
if (version != version2)
{
throw new CorruptIndexException("Format versions mismatch");
}
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this.data);
}
}
}
protected BinaryDictionary()
{
int[] targetMapOffsets = null, targetMap = null;
string[] posDict = null;
string[] inflFormDict = null;
string[] inflTypeDict = null;
ByteBuffer buffer; // LUCENENET: IDE0059: Remove unnecessary value assignment
using (Stream mapIS = GetResource(TARGETMAP_FILENAME_SUFFIX))
{
DataInput @in = new InputStreamDataInput(mapIS);
CodecUtil.CheckHeader(@in, TARGETMAP_HEADER, VERSION, VERSION);
targetMap = new int[@in.ReadVInt32()];
targetMapOffsets = new int[@in.ReadVInt32()];
int accum = 0, sourceId = 0;
for (int ofs = 0; ofs < targetMap.Length; ofs++)
{
int val = @in.ReadVInt32();
if ((val & 0x01) != 0)
{
targetMapOffsets[sourceId] = ofs;
sourceId++;
}
accum += val.TripleShift(1);
targetMap[ofs] = accum;
}
if (sourceId + 1 != targetMapOffsets.Length)
{
throw new IOException("targetMap file format broken");
}
targetMapOffsets[sourceId] = targetMap.Length;
}
using (Stream posIS = GetResource(POSDICT_FILENAME_SUFFIX))
{
DataInput @in = new InputStreamDataInput(posIS);
CodecUtil.CheckHeader(@in, POSDICT_HEADER, VERSION, VERSION);
int posSize = @in.ReadVInt32();
posDict = new string[posSize];
inflTypeDict = new string[posSize];
inflFormDict = new string[posSize];
for (int j = 0; j < posSize; j++)
{
posDict[j] = @in.ReadString();
inflTypeDict[j] = @in.ReadString();
inflFormDict[j] = @in.ReadString();
// this is how we encode null inflections
if (inflTypeDict[j].Length == 0)
{
inflTypeDict[j] = null;
}
if (inflFormDict[j].Length == 0)
{
inflFormDict[j] = null;
}
}
}
ByteBuffer tmpBuffer;
using (Stream dictIS = GetResource(DICT_FILENAME_SUFFIX))
{
// no buffering here, as we load in one large buffer
DataInput @in = new InputStreamDataInput(dictIS);
CodecUtil.CheckHeader(@in, DICT_HEADER, VERSION, VERSION);
int size = @in.ReadVInt32();
tmpBuffer = ByteBuffer.Allocate(size); // AllocateDirect..?
int read = dictIS.Read(tmpBuffer.Array, 0, size);
if (read != size)
{
throw EOFException.Create("Cannot read whole dictionary");
}
}
buffer = tmpBuffer.AsReadOnlyBuffer();
this.targetMap = targetMap;
this.targetMapOffsets = targetMapOffsets;
this.posDict = posDict;
this.inflTypeDict = inflTypeDict;
this.inflFormDict = inflFormDict;
this.buffer = buffer;
}
/// <summary>
/// Sole constructor. </summary>
public CompressingTermVectorsReader(Directory d, SegmentInfo si, string segmentSuffix, FieldInfos fn, IOContext context, string formatName, CompressionMode compressionMode)
{
this.compressionMode = compressionMode;
string segment = si.Name;
bool success = false;
fieldInfos = fn;
numDocs = si.DocCount;
ChecksumIndexInput indexStream = null;
try
{
// Load the index into memory
string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION);
indexStream = d.OpenChecksumInput(indexStreamFN, context);
string codecNameIdx = formatName + CompressingTermVectorsWriter.CODEC_SFX_IDX;
version = CodecUtil.CheckHeader(indexStream, codecNameIdx, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameIdx) == indexStream.GetFilePointer());
}
indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
if (version >= CompressingTermVectorsWriter.VERSION_CHECKSUM)
{
indexStream.ReadVInt64(); // the end of the data file
CodecUtil.CheckFooter(indexStream);
}
else
{
#pragma warning disable 612, 618
CodecUtil.CheckEOF(indexStream);
#pragma warning restore 612, 618
}
indexStream.Dispose();
indexStream = null;
// Open the data file and read metadata
string vectorsStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, CompressingTermVectorsWriter.VECTORS_EXTENSION);
vectorsStream = d.OpenInput(vectorsStreamFN, context);
string codecNameDat = formatName + CompressingTermVectorsWriter.CODEC_SFX_DAT;
int version2 = CodecUtil.CheckHeader(vectorsStream, codecNameDat, CompressingTermVectorsWriter.VERSION_START, CompressingTermVectorsWriter.VERSION_CURRENT);
if (version != version2)
{
throw new Exception("Version mismatch between stored fields index and data: " + version + " != " + version2);
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(CodecUtil.HeaderLength(codecNameDat) == vectorsStream.GetFilePointer());
}
packedIntsVersion = vectorsStream.ReadVInt32();
chunkSize = vectorsStream.ReadVInt32();
decompressor = compressionMode.NewDecompressor();
this.reader = new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, CompressingTermVectorsWriter.BLOCK_SIZE, 0);
success = true;
}
finally
{
if (!success)
{
IOUtils.DisposeWhileHandlingException(this, indexStream);
}
}
}
/// <summary>
/// Reads the snapshots information from the given <seealso cref="Directory"/>. this
/// method can be used if the snapshots information is needed, however you
/// cannot instantiate the deletion policy (because e.g., some other process
/// keeps a lock on the snapshots directory).
/// </summary>
private void LoadPriorSnapshots()
{
lock (this)
{
long genLoaded = -1;
IOException ioe = null;
IList <string> snapshotFiles = new List <string>();
foreach (string file in dir.ListAll())
{
if (file.StartsWith(SNAPSHOTS_PREFIX, StringComparison.Ordinal))
{
long gen = Convert.ToInt64(file.Substring(SNAPSHOTS_PREFIX.Length));
if (genLoaded == -1 || gen > genLoaded)
{
snapshotFiles.Add(file);
IDictionary <long, int> m = new Dictionary <long, int>();
IndexInput @in = dir.OpenInput(file, IOContext.DEFAULT);
try
{
CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_START);
int count = @in.ReadVInt32();
for (int i = 0; i < count; i++)
{
long commitGen = @in.ReadVInt64();
int refCount = @in.ReadVInt32();
m[commitGen] = refCount;
}
}
catch (IOException ioe2)
{
// Save first exception & throw in the end
if (ioe == null)
{
ioe = ioe2;
}
}
finally
{
@in.Dispose();
}
genLoaded = gen;
m_refCounts.Clear();
m_refCounts.PutAll(m);
}
}
}
if (genLoaded == -1)
{
// Nothing was loaded...
if (ioe != null)
{
// ... not for lack of trying:
throw ioe;
}
}
else
{
if (snapshotFiles.Count > 1)
{
// Remove any broken / old snapshot files:
string curFileName = SNAPSHOTS_PREFIX + genLoaded;
foreach (string file in snapshotFiles)
{
if (!curFileName.Equals(file, StringComparison.Ordinal))
{
dir.DeleteFile(file);
}
}
}
nextWriteGen = 1 + genLoaded;
}
}
}
