private NumericDocValues LoadVarIntsField(FieldInfo field, IndexInput input)
{
CodecUtil.CheckHeader(input, Lucene40DocValuesFormat.VAR_INTS_CODEC_NAME, Lucene40DocValuesFormat.VAR_INTS_VERSION_START, Lucene40DocValuesFormat.VAR_INTS_VERSION_CURRENT);
byte header = input.ReadByte();
if (header == Lucene40DocValuesFormat.VAR_INTS_FIXED_64)
{
int maxDoc = State.SegmentInfo.DocCount;
long[] values = new long[maxDoc];
for (int i = 0; i < values.Length; i++)
{
values[i] = input.ReadLong();
}
RamBytesUsed_Renamed.AddAndGet(RamUsageEstimator.SizeOf(values));
return(new NumericDocValuesAnonymousInnerClassHelper(this, values));
}
else if (header == Lucene40DocValuesFormat.VAR_INTS_PACKED)
{
long minValue = input.ReadLong();
long defaultValue = input.ReadLong();
PackedInts.Reader reader = PackedInts.GetReader(input);
RamBytesUsed_Renamed.AddAndGet(reader.RamBytesUsed());
return(new NumericDocValuesAnonymousInnerClassHelper2(this, minValue, defaultValue, reader));
}
else
{
throw new CorruptIndexException("invalid VAR_INTS header byte: " + header + " (resource=" + input + ")");
}
}
private SortedDocValues LoadBytesFixedSorted(FieldInfo field, IndexInput data, IndexInput index)
{
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_SORTED_VERSION_CURRENT);
int fixedLength = data.ReadInt();
int valueCount = index.ReadInt();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, fixedLength * (long)valueCount);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInts.Reader reader = PackedInts.GetReader(index);
RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed());
return(CorrectBuggyOrds(new SortedDocValuesAnonymousInnerClassHelper(this, fixedLength, valueCount, bytesReader, reader)));
}
private SortedDocValues LoadBytesVarSorted(FieldInfo field, IndexInput data, IndexInput index)
{
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_VAR_SORTED_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_START, Lucene40DocValuesFormat.BYTES_VAR_SORTED_VERSION_CURRENT);
long maxAddress = index.ReadLong();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, maxAddress);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInts.Reader addressReader = PackedInts.GetReader(index);
PackedInts.Reader ordsReader = PackedInts.GetReader(index);
int valueCount = addressReader.Size() - 1;
RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + addressReader.RamBytesUsed() + ordsReader.RamBytesUsed());
return(CorrectBuggyOrds(new SortedDocValuesAnonymousInnerClassHelper2(this, bytesReader, addressReader, ordsReader, valueCount)));
}
private BinaryDocValues LoadBytesFixedDeref(FieldInfo field)
{
string dataName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "dat");
string indexName = IndexFileNames.SegmentFileName(State.SegmentInfo.Name + "_" + Convert.ToString(field.Number), SegmentSuffix, "idx");
IndexInput data = null;
IndexInput index = null;
bool success = false;
try
{
data = Dir.OpenInput(dataName, State.Context);
CodecUtil.CheckHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
index = Dir.OpenInput(indexName, State.Context);
CodecUtil.CheckHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_START, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
int fixedLength = data.ReadInt();
int valueCount = index.ReadInt();
PagedBytes bytes = new PagedBytes(16);
bytes.Copy(data, fixedLength * (long)valueCount);
PagedBytes.Reader bytesReader = bytes.Freeze(true);
PackedInts.Reader reader = PackedInts.GetReader(index);
CodecUtil.CheckEOF(data);
CodecUtil.CheckEOF(index);
RamBytesUsed_Renamed.AddAndGet(bytes.RamBytesUsed() + reader.RamBytesUsed());
success = true;
return(new BinaryDocValuesAnonymousInnerClassHelper3(this, fixedLength, bytesReader, reader));
}
finally
{
if (success)
{
IOUtils.Close(data, index);
}
else
{
IOUtils.CloseWhileHandlingException(data, index);
}
}
}
public CoreFieldIndex(long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart,
int numIndexTerms, FixedGapTermsIndexReader fgtir)
{
TermsStart = termsStart;
TermBytesStart = fgtir._termBytes.Pointer;
var clone = (IndexInput)fgtir._input.Clone();
clone.Seek(indexStart);
// -1 is passed to mean "don't load term index", but
// if we are then later loaded it's overwritten with
// a real value
Debug.Assert(fgtir._indexDivisor > 0);
NumIndexTerms = 1 + (numIndexTerms - 1) / fgtir._indexDivisor;
Debug.Assert(NumIndexTerms > 0,
String.Format("NumIndexTerms: {0}, IndexDivisor: {1}", NumIndexTerms, fgtir._indexDivisor));
if (fgtir._indexDivisor == 1)
{
// Default (load all index terms) is fast -- slurp in the images from disk:
try
{
var numTermBytes = packedIndexStart - indexStart;
fgtir._termBytes.Copy(clone, numTermBytes);
// records offsets into main terms dict file
TermsDictOffsets = PackedInts.GetReader(clone);
Debug.Assert(TermsDictOffsets.Size() == numIndexTerms);
// records offsets into byte[] term data
TermOffsets = PackedInts.GetReader(clone);
Debug.Assert(TermOffsets.Size() == 1 + numIndexTerms);
}
finally
{
clone.Dispose();
}
}
else
{
// Get packed iterators
var clone1 = (IndexInput)fgtir._input.Clone();
var clone2 = (IndexInput)fgtir._input.Clone();
try
{
// Subsample the index terms
clone1.Seek(packedIndexStart);
PackedInts.ReaderIterator termsDictOffsetsIter = PackedInts.GetReaderIterator(clone1,
PackedInts.DEFAULT_BUFFER_SIZE);
clone2.Seek(packedOffsetsStart);
PackedInts.ReaderIterator termOffsetsIter = PackedInts.GetReaderIterator(clone2,
PackedInts.DEFAULT_BUFFER_SIZE);
// TODO: often we can get by w/ fewer bits per
// value, below.. .but this'd be more complex:
// we'd have to try @ fewer bits and then grow
// if we overflowed it.
PackedInts.Mutable termsDictOffsetsM = PackedInts.GetMutable(NumIndexTerms,
termsDictOffsetsIter.BitsPerValue, PackedInts.DEFAULT);
PackedInts.Mutable termOffsetsM = PackedInts.GetMutable(NumIndexTerms + 1,
termOffsetsIter.BitsPerValue, PackedInts.DEFAULT);
TermsDictOffsets = termsDictOffsetsM;
TermOffsets = termOffsetsM;
var upto = 0;
long termOffsetUpto = 0;
while (upto < NumIndexTerms)
{
// main file offset copies straight over
termsDictOffsetsM.Set(upto, termsDictOffsetsIter.Next());
termOffsetsM.Set(upto, termOffsetUpto);
var termOffset = termOffsetsIter.Next();
var nextTermOffset = termOffsetsIter.Next();
var numTermBytes = (int)(nextTermOffset - termOffset);
clone.Seek(indexStart + termOffset);
Debug.Assert(indexStart + termOffset < clone.Length(),
String.Format("IndexStart: {0}, TermOffset: {1}, Len: {2}", indexStart, termOffset,
clone.Length()));
Debug.Assert(indexStart + termOffset + numTermBytes < clone.Length());
fgtir._termBytes.Copy(clone, numTermBytes);
termOffsetUpto += numTermBytes;
upto++;
if (upto == NumIndexTerms)
{
break;
}
// skip terms:
termsDictOffsetsIter.Next();
for (var i = 0; i < fgtir._indexDivisor - 2; i++)
{
termOffsetsIter.Next();
termsDictOffsetsIter.Next();
}
}
termOffsetsM.Set(upto, termOffsetUpto);
}
finally
{
clone1.Dispose();
clone2.Dispose();
clone.Dispose();
}
}
}