internal override void PackPendingValues()
{
// compute max delta
long minValue = Pending[0];
long maxValue = Pending[0];
for (int i = 1; i < PendingOff; ++i)
{
minValue = Math.Min(minValue, Pending[i]);
maxValue = Math.Max(maxValue, Pending[i]);
}
long delta = maxValue - minValue;
MinValues[ValuesOff] = minValue;
if (delta == 0)
{
Values[ValuesOff] = new PackedInts.NullReader(PendingOff);
}
else
{
// build a new packed reader
int bitsRequired = delta < 0 ? 64 : PackedInts.BitsRequired(delta);
for (int i = 0; i < PendingOff; ++i)
{
Pending[i] -= minValue;
}
PackedInts.Mutable mutable = PackedInts.GetMutable(PendingOff, bitsRequired, AcceptableOverheadRatio);
for (int i = 0; i < PendingOff;)
{
i += mutable.Set(i, Pending, i, PendingOff - i);
}
Values[ValuesOff] = mutable;
}
}
/// <summary>
/// Compress <code>bytes[off:off+len]</code> into <code>out</code> using
/// at most 16KB of memory. <code>ht</code> shouldn't be shared across threads
/// but can safely be reused.
/// </summary>
public static void Compress(byte[] bytes, int off, int len, DataOutput @out, HashTable ht)
{
int @base = off;
int end = off + len;
int anchor = off++;
if (len > LAST_LITERALS + MIN_MATCH)
{
int limit = end - LAST_LITERALS;
int matchLimit = limit - MIN_MATCH;
ht.Reset(len);
int hashLog = ht.HashLog;
PackedInts.Mutable hashTable = ht.hashTable;
while (off <= limit)
{
// find a match
int @ref;
while (true)
{
if (off >= matchLimit)
{
goto mainBreak;
}
int v = ReadInt(bytes, off);
int h = Hash(v, hashLog);
@ref = @base + (int)hashTable.Get(h);
Debug.Assert(PackedInts.BitsRequired(off - @base) <= hashTable.BitsPerValue);
hashTable.Set(h, off - @base);
if (off - @ref < MAX_DISTANCE && ReadInt(bytes, @ref) == v)
{
break;
}
++off;
}
// compute match length
int matchLen = MIN_MATCH + CommonBytes(bytes, @ref + MIN_MATCH, off + MIN_MATCH, limit);
EncodeSequence(bytes, anchor, @ref, off, matchLen, @out);
off += matchLen;
anchor = off;
mainContinue :;
}
mainBreak :;
}
// last literals
int literalLen = end - anchor;
Debug.Assert(literalLen >= LAST_LITERALS || literalLen == len);
EncodeLastLiterals(bytes, anchor, end - anchor, @out);
}
internal void Reset(int len)
{
int bitsPerOffset = PackedInts.BitsRequired(len - LAST_LITERALS);
int bitsPerOffsetLog = 32 - Number.NumberOfLeadingZeros(bitsPerOffset - 1);
HashLog = MEMORY_USAGE + 3 - bitsPerOffsetLog;
if (hashTable == null || hashTable.Size() < 1 << HashLog || hashTable.BitsPerValue < bitsPerOffset)
{
hashTable = PackedInts.GetMutable(1 << HashLog, bitsPerOffset, PackedInts.DEFAULT);
}
else
{
hashTable.Clear();
}
}
private void EnsureCapacity(long value)
{
if ((value & CurrentMask) == value)
{
return;
}
int bitsRequired = value < 0 ? 64 : PackedInts.BitsRequired(value);
Debug.Assert(bitsRequired > Current.BitsPerValue);
int valueCount = Size();
PackedInts.Mutable next = PackedInts.GetMutable(valueCount, bitsRequired, AcceptableOverheadRatio);
PackedInts.Copy(Current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE);
Current = next;
CurrentMask = Mask(Current.BitsPerValue);
}
internal override void PackPendingValues()
{
// compute max delta
long minValue = Pending[0];
long maxValue = Pending[0];
for (int i = 1; i < PendingOff; ++i)
{
minValue = Math.Min(minValue, Pending[i]);
maxValue = Math.Max(maxValue, Pending[i]);
}
// build a new packed reader
int bitsRequired = minValue < 0 ? 64 : PackedInts.BitsRequired(maxValue);
PackedInts.Mutable mutable = PackedInts.GetMutable(PendingOff, bitsRequired, AcceptableOverheadRatio);
for (int i = 0; i < PendingOff;)
{
i += mutable.Set(i, Pending, i, PendingOff - i);
}
Values[ValuesOff] = mutable;
}
internal override void PackPendingValues()
{
Debug.Assert(PendingOff > 0);
MinValues[ValuesOff] = Pending[0];
Averages[ValuesOff] = PendingOff == 1 ? 0 : (float)(Pending[PendingOff - 1] - Pending[0]) / (PendingOff - 1);
for (int i = 0; i < PendingOff; ++i)
{
Pending[i] = ZigZagEncode(Pending[i] - MinValues[ValuesOff] - (long)(Averages[ValuesOff] * (long)i));
}
long maxDelta = 0;
for (int i = 0; i < PendingOff; ++i)
{
if (Pending[i] < 0)
{
maxDelta = -1;
break;
}
else
{
maxDelta = Math.Max(maxDelta, Pending[i]);
}
}
if (maxDelta == 0)
{
Values[ValuesOff] = new PackedInts.NullReader(PendingOff);
}
else
{
int bitsRequired = maxDelta < 0 ? 64 : PackedInts.BitsRequired(maxDelta);
PackedInts.Mutable mutable = PackedInts.GetMutable(PendingOff, bitsRequired, AcceptableOverheadRatio);
for (int i = 0; i < PendingOff;)
{
i += mutable.Set(i, Pending, i, PendingOff - i);
}
Values[ValuesOff] = mutable;
}
}
/// <param name="startBitsPerValue"> the initial number of bits per value, may grow depending on the data </param>
/// <param name="valueCount"> the number of values </param>
/// <param name="acceptableOverheadRatio"> an acceptable overhead ratio </param>
public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio)
{
this.AcceptableOverheadRatio = acceptableOverheadRatio;
Current = PackedInts.GetMutable(valueCount, startBitsPerValue, this.AcceptableOverheadRatio);
CurrentMask = Mask(Current.BitsPerValue);
}
internal void Reset(int len)
{
int bitsPerOffset = PackedInts.BitsRequired(len - LAST_LITERALS);
int bitsPerOffsetLog = 32 - Number.NumberOfLeadingZeros(bitsPerOffset - 1);
HashLog = MEMORY_USAGE + 3 - bitsPerOffsetLog;
if (hashTable == null || hashTable.Size() < 1 << HashLog || hashTable.BitsPerValue < bitsPerOffset)
{
hashTable = PackedInts.GetMutable(1 << HashLog, bitsPerOffset, PackedInts.DEFAULT);
}
else
{
hashTable.Clear();
}
}
public CoreFieldIndex(long indexStart, long termsStart, long packedIndexStart, long packedOffsetsStart,
int numIndexTerms, FixedGapTermsIndexReader fgtir)
{
TermsStart = termsStart;
TermBytesStart = fgtir._termBytes.Pointer;
var clone = (IndexInput)fgtir._input.Clone();
clone.Seek(indexStart);
// -1 is passed to mean "don't load term index", but
// if we are then later loaded it's overwritten with
// a real value
Debug.Assert(fgtir._indexDivisor > 0);
NumIndexTerms = 1 + (numIndexTerms - 1) / fgtir._indexDivisor;
Debug.Assert(NumIndexTerms > 0,
String.Format("NumIndexTerms: {0}, IndexDivisor: {1}", NumIndexTerms, fgtir._indexDivisor));
if (fgtir._indexDivisor == 1)
{
// Default (load all index terms) is fast -- slurp in the images from disk:
try
{
var numTermBytes = packedIndexStart - indexStart;
fgtir._termBytes.Copy(clone, numTermBytes);
// records offsets into main terms dict file
TermsDictOffsets = PackedInts.GetReader(clone);
Debug.Assert(TermsDictOffsets.Size() == numIndexTerms);
// records offsets into byte[] term data
TermOffsets = PackedInts.GetReader(clone);
Debug.Assert(TermOffsets.Size() == 1 + numIndexTerms);
}
finally
{
clone.Dispose();
}
}
else
{
// Get packed iterators
var clone1 = (IndexInput)fgtir._input.Clone();
var clone2 = (IndexInput)fgtir._input.Clone();
try
{
// Subsample the index terms
clone1.Seek(packedIndexStart);
PackedInts.ReaderIterator termsDictOffsetsIter = PackedInts.GetReaderIterator(clone1,
PackedInts.DEFAULT_BUFFER_SIZE);
clone2.Seek(packedOffsetsStart);
PackedInts.ReaderIterator termOffsetsIter = PackedInts.GetReaderIterator(clone2,
PackedInts.DEFAULT_BUFFER_SIZE);
// TODO: often we can get by w/ fewer bits per
// value, below.. .but this'd be more complex:
// we'd have to try @ fewer bits and then grow
// if we overflowed it.
PackedInts.Mutable termsDictOffsetsM = PackedInts.GetMutable(NumIndexTerms,
termsDictOffsetsIter.BitsPerValue, PackedInts.DEFAULT);
PackedInts.Mutable termOffsetsM = PackedInts.GetMutable(NumIndexTerms + 1,
termOffsetsIter.BitsPerValue, PackedInts.DEFAULT);
TermsDictOffsets = termsDictOffsetsM;
TermOffsets = termOffsetsM;
var upto = 0;
long termOffsetUpto = 0;
while (upto < NumIndexTerms)
{
// main file offset copies straight over
termsDictOffsetsM.Set(upto, termsDictOffsetsIter.Next());
termOffsetsM.Set(upto, termOffsetUpto);
var termOffset = termOffsetsIter.Next();
var nextTermOffset = termOffsetsIter.Next();
var numTermBytes = (int)(nextTermOffset - termOffset);
clone.Seek(indexStart + termOffset);
Debug.Assert(indexStart + termOffset < clone.Length(),
String.Format("IndexStart: {0}, TermOffset: {1}, Len: {2}", indexStart, termOffset,
clone.Length()));
Debug.Assert(indexStart + termOffset + numTermBytes < clone.Length());
fgtir._termBytes.Copy(clone, numTermBytes);
termOffsetUpto += numTermBytes;
upto++;
if (upto == NumIndexTerms)
{
break;
}
// skip terms:
termsDictOffsetsIter.Next();
for (var i = 0; i < fgtir._indexDivisor - 2; i++)
{
termOffsetsIter.Next();
termsDictOffsetsIter.Next();
}
}
termOffsetsM.Set(upto, termOffsetUpto);
}
finally
{
clone1.Dispose();
clone2.Dispose();
clone.Dispose();
}
}
}
public override Fields Get(int doc)
{
EnsureOpen();
// seek to the right place
{
long startPointer = indexReader.GetStartPointer(doc);
vectorsStream.Seek(startPointer);
}
// decode
// - docBase: first doc ID of the chunk
// - chunkDocs: number of docs of the chunk
int docBase = vectorsStream.ReadVInt();
int chunkDocs = vectorsStream.ReadVInt();
if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs)
{
throw new CorruptIndexException("docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc + " (resource=" + vectorsStream + ")");
}
int skip; // number of fields to skip
int numFields; // number of fields of the document we're looking for
int totalFields; // total number of fields of the chunk (sum for all docs)
if (chunkDocs == 1)
{
skip = 0;
numFields = totalFields = vectorsStream.ReadVInt();
}
else
{
reader.Reset(vectorsStream, chunkDocs);
int sum = 0;
for (int i = docBase; i < doc; ++i)
{
sum += (int)reader.Next();
}
skip = sum;
numFields = (int)reader.Next();
sum += numFields;
for (int i = doc + 1; i < docBase + chunkDocs; ++i)
{
sum += (int)reader.Next();
}
totalFields = sum;
}
if (numFields == 0)
{
// no vectors
return(null);
}
// read field numbers that have term vectors
int[] fieldNums;
{
int token = vectorsStream.ReadByte() & 0xFF;
Debug.Assert(token != 0); // means no term vectors, cannot happen since we checked for numFields == 0
int bitsPerFieldNum = token & 0x1F;
int totalDistinctFields = (int)((uint)token >> 5);
if (totalDistinctFields == 0x07)
{
totalDistinctFields += vectorsStream.ReadVInt();
}
++totalDistinctFields;
PackedInts.ReaderIterator it = PackedInts.GetReaderIteratorNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalDistinctFields, bitsPerFieldNum, 1);
fieldNums = new int[totalDistinctFields];
for (int i = 0; i < totalDistinctFields; ++i)
{
fieldNums[i] = (int)it.Next();
}
}
// read field numbers and flags
int[] fieldNumOffs = new int[numFields];
PackedInts.Reader flags;
{
int bitsPerOff = PackedInts.BitsRequired(fieldNums.Length - 1);
PackedInts.Reader allFieldNumOffs = PackedInts.GetReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff);
switch (vectorsStream.ReadVInt())
{
case 0:
PackedInts.Reader fieldFlags = PackedInts.GetReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, fieldNums.Length, CompressingTermVectorsWriter.FLAGS_BITS);
PackedInts.Mutable f = PackedInts.GetMutable(totalFields, CompressingTermVectorsWriter.FLAGS_BITS, PackedInts.COMPACT);
for (int i = 0; i < totalFields; ++i)
{
int fieldNumOff = (int)allFieldNumOffs.Get(i);
Debug.Assert(fieldNumOff >= 0 && fieldNumOff < fieldNums.Length);
int fgs = (int)fieldFlags.Get(fieldNumOff);
f.Set(i, fgs);
}
flags = f;
break;
case 1:
flags = PackedInts.GetReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, CompressingTermVectorsWriter.FLAGS_BITS);
break;
default:
throw new Exception();
}
for (int i = 0; i < numFields; ++i)
{
fieldNumOffs[i] = (int)allFieldNumOffs.Get(skip + i);
}
}
// number of terms per field for all fields
PackedInts.Reader numTerms;
int totalTerms;
{
int bitsRequired = vectorsStream.ReadVInt();
numTerms = PackedInts.GetReaderNoHeader(vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsRequired);
int sum = 0;
for (int i = 0; i < totalFields; ++i)
{
sum += (int)numTerms.Get(i);
}
totalTerms = sum;
}
// term lengths
int docOff = 0, docLen = 0, totalLen;
int[] fieldLengths = new int[numFields];
int[][] prefixLengths = new int[numFields][];
int[][] suffixLengths = new int[numFields][];
{
reader.Reset(vectorsStream, totalTerms);
// skip
int toSkip = 0;
for (int i = 0; i < skip; ++i)
{
toSkip += (int)numTerms.Get(i);
}
reader.Skip(toSkip);
// read prefix lengths
for (int i = 0; i < numFields; ++i)
{
int termCount = (int)numTerms.Get(skip + i);
int[] fieldPrefixLengths = new int[termCount];
prefixLengths[i] = fieldPrefixLengths;
for (int j = 0; j < termCount;)
{
LongsRef next = reader.Next(termCount - j);
for (int k = 0; k < next.Length; ++k)
{
fieldPrefixLengths[j++] = (int)next.Longs[next.Offset + k];
}
}
}
reader.Skip(totalTerms - reader.Ord());
reader.Reset(vectorsStream, totalTerms);
// skip
toSkip = 0;
for (int i = 0; i < skip; ++i)
{
for (int j = 0; j < numTerms.Get(i); ++j)
{
docOff += (int)reader.Next();
}
}
for (int i = 0; i < numFields; ++i)
{
int termCount = (int)numTerms.Get(skip + i);
int[] fieldSuffixLengths = new int[termCount];
suffixLengths[i] = fieldSuffixLengths;
for (int j = 0; j < termCount;)
{
LongsRef next = reader.Next(termCount - j);
for (int k = 0; k < next.Length; ++k)
{
fieldSuffixLengths[j++] = (int)next.Longs[next.Offset + k];
}
}
fieldLengths[i] = Sum(suffixLengths[i]);
docLen += fieldLengths[i];
}
totalLen = docOff + docLen;
for (int i = skip + numFields; i < totalFields; ++i)
{
for (int j = 0; j < numTerms.Get(i); ++j)
{
totalLen += (int)reader.Next();
}
}
}
// term freqs
int[] termFreqs = new int[totalTerms];
{
reader.Reset(vectorsStream, totalTerms);
for (int i = 0; i < totalTerms;)
{
LongsRef next = reader.Next(totalTerms - i);
for (int k = 0; k < next.Length; ++k)
{
termFreqs[i++] = 1 + (int)next.Longs[next.Offset + k];
}
}
}
// total number of positions, offsets and payloads
int totalPositions = 0, totalOffsets = 0, totalPayloads = 0;
for (int i = 0, termIndex = 0; i < totalFields; ++i)
{
int f = (int)flags.Get(i);
int termCount = (int)numTerms.Get(i);
for (int j = 0; j < termCount; ++j)
{
int freq = termFreqs[termIndex++];
if ((f & CompressingTermVectorsWriter.POSITIONS) != 0)
{
totalPositions += freq;
}
if ((f & CompressingTermVectorsWriter.OFFSETS) != 0)
{
totalOffsets += freq;
}
if ((f & CompressingTermVectorsWriter.PAYLOADS) != 0)
{
totalPayloads += freq;
}
}
Debug.Assert(i != totalFields - 1 || termIndex == totalTerms, termIndex + " " + totalTerms);
}
int[][] positionIndex = PositionIndex(skip, numFields, numTerms, termFreqs);
int[][] positions, startOffsets, lengths;
if (totalPositions > 0)
{
positions = ReadPositions(skip, numFields, flags, numTerms, termFreqs, CompressingTermVectorsWriter.POSITIONS, totalPositions, positionIndex);
}
else
{
positions = new int[numFields][];
}
if (totalOffsets > 0)
{
// average number of chars per term
float[] charsPerTerm = new float[fieldNums.Length];
for (int i = 0; i < charsPerTerm.Length; ++i)
{
charsPerTerm[i] = Number.IntBitsToFloat(vectorsStream.ReadInt());
}
startOffsets = ReadPositions(skip, numFields, flags, numTerms, termFreqs, CompressingTermVectorsWriter.OFFSETS, totalOffsets, positionIndex);
lengths = ReadPositions(skip, numFields, flags, numTerms, termFreqs, CompressingTermVectorsWriter.OFFSETS, totalOffsets, positionIndex);
for (int i = 0; i < numFields; ++i)
{
int[] fStartOffsets = startOffsets[i];
int[] fPositions = positions[i];
// patch offsets from positions
if (fStartOffsets != null && fPositions != null)
{
float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]];
for (int j = 0; j < startOffsets[i].Length; ++j)
{
fStartOffsets[j] += (int)(fieldCharsPerTerm * fPositions[j]);
}
}
if (fStartOffsets != null)
{
int[] fPrefixLengths = prefixLengths[i];
int[] fSuffixLengths = suffixLengths[i];
int[] fLengths = lengths[i];
for (int j = 0, end = (int)numTerms.Get(skip + i); j < end; ++j)
{
// delta-decode start offsets and patch lengths using term lengths
int termLength = fPrefixLengths[j] + fSuffixLengths[j];
lengths[i][positionIndex[i][j]] += termLength;
for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k)
{
fStartOffsets[k] += fStartOffsets[k - 1];
fLengths[k] += termLength;
}
}
}
}
}
else
{
startOffsets = lengths = new int[numFields][];
}
if (totalPositions > 0)
{
// delta-decode positions
for (int i = 0; i < numFields; ++i)
{
int[] fPositions = positions[i];
int[] fpositionIndex = positionIndex[i];
if (fPositions != null)
{
for (int j = 0, end = (int)numTerms.Get(skip + i); j < end; ++j)
{
// delta-decode start offsets
for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k)
{
fPositions[k] += fPositions[k - 1];
}
}
}
}
}
// payload lengths
int[][] payloadIndex = new int[numFields][];
int totalPayloadLength = 0;
int payloadOff = 0;
int payloadLen = 0;
if (totalPayloads > 0)
{
reader.Reset(vectorsStream, totalPayloads);
// skip
int termIndex = 0;
for (int i = 0; i < skip; ++i)
{
int f = (int)flags.Get(i);
int termCount = (int)numTerms.Get(i);
if ((f & CompressingTermVectorsWriter.PAYLOADS) != 0)
{
for (int j = 0; j < termCount; ++j)
{
int freq = termFreqs[termIndex + j];
for (int k = 0; k < freq; ++k)
{
int l = (int)reader.Next();
payloadOff += l;
}
}
}
termIndex += termCount;
}
totalPayloadLength = payloadOff;
// read doc payload lengths
for (int i = 0; i < numFields; ++i)
{
int f = (int)flags.Get(skip + i);
int termCount = (int)numTerms.Get(skip + i);
if ((f & CompressingTermVectorsWriter.PAYLOADS) != 0)
{
int totalFreq = positionIndex[i][termCount];
payloadIndex[i] = new int[totalFreq + 1];
int posIdx = 0;
payloadIndex[i][posIdx] = payloadLen;
for (int j = 0; j < termCount; ++j)
{
int freq = termFreqs[termIndex + j];
for (int k = 0; k < freq; ++k)
{
int payloadLength = (int)reader.Next();
payloadLen += payloadLength;
payloadIndex[i][posIdx + 1] = payloadLen;
++posIdx;
}
}
Debug.Assert(posIdx == totalFreq);
}
termIndex += termCount;
}
totalPayloadLength += payloadLen;
for (int i = skip + numFields; i < totalFields; ++i)
{
int f = (int)flags.Get(i);
int termCount = (int)numTerms.Get(i);
if ((f & CompressingTermVectorsWriter.PAYLOADS) != 0)
{
for (int j = 0; j < termCount; ++j)
{
int freq = termFreqs[termIndex + j];
for (int k = 0; k < freq; ++k)
{
totalPayloadLength += (int)reader.Next();
}
}
}
termIndex += termCount;
}
Debug.Assert(termIndex == totalTerms, termIndex + " " + totalTerms);
}
// decompress data
BytesRef suffixBytes = new BytesRef();
decompressor.Decompress(vectorsStream, totalLen + totalPayloadLength, docOff + payloadOff, docLen + payloadLen, suffixBytes);
suffixBytes.Length = docLen;
BytesRef payloadBytes = new BytesRef(suffixBytes.Bytes, suffixBytes.Offset + docLen, payloadLen);
int[] FieldFlags = new int[numFields];
for (int i = 0; i < numFields; ++i)
{
FieldFlags[i] = (int)flags.Get(skip + i);
}
int[] fieldNumTerms = new int[numFields];
for (int i = 0; i < numFields; ++i)
{
fieldNumTerms[i] = (int)numTerms.Get(skip + i);
}
int[][] fieldTermFreqs = new int[numFields][];
{
int termIdx = 0;
for (int i = 0; i < skip; ++i)
{
termIdx += (int)numTerms.Get(i);
}
for (int i = 0; i < numFields; ++i)
{
int termCount = (int)numTerms.Get(skip + i);
fieldTermFreqs[i] = new int[termCount];
for (int j = 0; j < termCount; ++j)
{
fieldTermFreqs[i][j] = termFreqs[termIdx++];
}
}
}
Debug.Assert(Sum(fieldLengths) == docLen, Sum(fieldLengths) + " != " + docLen);
return(new TVFields(this, fieldNums, FieldFlags, fieldNumOffs, fieldNumTerms, fieldLengths, prefixLengths, suffixLengths, fieldTermFreqs, positionIndex, positions, startOffsets, lengths, payloadBytes, payloadIndex, suffixBytes));
}
