internal Packed8ThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount)
: this(valueCount)
{
@in.ReadBytes(Blocks, 0, 3 * valueCount);
// because packed ints have not always been byte-aligned
var remaining = (int)(PackedInts.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 24) - 3L * valueCount * 1);
for (int i = 0; i < remaining; ++i)
{
@in.ReadByte();
}
}
public override void Decompress(DataInput @in, int originalLength, int offset, int length, BytesRef bytes)
{
Debug.Assert(offset + length <= originalLength);
if (bytes.Bytes.Length < originalLength)
{
bytes.Bytes = new byte[ArrayUtil.Oversize(originalLength, 1)];
}
@in.ReadBytes(bytes.Bytes, 0, offset + length);
bytes.Offset = offset;
bytes.Length = length;
}
/// <summary>
/// Decompress at least <code>decompressedLen</code> bytes into
/// <code>dest[dOff:]</code>. Please note that <code>dest</code> must be large
/// enough to be able to hold <b>all</b> decompressed data (meaning that you
/// need to know the total decompressed length).
/// </summary>
public static int Decompress(DataInput compressed, int decompressedLen, byte[] dest, int dOff)
{
int destEnd = dest.Length;
do
{
// literals
int token = compressed.ReadByte() & 0xFF;
int literalLen = (int)(((uint)token) >> 4);
if (literalLen != 0)
{
if (literalLen == 0x0F)
{
byte len;
while ((len = compressed.ReadByte()) == 0xFF)
{
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
compressed.ReadBytes(dest, dOff, literalLen);
dOff += literalLen;
}
if (dOff >= decompressedLen)
{
break;
}
// matchs
var byte1 = compressed.ReadByte();
var byte2 = compressed.ReadByte();
int matchDec = (byte1 & 0xFF) | ((byte2 & 0xFF) << 8);
Debug.Assert(matchDec > 0);
int matchLen = token & 0x0F;
if (matchLen == 0x0F)
{
int len;
while ((len = compressed.ReadByte()) == 0xFF)
{
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
// copying a multiple of 8 bytes can make decompression from 5% to 10% faster
int fastLen = (int)((matchLen + 7) & 0xFFFFFFF8);
if (matchDec < matchLen || dOff + fastLen > destEnd)
{
// overlap -> naive incremental copy
for (int @ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++@ref, ++dOff)
{
dest[dOff] = dest[@ref];
}
}
else
{
// no overlap -> arraycopy
Array.Copy(dest, dOff - matchDec, dest, dOff, fastLen);
dOff += matchLen;
}
} while (dOff < decompressedLen);
return dOff;
}
public override void AddProx(int numProx, DataInput positions, DataInput offsets)
{
if (Payloads)
{
// TODO, maybe overkill and just call super.addProx() in this case?
// we do avoid buffering the offsets in RAM though.
for (int i = 0; i < numProx; i++)
{
int code = positions.ReadVInt();
if ((code & 1) == 1)
{
int length = positions.ReadVInt();
Scratch.Grow(length);
Scratch.Length = length;
positions.ReadBytes(Scratch.Bytes, Scratch.Offset, Scratch.Length);
WritePosition((int)((uint)code >> 1), Scratch);
}
else
{
WritePosition((int)((uint)code >> 1), null);
}
}
Tvf.WriteBytes(PayloadData.Bytes, PayloadData.Offset, PayloadData.Length);
}
else if (positions != null)
{
// pure positions, no payloads
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt((int)((uint)positions.ReadVInt() >> 1));
}
}
if (offsets != null)
{
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt(offsets.ReadVInt());
Tvf.WriteVInt(offsets.ReadVInt());
}
}
}
/// <summary>
/// Called by IndexWriter when writing new segments.
/// <p>
/// this is an expert API that allows the codec to consume
/// positions and offsets directly from the indexer.
/// <p>
/// The default implementation calls <seealso cref="#addPosition(int, int, int, BytesRef)"/>,
/// but subclasses can override this if they want to efficiently write
/// all the positions, then all the offsets, for example.
/// <p>
/// NOTE: this API is extremely expert and subject to change or removal!!!
/// @lucene.internal
/// </summary>
// TODO: we should probably nuke this and make a more efficient 4.x format
// PreFlex-RW could then be slow and buffer (its only used in tests...)
public virtual void AddProx(int numProx, DataInput positions, DataInput offsets)
{
int position = 0;
int lastOffset = 0;
BytesRef payload = null;
for (int i = 0; i < numProx; i++)
{
int startOffset;
int endOffset;
BytesRef thisPayload;
if (positions == null)
{
position = -1;
thisPayload = null;
}
else
{
int code = positions.ReadVInt();
position += (int)((uint)code >> 1);
if ((code & 1) != 0)
{
// this position has a payload
int payloadLength = positions.ReadVInt();
if (payload == null)
{
payload = new BytesRef();
payload.Bytes = new byte[payloadLength];
}
else if (payload.Bytes.Length < payloadLength)
{
payload.Grow(payloadLength);
}
positions.ReadBytes(payload.Bytes, 0, payloadLength);
payload.Length = payloadLength;
thisPayload = payload;
}
else
{
thisPayload = null;
}
}
if (offsets == null)
{
startOffset = endOffset = -1;
}
else
{
startOffset = lastOffset + offsets.ReadVInt();
endOffset = startOffset + offsets.ReadVInt();
lastOffset = endOffset;
}
AddPosition(position, startOffset, endOffset, thisPayload);
}
}