internal Packed8ThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount) : this(valueCount) { @in.ReadBytes(Blocks, 0, 3 * valueCount); // because packed ints have not always been byte-aligned var remaining = (int)(PackedInts.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 24) - 3L * valueCount * 1); for (int i = 0; i < remaining; ++i) { @in.ReadByte(); } }
public override void Decompress(DataInput @in, int originalLength, int offset, int length, BytesRef bytes) { Debug.Assert(offset + length <= originalLength); if (bytes.Bytes.Length < originalLength) { bytes.Bytes = new byte[ArrayUtil.Oversize(originalLength, 1)]; } @in.ReadBytes(bytes.Bytes, 0, offset + length); bytes.Offset = offset; bytes.Length = length; }
/// <summary> /// Decompress at least <code>decompressedLen</code> bytes into /// <code>dest[dOff:]</code>. Please note that <code>dest</code> must be large /// enough to be able to hold <b>all</b> decompressed data (meaning that you /// need to know the total decompressed length). /// </summary> public static int Decompress(DataInput compressed, int decompressedLen, byte[] dest, int dOff) { int destEnd = dest.Length; do { // literals int token = compressed.ReadByte() & 0xFF; int literalLen = (int)(((uint)token) >> 4); if (literalLen != 0) { if (literalLen == 0x0F) { byte len; while ((len = compressed.ReadByte()) == 0xFF) { literalLen += 0xFF; } literalLen += len & 0xFF; } compressed.ReadBytes(dest, dOff, literalLen); dOff += literalLen; } if (dOff >= decompressedLen) { break; } // matchs var byte1 = compressed.ReadByte(); var byte2 = compressed.ReadByte(); int matchDec = (byte1 & 0xFF) | ((byte2 & 0xFF) << 8); Debug.Assert(matchDec > 0); int matchLen = token & 0x0F; if (matchLen == 0x0F) { int len; while ((len = compressed.ReadByte()) == 0xFF) { matchLen += 0xFF; } matchLen += len & 0xFF; } matchLen += MIN_MATCH; // copying a multiple of 8 bytes can make decompression from 5% to 10% faster int fastLen = (int)((matchLen + 7) & 0xFFFFFFF8); if (matchDec < matchLen || dOff + fastLen > destEnd) { // overlap -> naive incremental copy for (int @ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++@ref, ++dOff) { dest[dOff] = dest[@ref]; } } else { // no overlap -> arraycopy Array.Copy(dest, dOff - matchDec, dest, dOff, fastLen); dOff += matchLen; } } while (dOff < decompressedLen); return dOff; }
public override void AddProx(int numProx, DataInput positions, DataInput offsets) { if (Payloads) { // TODO, maybe overkill and just call super.addProx() in this case? // we do avoid buffering the offsets in RAM though. for (int i = 0; i < numProx; i++) { int code = positions.ReadVInt(); if ((code & 1) == 1) { int length = positions.ReadVInt(); Scratch.Grow(length); Scratch.Length = length; positions.ReadBytes(Scratch.Bytes, Scratch.Offset, Scratch.Length); WritePosition((int)((uint)code >> 1), Scratch); } else { WritePosition((int)((uint)code >> 1), null); } } Tvf.WriteBytes(PayloadData.Bytes, PayloadData.Offset, PayloadData.Length); } else if (positions != null) { // pure positions, no payloads for (int i = 0; i < numProx; i++) { Tvf.WriteVInt((int)((uint)positions.ReadVInt() >> 1)); } } if (offsets != null) { for (int i = 0; i < numProx; i++) { Tvf.WriteVInt(offsets.ReadVInt()); Tvf.WriteVInt(offsets.ReadVInt()); } } }
/// <summary> /// Called by IndexWriter when writing new segments. /// <p> /// this is an expert API that allows the codec to consume /// positions and offsets directly from the indexer. /// <p> /// The default implementation calls <seealso cref="#addPosition(int, int, int, BytesRef)"/>, /// but subclasses can override this if they want to efficiently write /// all the positions, then all the offsets, for example. /// <p> /// NOTE: this API is extremely expert and subject to change or removal!!! /// @lucene.internal /// </summary> // TODO: we should probably nuke this and make a more efficient 4.x format // PreFlex-RW could then be slow and buffer (its only used in tests...) public virtual void AddProx(int numProx, DataInput positions, DataInput offsets) { int position = 0; int lastOffset = 0; BytesRef payload = null; for (int i = 0; i < numProx; i++) { int startOffset; int endOffset; BytesRef thisPayload; if (positions == null) { position = -1; thisPayload = null; } else { int code = positions.ReadVInt(); position += (int)((uint)code >> 1); if ((code & 1) != 0) { // this position has a payload int payloadLength = positions.ReadVInt(); if (payload == null) { payload = new BytesRef(); payload.Bytes = new byte[payloadLength]; } else if (payload.Bytes.Length < payloadLength) { payload.Grow(payloadLength); } positions.ReadBytes(payload.Bytes, 0, payloadLength); payload.Length = payloadLength; thisPayload = payload; } else { thisPayload = null; } } if (offsets == null) { startOffset = endOffset = -1; } else { startOffset = lastOffset + offsets.ReadVInt(); endOffset = startOffset + offsets.ReadVInt(); lastOffset = endOffset; } AddPosition(position, startOffset, endOffset, thisPayload); } }