public static Packed64SingleBlock Create(DataInput @in, int valueCount, int bitsPerValue)
{
Packed64SingleBlock reader = Create(valueCount, bitsPerValue);
for (int i = 0; i < reader.Blocks.Length; ++i)
{
reader.Blocks[i] = @in.ReadLong();
}
return reader;
}
internal Packed8ThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount)
: this(valueCount)
{
@in.ReadBytes(Blocks, 0, 3 * valueCount);
// because packed ints have not always been byte-aligned
var remaining = (int)(PackedInts.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 24) - 3L * valueCount * 1);
for (int i = 0; i < remaining; ++i)
{
@in.ReadByte();
}
}
internal Packed16ThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount)
: this(valueCount)
{
for (int i = 0; i < 3 * valueCount; ++i)
{
Blocks[i] = @in.ReadShort();
}
// because packed ints have not always been byte-aligned
int remaining = (int)(PackedInts.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 48) - 3L * valueCount * 2);
for (int i = 0; i < remaining; ++i)
{
@in.ReadByte();
}
}
internal Direct32(int packedIntsVersion, DataInput @in, int valueCount)
: this(valueCount)
{
for (int i = 0; i < valueCount; ++i)
{
Values[i] = @in.ReadInt();
}
// because packed ints have not always been byte-aligned
int remaining = (int)(PackedInts.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 32) - 4L * valueCount);
for (int i = 0; i < remaining; ++i)
{
@in.ReadByte();
}
}
public override void AddProx(int numProx, DataInput positions, DataInput offsets)
{
if (Payloads)
{
// TODO, maybe overkill and just call super.addProx() in this case?
// we do avoid buffering the offsets in RAM though.
for (int i = 0; i < numProx; i++)
{
int code = positions.ReadVInt();
if ((code & 1) == 1)
{
int length = positions.ReadVInt();
Scratch.Grow(length);
Scratch.Length = length;
positions.ReadBytes(Scratch.Bytes, Scratch.Offset, Scratch.Length);
WritePosition((int)((uint)code >> 1), Scratch);
}
else
{
WritePosition((int)((uint)code >> 1), null);
}
}
Tvf.WriteBytes(PayloadData.Bytes, PayloadData.Offset, PayloadData.Length);
}
else if (positions != null)
{
// pure positions, no payloads
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt((int)((uint)positions.ReadVInt() >> 1));
}
}
if (offsets != null)
{
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt(offsets.ReadVInt());
Tvf.WriteVInt(offsets.ReadVInt());
}
}
}
/// <summary>
/// Decompress at least <code>decompressedLen</code> bytes into
/// <code>dest[dOff:]</code>. Please note that <code>dest</code> must be large
/// enough to be able to hold <b>all</b> decompressed data (meaning that you
/// need to know the total decompressed length).
/// </summary>
public static int Decompress(DataInput compressed, int decompressedLen, byte[] dest, int dOff)
{
int destEnd = dest.Length;
do
{
// literals
int token = compressed.ReadByte() & 0xFF;
int literalLen = (int)(((uint)token) >> 4);
if (literalLen != 0)
{
if (literalLen == 0x0F)
{
byte len;
while ((len = compressed.ReadByte()) == 0xFF)
{
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
compressed.ReadBytes(dest, dOff, literalLen);
dOff += literalLen;
}
if (dOff >= decompressedLen)
{
break;
}
// matchs
var byte1 = compressed.ReadByte();
var byte2 = compressed.ReadByte();
int matchDec = (byte1 & 0xFF) | ((byte2 & 0xFF) << 8);
Debug.Assert(matchDec > 0);
int matchLen = token & 0x0F;
if (matchLen == 0x0F)
{
int len;
while ((len = compressed.ReadByte()) == 0xFF)
{
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
// copying a multiple of 8 bytes can make decompression from 5% to 10% faster
int fastLen = (int)((matchLen + 7) & 0xFFFFFFF8);
if (matchDec < matchLen || dOff + fastLen > destEnd)
{
// overlap -> naive incremental copy
for (int @ref = dOff - matchDec, end = dOff + matchLen; dOff < end; ++@ref, ++dOff)
{
dest[dOff] = dest[@ref];
}
}
else
{
// no overlap -> arraycopy
Array.Copy(dest, dOff - matchDec, dest, dOff, fastLen);
dOff += matchLen;
}
} while (dOff < decompressedLen);
return dOff;
}
/// <summary> /// Actually decode metadata for next term </summary> /// <seealso cref= PostingsWriterBase#encodeTerm </seealso> public abstract void DecodeTerm(long[] longs, DataInput @in, FieldInfo fieldInfo, BlockTermState state, bool absolute);
/// <summary>
/// Expert: reads only the metadata from a stream. this is useful to later
/// restore a stream or open a direct reader via
/// <seealso cref="#getReaderNoHeader(DataInput, Header)"/>
/// or <seealso cref="#getDirectReaderNoHeader(IndexInput, Header)"/>. </summary>
/// <param name="in"> the stream to read data </param>
/// <returns> packed integer metadata. </returns>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
/// <seealso cref= #getReaderNoHeader(DataInput, Header) </seealso>
/// <seealso cref= #getDirectReaderNoHeader(IndexInput, Header) </seealso>
public static Header ReadHeader(DataInput @in)
{
int version = CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
int bitsPerValue = @in.ReadVInt();
Debug.Assert(bitsPerValue > 0 && bitsPerValue <= 64, "bitsPerValue=" + bitsPerValue);
int valueCount = @in.ReadVInt();
Format format = Format.ById(@in.ReadVInt());
return new Header(format, valueCount, bitsPerValue, version);
}
public override void Decompress(DataInput @in, int originalLength, int offset, int length, BytesRef bytes)
{
Debug.Assert(offset + length <= originalLength);
if (bytes.Bytes.Length < originalLength)
{
bytes.Bytes = new byte[ArrayUtil.Oversize(originalLength, 1)];
}
@in.ReadBytes(bytes.Bytes, 0, offset + length);
bytes.Offset = offset;
bytes.Length = length;
}
/// <summary>
/// Retrieve PackedInts as a <seealso cref="ReaderIterator"/> </summary>
/// <param name="in"> positioned at the beginning of a stored packed int structure. </param>
/// <param name="mem"> how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration) </param>
/// <returns> an iterator to access the values </returns>
/// <exception cref="IOException"> if the structure could not be retrieved.
/// @lucene.internal </exception>
public static ReaderIterator GetReaderIterator(DataInput @in, int mem)
{
int version = CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
int bitsPerValue = @in.ReadVInt();
Debug.Assert(bitsPerValue > 0 && bitsPerValue <= 64, "bitsPerValue=" + bitsPerValue);
int valueCount = @in.ReadVInt();
Format format = Format.ById(@in.ReadVInt());
return GetReaderIteratorNoHeader(@in, format, version, valueCount, bitsPerValue, mem);
}
protected internal ReaderIteratorImpl(int valueCount, int bitsPerValue, DataInput @in)
{
this.@in = @in;
this.bitsPerValue = bitsPerValue;
this.valueCount = valueCount;
}
private void CheckReads(DataInput @is, Type expectedEx)
{
Assert.AreEqual(128, @is.ReadVInt());
Assert.AreEqual(16383, @is.ReadVInt());
Assert.AreEqual(16384, @is.ReadVInt());
Assert.AreEqual(16385, @is.ReadVInt());
Assert.AreEqual(int.MaxValue, @is.ReadVInt());
Assert.AreEqual(-1, @is.ReadVInt());
Assert.AreEqual((long)int.MaxValue, @is.ReadVLong());
Assert.AreEqual(long.MaxValue, @is.ReadVLong());
Assert.AreEqual("Lucene", @is.ReadString());
Assert.AreEqual("\u00BF", @is.ReadString());
Assert.AreEqual("Lu\u00BFce\u00BFne", @is.ReadString());
Assert.AreEqual("\u2620", @is.ReadString());
Assert.AreEqual("Lu\u2620ce\u2620ne", @is.ReadString());
Assert.AreEqual("\uD834\uDD1E", @is.ReadString());
Assert.AreEqual("\uD834\uDD1E\uD834\uDD60", @is.ReadString());
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne", @is.ReadString());
Assert.AreEqual("\u0000", @is.ReadString());
Assert.AreEqual("Lu\u0000ce\u0000ne", @is.ReadString());
try
{
@is.ReadVInt();
Assert.Fail("Should throw " + expectedEx.Name);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.StartsWith("Invalid vInt"));
Assert.IsTrue(expectedEx.IsInstanceOfType(e));
}
Assert.AreEqual(1, @is.ReadVInt()); // guard value
try
{
@is.ReadVLong();
Assert.Fail("Should throw " + expectedEx.Name);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.StartsWith("Invalid vLong"));
Assert.IsTrue(expectedEx.IsInstanceOfType(e));
}
Assert.AreEqual(1L, @is.ReadVLong()); // guard value
}
private void CheckRandomReads(DataInput @is)
{
for (int i = 0; i < COUNT; i++)
{
Assert.AreEqual(INTS[i], @is.ReadVInt());
Assert.AreEqual(INTS[i], @is.ReadInt());
Assert.AreEqual(LONGS[i], @is.ReadVLong());
Assert.AreEqual(LONGS[i], @is.ReadLong());
}
}
public override void CopyBytes(DataInput input, long numBytes)
{
@delegate.CopyBytes(input, numBytes);
}
/// <summary> /// Actually decode metadata for next term </summary> /// <seealso cref= PostingsWriterBase#encodeTerm </seealso> public abstract void DecodeTerm(long[] longs, DataInput @in, FieldInfo fieldInfo, BlockTermState state, bool absolute);
/// <summary>
/// Decode an output value previously written with {@link
/// #write(Object, DataOutput)}.
/// </summary>
public abstract T Read(DataInput @in);
/// <summary>
/// Decode an output value previously written with {@link
/// #writeFinalOutput(Object, DataOutput)}. By default this
/// just calls <seealso cref="#read(DataInput)"/>.
/// </summary>
public virtual T ReadFinalOutput(DataInput @in)
{
return(Read(@in));
}
/// <summary>
/// Expert: Restore a <seealso cref="ReaderIterator"/> from a stream without reading
/// metadata at the beginning of the stream. this method is useful to restore
/// data from streams which have been created using
/// <seealso cref="PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)"/>.
/// </summary>
/// <param name="in"> the stream to read data from, positioned at the beginning of the packed values </param>
/// <param name="format"> the format used to serialize </param>
/// <param name="version"> the version used to serialize the data </param>
/// <param name="valueCount"> how many values the stream holds </param>
/// <param name="bitsPerValue"> the number of bits per value </param>
/// <param name="mem"> how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration) </param>
/// <returns> a ReaderIterator </returns>
/// <seealso cref= PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)
/// @lucene.internal </seealso>
public static ReaderIterator GetReaderIteratorNoHeader(DataInput @in, Format format, int version, int valueCount, int bitsPerValue, int mem)
{
CheckVersion(version);
return new PackedReaderIterator(format, version, valueCount, bitsPerValue, @in, mem);
}
/// <summary>
/// Create a new instance that wraps <code>in</code>.
/// </summary>
public PackedDataInput(DataInput @in)
{
this.@in = @in;
SkipToNextByte();
}
/// <summary>
/// Expert: Restore a <seealso cref="Reader"/> from a stream without reading metadata at
/// the beginning of the stream. this method is useful to restore data from
/// streams which have been created using
/// <seealso cref="PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)"/>.
/// </summary>
/// <param name="in"> the stream to read data from, positioned at the beginning of the packed values </param>
/// <param name="format"> the format used to serialize </param>
/// <param name="version"> the version used to serialize the data </param>
/// <param name="valueCount"> how many values the stream holds </param>
/// <param name="bitsPerValue"> the number of bits per value </param>
/// <returns> a Reader </returns>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
/// <seealso cref= PackedInts#getWriterNoHeader(DataOutput, Format, int, int, int)
/// @lucene.internal </seealso>
public static Reader GetReaderNoHeader(DataInput @in, Format format, int version, int valueCount, int bitsPerValue)
{
CheckVersion(version);
if (format == PackedInts.Format.PACKED_SINGLE_BLOCK)
{
return Packed64SingleBlock.Create(@in, valueCount, bitsPerValue);
}
else if (format == PackedInts.Format.PACKED)
{
switch (bitsPerValue)
{
case 8:
return new Direct8(version, @in, valueCount);
case 16:
return new Direct16(version, @in, valueCount);
case 32:
return new Direct32(version, @in, valueCount);
case 64:
return new Direct64(version, @in, valueCount);
case 24:
if (valueCount <= Packed8ThreeBlocks.MAX_SIZE)
{
return new Packed8ThreeBlocks(version, @in, valueCount);
}
break;
case 48:
if (valueCount <= Packed16ThreeBlocks.MAX_SIZE)
{
return new Packed16ThreeBlocks(version, @in, valueCount);
}
break;
}
return new Packed64(version, @in, valueCount, bitsPerValue);
}
else
{
throw new InvalidOperationException("Unknown Writer format: " + format);
}
}
/// <summary>
/// Expert: Restore a <seealso cref="Reader"/> from a stream without reading metadata at
/// the beginning of the stream. this method is useful to restore data when
/// metadata has been previously read using <seealso cref="#readHeader(DataInput)"/>.
/// </summary>
/// <param name="in"> the stream to read data from, positioned at the beginning of the packed values </param>
/// <param name="header"> metadata result from <code>readHeader()</code> </param>
/// <returns> a Reader </returns>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
/// <seealso cref= #readHeader(DataInput)
/// @lucene.internal </seealso>
public static Reader GetReaderNoHeader(DataInput @in, Header header)
{
return GetReaderNoHeader(@in, header.format, header.version, header.valueCount, header.bitsPerValue);
}
/// <summary>
/// Called by IndexWriter when writing new segments.
/// <p>
/// this is an expert API that allows the codec to consume
/// positions and offsets directly from the indexer.
/// <p>
/// The default implementation calls <seealso cref="#addPosition(int, int, int, BytesRef)"/>,
/// but subclasses can override this if they want to efficiently write
/// all the positions, then all the offsets, for example.
/// <p>
/// NOTE: this API is extremely expert and subject to change or removal!!!
/// @lucene.internal
/// </summary>
// TODO: we should probably nuke this and make a more efficient 4.x format
// PreFlex-RW could then be slow and buffer (its only used in tests...)
public virtual void AddProx(int numProx, DataInput positions, DataInput offsets)
{
int position = 0;
int lastOffset = 0;
BytesRef payload = null;
for (int i = 0; i < numProx; i++)
{
int startOffset;
int endOffset;
BytesRef thisPayload;
if (positions == null)
{
position = -1;
thisPayload = null;
}
else
{
int code = positions.ReadVInt();
position += (int)((uint)code >> 1);
if ((code & 1) != 0)
{
// this position has a payload
int payloadLength = positions.ReadVInt();
if (payload == null)
{
payload = new BytesRef();
payload.Bytes = new byte[payloadLength];
}
else if (payload.Bytes.Length < payloadLength)
{
payload.Grow(payloadLength);
}
positions.ReadBytes(payload.Bytes, 0, payloadLength);
payload.Length = payloadLength;
thisPayload = payload;
}
else
{
thisPayload = null;
}
}
if (offsets == null)
{
startOffset = endOffset = -1;
}
else
{
startOffset = lastOffset + offsets.ReadVInt();
endOffset = startOffset + offsets.ReadVInt();
lastOffset = endOffset;
}
AddPosition(position, startOffset, endOffset, thisPayload);
}
}
