public override void AddProx(int numProx, DataInput positions, DataInput offsets)
{
if (Payloads)
{
// TODO, maybe overkill and just call super.addProx() in this case?
// we do avoid buffering the offsets in RAM though.
for (int i = 0; i < numProx; i++)
{
int code = positions.ReadVInt();
if ((code & 1) == 1)
{
int length = positions.ReadVInt();
Scratch.Grow(length);
Scratch.Length = length;
positions.ReadBytes(Scratch.Bytes, Scratch.Offset, Scratch.Length);
WritePosition((int)((uint)code >> 1), Scratch);
}
else
{
WritePosition((int)((uint)code >> 1), null);
}
}
Tvf.WriteBytes(PayloadData.Bytes, PayloadData.Offset, PayloadData.Length);
}
else if (positions != null)
{
// pure positions, no payloads
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt((int)((uint)positions.ReadVInt() >> 1));
}
}
if (offsets != null)
{
for (int i = 0; i < numProx; i++)
{
Tvf.WriteVInt(offsets.ReadVInt());
Tvf.WriteVInt(offsets.ReadVInt());
}
}
}
/// <summary>
/// Expert: reads only the metadata from a stream. this is useful to later
/// restore a stream or open a direct reader via
/// <seealso cref="#getReaderNoHeader(DataInput, Header)"/>
/// or <seealso cref="#getDirectReaderNoHeader(IndexInput, Header)"/>. </summary>
/// <param name="in"> the stream to read data </param>
/// <returns> packed integer metadata. </returns>
/// <exception cref="IOException"> If there is a low-level I/O error </exception>
/// <seealso cref= #getReaderNoHeader(DataInput, Header) </seealso>
/// <seealso cref= #getDirectReaderNoHeader(IndexInput, Header) </seealso>
public static Header ReadHeader(DataInput @in)
{
int version = CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
int bitsPerValue = @in.ReadVInt();
Debug.Assert(bitsPerValue > 0 && bitsPerValue <= 64, "bitsPerValue=" + bitsPerValue);
int valueCount = @in.ReadVInt();
Format format = Format.ById(@in.ReadVInt());
return new Header(format, valueCount, bitsPerValue, version);
}
/// <summary>
/// Retrieve PackedInts as a <seealso cref="ReaderIterator"/> </summary>
/// <param name="in"> positioned at the beginning of a stored packed int structure. </param>
/// <param name="mem"> how much memory the iterator is allowed to use to read-ahead (likely to speed up iteration) </param>
/// <returns> an iterator to access the values </returns>
/// <exception cref="IOException"> if the structure could not be retrieved.
/// @lucene.internal </exception>
public static ReaderIterator GetReaderIterator(DataInput @in, int mem)
{
int version = CodecUtil.CheckHeader(@in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
int bitsPerValue = @in.ReadVInt();
Debug.Assert(bitsPerValue > 0 && bitsPerValue <= 64, "bitsPerValue=" + bitsPerValue);
int valueCount = @in.ReadVInt();
Format format = Format.ById(@in.ReadVInt());
return GetReaderIteratorNoHeader(@in, format, version, valueCount, bitsPerValue, mem);
}
private void CheckReads(DataInput @is, Type expectedEx)
{
Assert.AreEqual(128, @is.ReadVInt());
Assert.AreEqual(16383, @is.ReadVInt());
Assert.AreEqual(16384, @is.ReadVInt());
Assert.AreEqual(16385, @is.ReadVInt());
Assert.AreEqual(int.MaxValue, @is.ReadVInt());
Assert.AreEqual(-1, @is.ReadVInt());
Assert.AreEqual((long)int.MaxValue, @is.ReadVLong());
Assert.AreEqual(long.MaxValue, @is.ReadVLong());
Assert.AreEqual("Lucene", @is.ReadString());
Assert.AreEqual("\u00BF", @is.ReadString());
Assert.AreEqual("Lu\u00BFce\u00BFne", @is.ReadString());
Assert.AreEqual("\u2620", @is.ReadString());
Assert.AreEqual("Lu\u2620ce\u2620ne", @is.ReadString());
Assert.AreEqual("\uD834\uDD1E", @is.ReadString());
Assert.AreEqual("\uD834\uDD1E\uD834\uDD60", @is.ReadString());
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne", @is.ReadString());
Assert.AreEqual("\u0000", @is.ReadString());
Assert.AreEqual("Lu\u0000ce\u0000ne", @is.ReadString());
try
{
@is.ReadVInt();
Assert.Fail("Should throw " + expectedEx.Name);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.StartsWith("Invalid vInt"));
Assert.IsTrue(expectedEx.IsInstanceOfType(e));
}
Assert.AreEqual(1, @is.ReadVInt()); // guard value
try
{
@is.ReadVLong();
Assert.Fail("Should throw " + expectedEx.Name);
}
catch (Exception e)
{
Assert.IsTrue(e.Message.StartsWith("Invalid vLong"));
Assert.IsTrue(expectedEx.IsInstanceOfType(e));
}
Assert.AreEqual(1L, @is.ReadVLong()); // guard value
}
private void CheckRandomReads(DataInput @is)
{
for (int i = 0; i < COUNT; i++)
{
Assert.AreEqual(INTS[i], @is.ReadVInt());
Assert.AreEqual(INTS[i], @is.ReadInt());
Assert.AreEqual(LONGS[i], @is.ReadVLong());
Assert.AreEqual(LONGS[i], @is.ReadLong());
}
}
/// <summary>
/// Called by IndexWriter when writing new segments.
/// <p>
/// this is an expert API that allows the codec to consume
/// positions and offsets directly from the indexer.
/// <p>
/// The default implementation calls <seealso cref="#addPosition(int, int, int, BytesRef)"/>,
/// but subclasses can override this if they want to efficiently write
/// all the positions, then all the offsets, for example.
/// <p>
/// NOTE: this API is extremely expert and subject to change or removal!!!
/// @lucene.internal
/// </summary>
// TODO: we should probably nuke this and make a more efficient 4.x format
// PreFlex-RW could then be slow and buffer (its only used in tests...)
public virtual void AddProx(int numProx, DataInput positions, DataInput offsets)
{
int position = 0;
int lastOffset = 0;
BytesRef payload = null;
for (int i = 0; i < numProx; i++)
{
int startOffset;
int endOffset;
BytesRef thisPayload;
if (positions == null)
{
position = -1;
thisPayload = null;
}
else
{
int code = positions.ReadVInt();
position += (int)((uint)code >> 1);
if ((code & 1) != 0)
{
// this position has a payload
int payloadLength = positions.ReadVInt();
if (payload == null)
{
payload = new BytesRef();
payload.Bytes = new byte[payloadLength];
}
else if (payload.Bytes.Length < payloadLength)
{
payload.Grow(payloadLength);
}
positions.ReadBytes(payload.Bytes, 0, payloadLength);
payload.Length = payloadLength;
thisPayload = payload;
}
else
{
thisPayload = null;
}
}
if (offsets == null)
{
startOffset = endOffset = -1;
}
else
{
startOffset = lastOffset + offsets.ReadVInt();
endOffset = startOffset + offsets.ReadVInt();
lastOffset = endOffset;
}
AddPosition(position, startOffset, endOffset, thisPayload);
}
}