public override int NextDoc()
{
while (true)
{
SkipPositions();
if (_postings.Eof())
{
return(_docId = NO_MORE_DOCS);
}
var code = _postings.ReadVInt();
_accum += (int)((uint)code >> 1); // shift off low bit
_freq = (code & 1) != 0 ? 1 : _postings.ReadVInt();
_posPending = _freq;
_startOffset = _storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
if (_liveDocs != null && !_liveDocs.Get(_accum))
{
continue;
}
_position = 0;
return(_docId = _accum);
}
}
/// <summary>
/// Load current stats shard </summary>
internal void RefillStats()
{
var offset = statsBlockOrd * outerInstance.numSkipInfo;
var statsFP = (int)outerInstance.skipInfo[offset];
statsReader.Position = statsFP;
for (int i = 0; i < INTERVAL && !statsReader.Eof(); i++)
{
int code = statsReader.ReadVInt();
if (outerInstance.HasFreqs())
{
docFreq_Renamed[i] = ((int)((uint)code >> 1));
if ((code & 1) == 1)
{
totalTermFreq_Renamed[i] = docFreq_Renamed[i];
}
else
{
totalTermFreq_Renamed[i] = docFreq_Renamed[i] + statsReader.ReadVLong();
}
}
else
{
docFreq_Renamed[i] = code;
}
}
}
public override long NextOrd()
{
if (Input.Eof())
{
return(NO_MORE_ORDS);
}
else
{
currentOrd += Input.ReadVLong();
return(currentOrd);
}
}
/// <summary>
/// Load current metadata shard </summary>
internal void RefillMetadata()
{
var offset = metaBlockOrd * outerInstance.numSkipInfo;
var metaLongsFP = (int)outerInstance.skipInfo[offset + 1];
var metaBytesFP = (int)outerInstance.skipInfo[offset + 2];
metaLongsReader.Position = metaLongsFP;
for (int j = 0; j < outerInstance.longsSize; j++)
{
longs[0][j] = outerInstance.skipInfo[offset + 3 + j] + metaLongsReader.ReadVLong();
}
bytesStart[0] = metaBytesFP;
bytesLength[0] = (int)metaLongsReader.ReadVLong();
for (int i = 1; i < INTERVAL && !metaLongsReader.Eof(); i++)
{
for (int j = 0; j < outerInstance.longsSize; j++)
{
longs[i][j] = longs[i - 1][j] + metaLongsReader.ReadVLong();
}
bytesStart[i] = bytesStart[i - 1] + bytesLength[i - 1];
bytesLength[i] = (int)metaLongsReader.ReadVLong();
}
}
internal virtual bool ReadSequence()
{
if (@in.Eof())
{
WordNum = int.MaxValue;
return(false);
}
int token = @in.ReadByte() & 0xFF;
if ((token & (1 << 7)) == 0)
{
int cleanLength = ReadCleanLength(@in, token);
WordNum += cleanLength;
}
else
{
AllOnesLength = ReadCleanLength(@in, token);
}
DirtyLength = ReadDirtyLength(@in, token);
Debug.Assert(@in.Length() - @in.Position >= DirtyLength, @in.Position + " " + @in.Length() + " " + DirtyLength);
++SequenceNum;
return(true);
}
public override int NextDoc()
{
while (true)
{
if (_postings.Eof())
{
return(_docId = NO_MORE_DOCS);
}
var code = _postings.ReadVInt();
if (_indexOptions == FieldInfo.IndexOptions.DOCS_ONLY)
{
_accum += code;
}
else
{
_accum += (int)((uint)code >> 1);; // shift off low bit
_freq = (code & 1) != 0 ? 1 : _postings.ReadVInt();
if (_indexOptions.Value.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0)
{
// Skip positions
if (_storePayloads)
{
for (var pos = 0; pos < _freq; pos++)
{
var posCode = _postings.ReadVInt();
if ((posCode & 1) != 0)
{
_payloadLength = _postings.ReadVInt();
}
if (_storeOffsets && (_postings.ReadVInt() & 1) != 0)
{
// new offset length
_postings.ReadVInt();
}
if (_payloadLength != 0)
{
_postings.SkipBytes(_payloadLength);
}
}
}
else
{
for (var pos = 0; pos < _freq; pos++)
{
// TODO: skipVInt
_postings.ReadVInt();
if (_storeOffsets && (_postings.ReadVInt() & 1) != 0)
{
// new offset length
_postings.ReadVInt();
}
}
}
}
}
if (_liveDocs == null || _liveDocs.Get(_accum))
{
return(_docId = _accum);
}
}
}
public virtual void TestVariableBinary()
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BVariableBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetRAMBufferSizeMB(256.0).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(false, 10)).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE));
Document doc = new Document();
sbyte[] bytes = new sbyte[4];
ByteArrayDataOutput encoder = new ByteArrayDataOutput((byte[])(Array)bytes);
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
encoder.Reset((byte[])(Array)bytes);
encoder.WriteVInt(i % 65535); // 1, 2, or 3 bytes
data.Length = encoder.Position;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
ByteArrayDataInput input = new ByteArrayDataInput();
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
dv.Get(i, scratch);
input.Reset((byte[])(Array)scratch.Bytes, scratch.Offset, scratch.Length);
Assert.AreEqual(expectedValue % 65535, input.ReadVInt());
Assert.IsTrue(input.Eof());
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestVariableBinary([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BVariableBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(scheduler)
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE);
IndexWriter w = new IndexWriter(dir, config);
Document doc = new Document();
var bytes = new byte[4];
ByteArrayDataOutput encoder = new ByteArrayDataOutput(bytes);
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
encoder.Reset(bytes);
encoder.WriteVInt(i % 65535); // 1, 2, or 3 bytes
data.Length = encoder.Position;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
ByteArrayDataInput input = new ByteArrayDataInput();
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
dv.Get(i, scratch);
input.Reset((byte[])(Array)scratch.Bytes, scratch.Offset, scratch.Length);
Assert.AreEqual(expectedValue % 65535, input.ReadVInt());
Assert.IsTrue(input.Eof());
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}