/// <summary>
/// Load current stats shard. </summary>
internal void RefillStats()
{
var offset = statsBlockOrd * outerInstance.numSkipInfo;
var statsFP = (int)outerInstance.skipInfo[offset];
statsReader.Position = statsFP;
for (int i = 0; i < INTERVAL && !statsReader.Eof; i++)
{
int code = statsReader.ReadVInt32();
if (outerInstance.HasFreqs)
{
docFreq[i] = code.TripleShift(1);
if ((code & 1) == 1)
{
totalTermFreq[i] = docFreq[i];
}
else
{
totalTermFreq[i] = docFreq[i] + statsReader.ReadVInt64();
}
}
else
{
docFreq[i] = code;
}
}
}
public override int NextDoc()
{
while (true)
{
SkipPositions();
if (_postings.Eof)
{
return(_docId = NO_MORE_DOCS);
}
var code = _postings.ReadVInt32();
_accum += (int)((uint)code >> 1); // shift off low bit
_freq = (code & 1) != 0 ? 1 : _postings.ReadVInt32();
_posPending = _freq;
_startOffset = _storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
if (_liveDocs != null && !_liveDocs.Get(_accum))
{
continue;
}
_position = 0;
return(_docId = _accum);
}
}
internal static int ReadDirtyLength(ByteArrayDataInput @in, int token)
{
int len = token & 0x0F;
if ((len & 0x08) != 0)
{
len = (len & 0x07) | (@in.ReadVInt32() << 3);
}
return(len);
}
private void DecodeMetaData()
{
//System.out.println("BTR.decodeMetadata mdUpto=" + metaDataUpto + " vs termCount=" + state.termBlockOrd + " state=" + state);
if (!seekPending)
{
// TODO: cutover to random-access API
// here.... really stupid that we have to decode N
// wasted term metadata just to get to the N+1th
// that we really need...
// lazily catch up on metadata decode:
int limit = state.TermBlockOrd;
bool absolute = metaDataUpto == 0;
// TODO: better API would be "jump straight to term=N"???
while (metaDataUpto < limit)
{
//System.out.println(" decode mdUpto=" + metaDataUpto);
// TODO: we could make "tiers" of metadata, ie,
// decode docFreq/totalTF but don't decode postings
// metadata; this way caller could get
// docFreq/totalTF w/o paying decode cost for
// postings
// TODO: if docFreq were bulk decoded we could
// just skipN here:
// docFreq, totalTermFreq
state.DocFreq = freqReader.ReadVInt32();
//System.out.println(" dF=" + state.docFreq);
if (outerInstance.fieldInfo.IndexOptions != IndexOptions.DOCS_ONLY)
{
state.TotalTermFreq = state.DocFreq + freqReader.ReadVInt64();
//System.out.println(" totTF=" + state.totalTermFreq);
}
// metadata
for (int i = 0; i < longs.Length; i++)
{
longs[i] = bytesReader.ReadVInt64();
}
outerInstance.outerInstance.postingsReader.DecodeTerm(longs, bytesReader, outerInstance.fieldInfo, state, absolute);
metaDataUpto++;
absolute = false;
}
}
else
{
//System.out.println(" skip! seekPending");
}
}
internal static int ReadCleanLength(ByteArrayDataInput @in, int token)
{
int len = ((int)((uint)token >> 4)) & 0x07;
int startPosition = @in.Position;
if ((len & 0x04) != 0)
{
len = (len & 0x03) | (@in.ReadVInt32() << 2);
}
if (startPosition != 1)
{
len += 2;
}
return(len);
}
private void DecodeMetaData()
{
if (!didDecode)
{
buffer.Reset(current.Output.Bytes, current.Output.Offset, current.Output.Length);
docFreq = buffer.ReadVInt32();
if (field.IndexOptions != IndexOptions.DOCS_ONLY)
{
totalTermFreq = docFreq + buffer.ReadVInt64();
}
else
{
totalTermFreq = -1;
}
postingsSpare.Bytes = current.Output.Bytes;
postingsSpare.Offset = buffer.Position;
postingsSpare.Length = current.Output.Length - (buffer.Position - current.Output.Offset);
//System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length);
didDecode = true;
}
}
/// <remarks>
/// TODO: we may want an alternate mode here which is
/// "if you are about to return NOT_FOUND I won't use
/// the terms data from that"; eg FuzzyTermsEnum will
/// (usually) just immediately call seek again if we
/// return NOT_FOUND so it's a waste for us to fill in
/// the term that was actually NOT_FOUND
/// </remarks>
public override SeekStatus SeekCeil(BytesRef target)
{
if (indexEnum == null)
{
throw new InvalidOperationException("terms index was not loaded");
}
//System.out.println("BTR.seek seg=" + segment + " target=" + fieldInfo.name + ":" + target.utf8ToString() + " " + target + " current=" + term().utf8ToString() + " " + term() + " indexIsCurrent=" + indexIsCurrent + " didIndexNext=" + didIndexNext + " seekPending=" + seekPending + " divisor=" + indexReader.getDivisor() + " this=" + this);
if (didIndexNext)
{
if (nextIndexTerm == null)
{
//System.out.println(" nextIndexTerm=null");
}
else
{
//System.out.println(" nextIndexTerm=" + nextIndexTerm.utf8ToString());
}
}
bool doSeek = true;
// See if we can avoid seeking, because target term
// is after current term but before next index term:
if (indexIsCurrent)
{
int cmp = BytesRef.UTF8SortedAsUnicodeComparer.Compare(term, target);
if (cmp == 0)
{
// Already at the requested term
return(SeekStatus.FOUND);
}
else if (cmp < 0)
{
// Target term is after current term
if (!didIndexNext)
{
if (indexEnum.Next() == -1)
{
nextIndexTerm = null;
}
else
{
nextIndexTerm = indexEnum.Term;
}
//System.out.println(" now do index next() nextIndexTerm=" + (nextIndexTerm == null ? "null" : nextIndexTerm.utf8ToString()));
didIndexNext = true;
}
if (nextIndexTerm == null || BytesRef.UTF8SortedAsUnicodeComparer.Compare(target, nextIndexTerm) < 0)
{
// Optimization: requested term is within the
// same term block we are now in; skip seeking
// (but do scanning):
doSeek = false;
//System.out.println(" skip seek: nextIndexTerm=" + (nextIndexTerm == null ? "null" : nextIndexTerm.utf8ToString()));
}
}
}
if (doSeek)
{
//System.out.println(" seek");
// Ask terms index to find biggest indexed term (=
// first term in a block) that's <= our text:
input.Seek(indexEnum.Seek(target));
bool result = NextBlock();
// Block must exist since, at least, the indexed term
// is in the block:
Debug.Assert(result);
indexIsCurrent = true;
didIndexNext = false;
blocksSinceSeek = 0;
if (doOrd)
{
state.Ord = indexEnum.Ord - 1;
}
term.CopyBytes(indexEnum.Term);
//System.out.println(" seek: term=" + term.utf8ToString());
}
else
{
//System.out.println(" skip seek");
if (state.TermBlockOrd == blockTermCount && !NextBlock())
{
indexIsCurrent = false;
return(SeekStatus.END);
}
}
seekPending = false;
int common = 0;
// Scan within block. We could do this by calling
// _next() and testing the resulting term, but this
// is wasteful. Instead, we first confirm the
// target matches the common prefix of this block,
// and then we scan the term bytes directly from the
// termSuffixesreader's byte[], saving a copy into
// the BytesRef term per term. Only when we return
// do we then copy the bytes into the term.
while (true)
{
// First, see if target term matches common prefix
// in this block:
if (common < termBlockPrefix)
{
int cmp = (term.Bytes[common] & 0xFF) - (target.Bytes[target.Offset + common] & 0xFF);
if (cmp < 0)
{
// TODO: maybe we should store common prefix
// in block header? (instead of relying on
// last term of previous block)
// Target's prefix is after the common block
// prefix, so term cannot be in this block
// but it could be in next block. We
// must scan to end-of-block to set common
// prefix for next block:
if (state.TermBlockOrd < blockTermCount)
{
while (state.TermBlockOrd < blockTermCount - 1)
{
state.TermBlockOrd++;
state.Ord++;
termSuffixesReader.SkipBytes(termSuffixesReader.ReadVInt32());
}
int suffix = termSuffixesReader.ReadVInt32();
term.Length = termBlockPrefix + suffix;
if (term.Bytes.Length < term.Length)
{
term.Grow(term.Length);
}
termSuffixesReader.ReadBytes(term.Bytes, termBlockPrefix, suffix);
}
state.Ord++;
if (!NextBlock())
{
indexIsCurrent = false;
return(SeekStatus.END);
}
common = 0;
}
else if (cmp > 0)
{
// Target's prefix is before the common prefix
// of this block, so we position to start of
// block and return NOT_FOUND:
Debug.Assert(state.TermBlockOrd == 0);
int suffix = termSuffixesReader.ReadVInt32();
term.Length = termBlockPrefix + suffix;
if (term.Bytes.Length < term.Length)
{
term.Grow(term.Length);
}
termSuffixesReader.ReadBytes(term.Bytes, termBlockPrefix, suffix);
return(SeekStatus.NOT_FOUND);
}
else
{
common++;
}
continue;
}
// Test every term in this block
while (true)
{
state.TermBlockOrd++;
state.Ord++;
int suffix = termSuffixesReader.ReadVInt32();
// We know the prefix matches, so just compare the new suffix:
int termLen = termBlockPrefix + suffix;
int bytePos = termSuffixesReader.Position;
bool next = false;
int limit = target.Offset + (termLen < target.Length ? termLen : target.Length);
int targetPos = target.Offset + termBlockPrefix;
while (targetPos < limit)
{
int cmp = (termSuffixes[bytePos++] & 0xFF) - (target.Bytes[targetPos++] & 0xFF);
if (cmp < 0)
{
// Current term is still before the target;
// keep scanning
next = true;
break;
}
else if (cmp > 0)
{
// Done! Current term is after target. Stop
// here, fill in real term, return NOT_FOUND.
term.Length = termBlockPrefix + suffix;
if (term.Bytes.Length < term.Length)
{
term.Grow(term.Length);
}
termSuffixesReader.ReadBytes(term.Bytes, termBlockPrefix, suffix);
//System.out.println(" NOT_FOUND");
return(SeekStatus.NOT_FOUND);
}
}
if (!next && target.Length <= termLen)
{
term.Length = termBlockPrefix + suffix;
if (term.Bytes.Length < term.Length)
{
term.Grow(term.Length);
}
termSuffixesReader.ReadBytes(term.Bytes, termBlockPrefix, suffix);
if (target.Length == termLen)
{
// Done! Exact match. Stop here, fill in
// real term, return FOUND.
//System.out.println(" FOUND");
return(SeekStatus.FOUND);
}
else
{
//System.out.println(" NOT_FOUND");
return(SeekStatus.NOT_FOUND);
}
}
if (state.TermBlockOrd == blockTermCount)
{
// Must pre-fill term for next block's common prefix
term.Length = termBlockPrefix + suffix;
if (term.Bytes.Length < term.Length)
{
term.Grow(term.Length);
}
termSuffixesReader.ReadBytes(term.Bytes, termBlockPrefix, suffix);
break;
}
else
{
termSuffixesReader.SkipBytes(suffix);
}
}
// The purpose of the terms dict index is to seek
// the enum to the closest index term before the
// term we are looking for. So, we should never
// cross another index term (besides the first
// one) while we are scanning:
Debug.Assert(indexIsCurrent);
if (!NextBlock())
{
//System.out.println(" END");
indexIsCurrent = false;
return(SeekStatus.END);
}
common = 0;
}
}
public override int NextDoc()
{
while (posPending > 0)
{
NextPosition();
}
while (true)
{
//System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
if (docUpto == numDocs)
{
//System.out.println(" END");
return(docID = NO_MORE_DOCS);
}
docUpto++;
int code = @in.ReadVInt32();
accum += (int)((uint)code >> 1);
if ((code & 1) != 0)
{
freq = 1;
}
else
{
freq = @in.ReadVInt32();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(freq > 0);
}
}
if (liveDocs == null || liveDocs.Get(accum))
{
pos = 0;
startOffset = storeOffsets ? 0 : -1;
posPending = freq;
//System.out.println(" return docID=" + accum + " freq=" + freq);
return(docID = accum);
}
// Skip positions
for (int posUpto = 0; posUpto < freq; posUpto++)
{
if (!storePayloads)
{
@in.ReadVInt32();
}
else
{
int skipCode = @in.ReadVInt32();
if ((skipCode & 1) != 0)
{
payloadLength = @in.ReadVInt32();
//System.out.println(" new payloadLen=" + payloadLength);
}
}
if (storeOffsets)
{
if ((@in.ReadVInt32() & 1) != 0)
{
// new offset length
offsetLength = @in.ReadVInt32();
}
}
if (storePayloads)
{
@in.SkipBytes(payloadLength);
}
}
}
}
public override int NextDoc()
{
while (true)
{
//System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this);
if (docUpto == numDocs)
{
// System.out.println(" END");
return(docID = NO_MORE_DOCS);
}
docUpto++;
if (indexOptions == IndexOptions.DOCS_ONLY)
{
accum += @in.ReadVInt32();
}
else
{
int code = @in.ReadVInt32();
accum += (int)((uint)code >> 1);
//System.out.println(" docID=" + accum + " code=" + code);
if ((code & 1) != 0)
{
freq = 1;
}
else
{
freq = @in.ReadVInt32();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(freq > 0);
}
}
if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
{
// Skip positions/payloads
for (int posUpto = 0; posUpto < freq; posUpto++)
{
if (!storePayloads)
{
@in.ReadVInt32();
}
else
{
int posCode = @in.ReadVInt32();
if ((posCode & 1) != 0)
{
payloadLen = @in.ReadVInt32();
}
@in.SkipBytes(payloadLen);
}
}
}
else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
{
// Skip positions/offsets/payloads
for (int posUpto = 0; posUpto < freq; posUpto++)
{
int posCode = @in.ReadVInt32();
if (storePayloads && ((posCode & 1) != 0))
{
payloadLen = @in.ReadVInt32();
}
if ((@in.ReadVInt32() & 1) != 0)
{
// new offset length
@in.ReadVInt32();
}
if (storePayloads)
{
@in.SkipBytes(payloadLen);
}
}
}
}
if (liveDocs == null || liveDocs.Get(accum))
{
//System.out.println(" return docID=" + accum + " freq=" + freq);
return(docID = accum);
}
}
}
// Interleaves all output tokens onto the futureOutputs:
private void AddOutput(BytesRef bytes, int matchInputLength, int matchEndOffset)
{
bytesReader.Reset(bytes.Bytes, bytes.Offset, bytes.Length);
int code = bytesReader.ReadVInt32();
bool keepOrig = (code & 0x1) == 0;
int count = (int)((uint)code >> 1);
//System.out.println(" addOutput count=" + count + " keepOrig=" + keepOrig);
for (int outputIDX = 0; outputIDX < count; outputIDX++)
{
synonyms.Words.Get(bytesReader.ReadVInt32(), scratchBytes);
//System.out.println(" outIDX=" + outputIDX + " bytes=" + scratchBytes.length);
UnicodeUtil.UTF8toUTF16(scratchBytes, scratchChars);
int lastStart = scratchChars.Offset;
int chEnd = lastStart + scratchChars.Length;
int outputUpto = nextRead;
for (int chIDX = lastStart; chIDX <= chEnd; chIDX++)
{
if (chIDX == chEnd || scratchChars.Chars[chIDX] == SynonymMap.WORD_SEPARATOR)
{
int outputLen = chIDX - lastStart;
// Caller is not allowed to have empty string in
// the output:
if (Debugging.AssertsEnabled)
{
Debugging.Assert(outputLen > 0, "output contains empty string: {0}", scratchChars);
}
int endOffset;
int posLen;
if (chIDX == chEnd && lastStart == scratchChars.Offset)
{
// This rule had a single output token, so, we set
// this output's endOffset to the current
// endOffset (ie, endOffset of the last input
// token it matched):
endOffset = matchEndOffset;
posLen = keepOrig ? matchInputLength : 1;
}
else
{
// This rule has more than one output token; we
// can't pick any particular endOffset for this
// case, so, we inherit the endOffset for the
// input token which this output overlaps:
endOffset = -1;
posLen = 1;
}
futureOutputs[outputUpto].Add(scratchChars.Chars, lastStart, outputLen, endOffset, posLen);
//System.out.println(" " + new String(scratchChars.chars, lastStart, outputLen) + " outputUpto=" + outputUpto);
lastStart = 1 + chIDX;
//System.out.println(" slot=" + outputUpto + " keepOrig=" + keepOrig);
outputUpto = RollIncr(outputUpto);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(futureOutputs[outputUpto].posIncr == 1, "outputUpto={0} vs nextWrite={1}", outputUpto, nextWrite);
}
}
}
}
int upto = nextRead;
for (int idx = 0; idx < matchInputLength; idx++)
{
futureInputs[upto].keepOrig |= keepOrig;
futureInputs[upto].matched = true;
upto = RollIncr(upto);
}
}
public virtual void TestVariableBinary([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
BaseDirectoryWrapper dir = NewFSDirectory(CreateTempDir("2BVariableBinary"));
if (dir is MockDirectoryWrapper)
{
((MockDirectoryWrapper)dir).Throttling = Throttling.NEVER;
}
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(OpenMode.CREATE);
IndexWriter w = new IndexWriter(dir, config);
Document doc = new Document();
var bytes = new byte[4];
ByteArrayDataOutput encoder = new ByteArrayDataOutput(bytes);
BytesRef data = new BytesRef(bytes);
BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
doc.Add(dvField);
for (int i = 0; i < int.MaxValue; i++)
{
encoder.Reset(bytes);
encoder.WriteVInt32(i % 65535); // 1, 2, or 3 bytes
data.Length = encoder.Position;
w.AddDocument(doc);
if (i % 100000 == 0)
{
Console.WriteLine("indexed: " + i);
Console.Out.Flush();
}
}
w.ForceMerge(1);
w.Dispose();
Console.WriteLine("verifying...");
Console.Out.Flush();
DirectoryReader r = DirectoryReader.Open(dir);
int expectedValue = 0;
ByteArrayDataInput input = new ByteArrayDataInput();
foreach (AtomicReaderContext context in r.Leaves)
{
AtomicReader reader = context.AtomicReader;
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.GetBinaryDocValues("dv");
for (int i = 0; i < reader.MaxDoc; i++)
{
dv.Get(i, scratch);
input.Reset(scratch.Bytes, scratch.Offset, scratch.Length);
Assert.AreEqual(expectedValue % 65535, input.ReadVInt32());
Assert.IsTrue(input.Eof);
expectedValue++;
}
}
r.Dispose();
dir.Dispose();
}
public override int NextDoc()
{
while (true)
{
if (_postings.Eof)
{
return(_docId = NO_MORE_DOCS);
}
var code = _postings.ReadVInt32();
if (_indexOptions == IndexOptions.DOCS_ONLY)
{
_accum += code;
}
else
{
_accum += (int)((uint)code >> 1);; // shift off low bit
_freq = (code & 1) != 0 ? 1 : _postings.ReadVInt32();
if (_indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0)
{
// Skip positions
if (_storePayloads)
{
for (var pos = 0; pos < _freq; pos++)
{
var posCode = _postings.ReadVInt32();
if ((posCode & 1) != 0)
{
_payloadLength = _postings.ReadVInt32();
}
if (_storeOffsets && (_postings.ReadVInt32() & 1) != 0)
{
// new offset length
_postings.ReadVInt32();
}
if (_payloadLength != 0)
{
_postings.SkipBytes(_payloadLength);
}
}
}
else
{
for (var pos = 0; pos < _freq; pos++)
{
// TODO: skipVInt
_postings.ReadVInt32();
if (_storeOffsets && (_postings.ReadVInt32() & 1) != 0)
{
// new offset length
_postings.ReadVInt32();
}
}
}
}
}
if (_liveDocs == null || _liveDocs.Get(_accum))
{
return(_docId = _accum);
}
}
}
public override int NextDoc()
{
while (true)
{
if (_postings.Eof)
{
return(_docId = NO_MORE_DOCS);
}
var code = _postings.ReadVInt32();
if (_indexOptions == IndexOptions.DOCS_ONLY)
{
_accum += code;
}
else
{
_accum += code.TripleShift(1);; // shift off low bit
_freq = (code & 1) != 0 ? 1 : _postings.ReadVInt32();
// LUCENENET specific - to avoid boxing, changed from CompareTo() to IndexOptionsComparer.Compare()
if (IndexOptionsComparer.Default.Compare(_indexOptions, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0)
{
// Skip positions
if (_storePayloads)
{
for (var pos = 0; pos < _freq; pos++)
{
var posCode = _postings.ReadVInt32();
if ((posCode & 1) != 0)
{
_payloadLength = _postings.ReadVInt32();
}
if (_storeOffsets && (_postings.ReadVInt32() & 1) != 0)
{
// new offset length
_postings.ReadVInt32();
}
if (_payloadLength != 0)
{
_postings.SkipBytes(_payloadLength);
}
}
}
else
{
for (var pos = 0; pos < _freq; pos++)
{
// TODO: skipVInt
_postings.ReadVInt32();
if (_storeOffsets && (_postings.ReadVInt32() & 1) != 0)
{
// new offset length
_postings.ReadVInt32();
}
}
}
}
}
if (_liveDocs is null || _liveDocs.Get(_accum))
{
return(_docId = _accum);
}
}
}
