public override DocsAndPositionsEnum DocsAndPositions(IBits liveDocs, DocsAndPositionsEnum reuse, DocsAndPositionsFlags flags)
{
Debug.Assert(state == State.POSITIONED, "DocsAndPositions(...) called on unpositioned TermsEnum");
// TODO: should we give this thing a random to be super-evil,
// and randomly *not* unwrap?
if (reuse is AssertingDocsAndPositionsEnum)
{
reuse = ((AssertingDocsAndPositionsEnum)reuse).m_input;
}
DocsAndPositionsEnum docs = base.DocsAndPositions(liveDocs, reuse, flags);
return(docs == null ? null : new AssertingDocsAndPositionsEnum(docs));
}
public override Similarity Get(string field)
{
lock (this)
{
Debug.Assert(field != null);
Similarity sim;
if (!previousMappings.TryGetValue(field, out sim) || sim == null)
{
sim = knownSims[Math.Max(0, Math.Abs(perFieldSeed ^ field.GetHashCode())) % knownSims.Count];
previousMappings[field] = sim;
}
return(sim);
}
}
[ExceptionToNetNumericConvention] // LUCENENET: This is for making test porting easier, keeping as-is
internal int randomIntBetween(int min, int max)
{
Debug.Assert(max >= min, "max must be >= min: " + min + ", " + max);
long range = (long)max - (long)min;
if (range < int.MaxValue)
{
return(min + Random.nextInt(1 + (int)range));
}
else
{
return(toIntExact(min + Random.Next(1 + (int)range)));
}
}
/// <summary>
/// 4.0-style docvalues byte </summary>
public virtual byte DocValuesByte(DocValuesType type, string legacyTypeAtt)
{
if (type == DocValuesType.NONE)
{
Debug.Assert(legacyTypeAtt == null);
return(0);
}
else
{
Debug.Assert(legacyTypeAtt != null);
//return (sbyte)LegacyDocValuesType.ordinalLookup[legacyTypeAtt];
return((byte)legacyTypeAtt.ToLegacyDocValuesType());
}
}
public override bool SeekExact(BytesRef text)
{
Debug.Assert(text.IsValid());
if (base.SeekExact(text))
{
state = State.POSITIONED;
return(true);
}
else
{
state = State.UNPOSITIONED;
return(false);
}
}
// Currently used only by assert statement
private int CompareToLastTerm(int fieldNumber, BytesRef term)
{
if (lastFieldNumber != fieldNumber)
{
int cmp = FieldName(fieldInfos, lastFieldNumber).CompareToOrdinal(FieldName(fieldInfos, fieldNumber));
// If there is a field named "" (empty string) then we
// will get 0 on this comparison, yet, it's "OK". But
// it's not OK if two different field numbers map to
// the same name.
if (cmp != 0 || lastFieldNumber != -1)
{
return(cmp);
}
}
scratchBytes.CopyBytes(term);
Debug.Assert(lastTerm.Offset == 0);
UnicodeUtil.UTF8toUTF16(lastTerm.Bytes, 0, lastTerm.Length, utf16Result1);
Debug.Assert(scratchBytes.Offset == 0);
UnicodeUtil.UTF8toUTF16(scratchBytes.Bytes, 0, scratchBytes.Length, utf16Result2);
int len;
if (utf16Result1.Length < utf16Result2.Length)
{
len = utf16Result1.Length;
}
else
{
len = utf16Result2.Length;
}
for (int i = 0; i < len; i++)
{
char ch1 = utf16Result1.Chars[i];
char ch2 = utf16Result2.Chars[i];
if (ch1 != ch2)
{
return(ch1 - ch2);
}
}
if (utf16Result1.Length == 0 && lastFieldNumber == -1)
{
// If there is a field named "" (empty string) with a term text of "" (empty string) then we
// will get 0 on this comparison, yet, it's "OK".
return(-1);
}
return(utf16Result1.Length - utf16Result2.Length);
}
public override SeekStatus SeekCeil(BytesRef term)
{
Debug.Assert(term.IsValid());
SeekStatus result = base.SeekCeil(term);
if (result == SeekStatus.END)
{
state = State.UNPOSITIONED;
}
else
{
state = State.POSITIONED;
}
return(result);
}
/// <summary>
/// Sets the values for the current skip data.
/// </summary>
public virtual void SetSkipData(int doc, bool storePayloads, int payloadLength, bool storeOffsets, int offsetLength)
{
Debug.Assert(storePayloads || payloadLength == -1);
Debug.Assert(storeOffsets || offsetLength == -1);
this.curDoc = doc;
this.curStorePayloads = storePayloads;
this.curPayloadLength = payloadLength;
this.curStoreOffsets = storeOffsets;
this.curOffsetLength = offsetLength;
this.curFreqPointer = freqOutput.GetFilePointer();
if (proxOutput != null)
{
this.curProxPointer = proxOutput.GetFilePointer();
}
}
private void AddFixedDerefBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values, int length)
{
field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_FIXED_DEREF.ToString());
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_FIXED_DEREF_VERSION_CURRENT);
// deduplicate
SortedSet <BytesRef> dictionary = new SortedSet <BytesRef>();
foreach (BytesRef v in values)
{
dictionary.Add(v == null ? new BytesRef() : BytesRef.DeepCopyOf(v));
}
/* values */
data.WriteInt32(length);
foreach (BytesRef v in dictionary)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
/* ordinals */
int valueCount = dictionary.Count;
Debug.Assert(valueCount > 0);
index.WriteInt32(valueCount);
int maxDoc = state.SegmentInfo.DocCount;
PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc, PackedInt32s.BitsRequired(valueCount - 1), PackedInt32s.DEFAULT);
BytesRef brefDummy;
foreach (BytesRef v in values)
{
brefDummy = v;
if (v == null)
{
brefDummy = new BytesRef();
}
//int ord = dictionary.HeadSet(brefDummy).Size();
int ord = dictionary.Count(@ref => @ref.CompareTo(brefDummy) < 0);
w.Add(ord);
}
w.Finish();
}
public override void AddPosition(int position, BytesRef payload, int startOffset, int endOffset)
{
Debug.Assert(startOffset == -1);
Debug.Assert(endOffset == -1);
current.positions[posUpto] = position;
if (payload != null && payload.Length > 0)
{
if (current.payloads == null)
{
current.payloads = new byte[current.positions.Length][];
}
var bytes = current.payloads[posUpto] = new byte[payload.Length];
Array.Copy(payload.Bytes, payload.Offset, bytes, 0, payload.Length);
}
posUpto++;
}
// TODO: we should separately track if we are 'at the end' ?
// someone should not call next() after it returns null!!!!
public override BytesRef Next()
{
Debug.Assert(state == State.INITIAL || state == State.POSITIONED, "Next() called on unpositioned TermsEnum");
BytesRef result = base.Next();
if (result == null)
{
state = State.UNPOSITIONED;
}
else
{
Debug.Assert(result.IsValid());
state = State.POSITIONED;
}
return(result);
}
public override BinaryDocValues GetBinaryDocValues(string field)
{
BinaryDocValues dv = base.GetBinaryDocValues(field);
FieldInfo fi = FieldInfos.FieldInfo(field);
if (dv != null)
{
Debug.Assert(fi != null);
Debug.Assert(fi.DocValuesType == DocValuesType.BINARY);
return(new AssertingBinaryDocValues(dv, MaxDoc));
}
else
{
Debug.Assert(fi == null || fi.DocValuesType != DocValuesType.BINARY);
return(null);
}
}
public override SortedSetDocValues GetSortedSetDocValues(string field)
{
SortedSetDocValues dv = base.GetSortedSetDocValues(field);
FieldInfo fi = FieldInfos.FieldInfo(field);
if (dv != null)
{
Debug.Assert(fi != null);
Debug.Assert(fi.DocValuesType == DocValuesType.SORTED_SET);
return(new AssertingSortedSetDocValues(dv, MaxDoc));
}
else
{
Debug.Assert(fi == null || fi.DocValuesType != DocValuesType.SORTED_SET);
return(null);
}
}
// NOTE: 4.0 file format docs are crazy/wrong here...
private void AddVarStraightBytesField(FieldInfo field, IndexOutput data, IndexOutput index, IEnumerable <BytesRef> values)
{
field.PutAttribute(legacyKey, LegacyDocValuesType.BYTES_VAR_STRAIGHT.ToString());
CodecUtil.WriteHeader(data, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_DAT, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
CodecUtil.WriteHeader(index, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_CODEC_NAME_IDX, Lucene40DocValuesFormat.BYTES_VAR_STRAIGHT_VERSION_CURRENT);
/* values */
long startPos = data.GetFilePointer();
foreach (BytesRef v in values)
{
if (v != null)
{
data.WriteBytes(v.Bytes, v.Offset, v.Length);
}
}
/* addresses */
long maxAddress = data.GetFilePointer() - startPos;
index.WriteVInt64(maxAddress);
int maxDoc = state.SegmentInfo.DocCount;
Debug.Assert(maxDoc != int.MaxValue); // unsupported by the 4.0 impl
PackedInt32s.Writer w = PackedInt32s.GetWriter(index, maxDoc + 1, PackedInt32s.BitsRequired(maxAddress), PackedInt32s.DEFAULT);
long currentPosition = 0;
foreach (BytesRef v in values)
{
w.Add(currentPosition);
if (v != null)
{
currentPosition += v.Length;
}
}
// write sentinel
Debug.Assert(currentPosition == maxAddress);
w.Add(currentPosition);
w.Finish();
}
public override NumericDocValues GetNormValues(string field)
{
NumericDocValues dv = base.GetNormValues(field);
FieldInfo fi = FieldInfos.FieldInfo(field);
if (dv != null)
{
Debug.Assert(fi != null);
Debug.Assert(fi.HasNorms);
return(new AssertingNumericDocValues(dv, MaxDoc));
}
else
{
Debug.Assert(fi == null || fi.HasNorms == false);
return(null);
}
}
// LUCENENET specific - de-nested AssertingNumericDocValues
// LUCENENET specific - de-nested AssertingBinaryDocValues
// LUCENENET specific - de-nested AssertingSortedDocValues
// LUCENENET specific - de-nested AssertingSortedSetDocValues
public override NumericDocValues GetNumericDocValues(string field)
{
NumericDocValues dv = base.GetNumericDocValues(field);
FieldInfo fi = FieldInfos.FieldInfo(field);
if (dv != null)
{
Debug.Assert(fi != null);
Debug.Assert(fi.DocValuesType == DocValuesType.NUMERIC);
return(new AssertingNumericDocValues(dv, MaxDoc));
}
else
{
Debug.Assert(fi == null || fi.DocValuesType != DocValuesType.NUMERIC);
return(null);
}
}
private static IndexReader MakeEmptyIndex(Random random, int numDocs)
{
Debug.Assert(numDocs > 0);
Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(random)));
for (int i = 0; i < numDocs; i++)
{
w.AddDocument(new Document());
}
w.ForceMerge(1);
w.Commit();
w.Dispose();
DirectoryReader reader = DirectoryReader.Open(d);
return(new AllDeletedFilterReader(LuceneTestCase.GetOnlySegmentReader(reader)));
}
public override int NextDoc()
{
Debug.Assert(state != DocsEnumState.FINISHED, "NextDoc() called after NO_MORE_DOCS");
int nextDoc = base.NextDoc();
Debug.Assert(nextDoc > doc, "backwards NextDoc from " + doc + " to " + nextDoc + " " + m_input);
if (nextDoc == DocIdSetIterator.NO_MORE_DOCS)
{
state = DocsEnumState.FINISHED;
}
else
{
state = DocsEnumState.ITERATING;
}
Debug.Assert(base.DocID == nextDoc);
return(doc = nextDoc);
}
public AssertingDocsEnum(DocsEnum @in, bool failOnUnsupportedDocID)
: base(@in)
{
try
{
int docid = @in.DocID;
Debug.Assert(docid == -1, @in.GetType() + ": invalid initial doc id: " + docid);
}
catch (System.NotSupportedException /*e*/)
{
if (failOnUnsupportedDocID)
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
}
doc = -1;
}
protected override int Correct(int currentOff)
{
int ret;
// LUCENENET NOTE: TryGetPredecessor is equivalent to TreeMap.lowerEntry() in Java
if (corrections.TryGetPredecessor(currentOff + 1, out KeyValuePair <int, int> lastEntry))
{
ret = currentOff + lastEntry.Value;
}
else
{
ret = currentOff;
}
Debug.Assert(ret >= 0, "currentOff=" + currentOff + ",diff=" + (ret - currentOff));
return(ret);
}
public override void End()
{
base.End();
int finalOffset = CorrectOffset(off);
offsetAtt.SetOffset(finalOffset, finalOffset);
// some tokenizers, such as limiting tokenizers, call End() before IncrementToken() returns false.
// these tests should disable this check (in general you should consume the entire stream)
try
{
Debug.Assert(!enableChecks || streamState == State.INCREMENT_FALSE, "End() called before IncrementToken() returned false!");
}
finally
{
streamState = State.END;
}
}
public override TermStatistics TermStatistics(Term term, TermContext context)
{
Debug.Assert(term != null);
long docFreq = 0;
long totalTermFreq = 0;
for (int nodeID = 0; nodeID < nodeVersions.Length; nodeID++)
{
TermStatistics subStats;
if (nodeID == MyNodeID)
{
subStats = base.TermStatistics(term, context);
}
else
{
TermAndShardVersion key = new TermAndShardVersion(nodeID, nodeVersions[nodeID], term);
subStats = outerInstance.termStatsCache[key];
// We pre-cached during rewrite so all terms
// better be here...
Debug.Assert(subStats != null);
}
long nodeDocFreq = subStats.DocFreq;
if (docFreq >= 0 && nodeDocFreq >= 0)
{
docFreq += nodeDocFreq;
}
else
{
docFreq = -1;
}
long nodeTotalTermFreq = subStats.TotalTermFreq;
if (totalTermFreq >= 0 && nodeTotalTermFreq >= 0)
{
totalTermFreq += nodeTotalTermFreq;
}
else
{
totalTermFreq = -1;
}
}
return(new TermStatistics(term.Bytes, docFreq, totalTermFreq));
}
public override int Advance(int target)
{
Debug.Assert(state != DocsEnumState.FINISHED, "Advance() called after NO_MORE_DOCS");
Debug.Assert(target > doc, "target must be > DocID, got " + target + " <= " + doc);
int advanced = base.Advance(target);
Debug.Assert(advanced >= target, "backwards advance from: " + target + " to: " + advanced);
if (advanced == DocIdSetIterator.NO_MORE_DOCS)
{
state = DocsEnumState.FINISHED;
}
else
{
state = DocsEnumState.ITERATING;
}
Debug.Assert(base.DocID == advanced);
return(doc = advanced);
}
public override IBits GetDocsWithField(string field)
{
IBits docsWithField = base.GetDocsWithField(field);
FieldInfo fi = FieldInfos.FieldInfo(field);
if (docsWithField != null)
{
Debug.Assert(fi != null);
Debug.Assert(fi.HasDocValues);
Debug.Assert(MaxDoc == docsWithField.Length);
docsWithField = new AssertingBits(docsWithField);
}
else
{
Debug.Assert(fi == null || fi.HasDocValues == false);
}
return(docsWithField);
}
// runs the term, returning the output, or null if term
// isn't accepted. if prefixLength is non-null it must be
// length 1 int array; prefixLength[0] is set to the length
// of the term prefix that matches
private T Run(FST <T> fst, Int32sRef term, int[] prefixLength)
{
Debug.Assert(prefixLength == null || prefixLength.Length == 1);
FST.Arc <T> arc = fst.GetFirstArc(new FST.Arc <T>());
T NO_OUTPUT = fst.Outputs.NoOutput;
T output = NO_OUTPUT;
FST.BytesReader fstReader = fst.GetBytesReader();
for (int i = 0; i <= term.Length; i++)
{
int label;
if (i == term.Length)
{
label = FST.END_LABEL;
}
else
{
label = term.Int32s[term.Offset + i];
}
// System.out.println(" loop i=" + i + " label=" + label + " output=" + fst.Outputs.outputToString(output) + " curArc: target=" + arc.target + " isFinal?=" + arc.isFinal());
if (fst.FindTargetArc(label, arc, arc, fstReader) == null)
{
// System.out.println(" not found");
if (prefixLength != null)
{
prefixLength[0] = i;
return(output);
}
else
{
return(default(T));
}
}
output = fst.Outputs.Add(output, arc.Output);
}
if (prefixLength != null)
{
prefixLength[0] = term.Length;
}
return(output);
}
public override MergeSpecification FindForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, IDictionary <SegmentCommitInfo, bool?> segmentsToMerge)
{
IList <SegmentCommitInfo> eligibleSegments = new List <SegmentCommitInfo>();
foreach (SegmentCommitInfo info in segmentInfos.Segments)
{
if (segmentsToMerge.ContainsKey(info))
{
eligibleSegments.Add(info);
}
}
//System.out.println("MRMP: findMerges sis=" + segmentInfos + " eligible=" + eligibleSegments);
MergeSpecification mergeSpec = null;
if (eligibleSegments.Count > 1 || (eligibleSegments.Count == 1 && eligibleSegments[0].HasDeletions))
{
mergeSpec = new MergeSpecification();
// Already shuffled having come out of a set but
// shuffle again for good measure:
eligibleSegments.Shuffle(random);
int upto = 0;
while (upto < eligibleSegments.Count)
{
int max = Math.Min(10, eligibleSegments.Count - upto);
int inc = max <= 2 ? max : TestUtil.NextInt32(random, 2, max);
mergeSpec.Add(new OneMerge(eligibleSegments.SubList(upto, upto + inc)));
upto += inc;
}
}
if (mergeSpec != null)
{
foreach (OneMerge merge in mergeSpec.Merges)
{
foreach (SegmentCommitInfo info in merge.Segments)
{
Debug.Assert(segmentsToMerge.ContainsKey(info));
}
}
}
return(mergeSpec);
}
protected virtual void Start(int numNodes, double runTimeSec, int maxSearcherAgeSeconds)
{
endTimeNanos = Time.NanoTime() + (long)(runTimeSec * 1000000000);
this.maxSearcherAgeSeconds = maxSearcherAgeSeconds;
m_nodes = new NodeState[numNodes];
for (int nodeID = 0; nodeID < numNodes; nodeID++)
{
m_nodes[nodeID] = new NodeState(this, Random, nodeID, numNodes);
}
long[] nodeVersions = new long[m_nodes.Length];
for (int nodeID = 0; nodeID < numNodes; nodeID++)
{
IndexSearcher s = m_nodes[nodeID].Mgr.Acquire();
try
{
nodeVersions[nodeID] = m_nodes[nodeID].Searchers.Record(s);
}
finally
{
m_nodes[nodeID].Mgr.Release(s);
}
}
for (int nodeID = 0; nodeID < numNodes; nodeID++)
{
IndexSearcher s = m_nodes[nodeID].Mgr.Acquire();
Debug.Assert(nodeVersions[nodeID] == m_nodes[nodeID].Searchers.Record(s));
Debug.Assert(s != null);
try
{
BroadcastNodeReopen(nodeID, nodeVersions[nodeID], s);
}
finally
{
m_nodes[nodeID].Mgr.Release(s);
}
}
changeIndicesThread = new ChangeIndices(this);
changeIndicesThread.Start();
}
public override void FinishTerm(BytesRef text, TermStats stats)
{
Debug.Assert(state == TermsConsumerState.START);
state = TermsConsumerState.INITIAL;
Debug.Assert(text.Equals(lastTerm));
Debug.Assert(stats.DocFreq > 0); // otherwise, this method should not be called.
Debug.Assert(stats.DocFreq == lastPostingsConsumer.docFreq);
sumDocFreq += stats.DocFreq;
if (fieldInfo.IndexOptions == IndexOptions.DOCS_ONLY)
{
Debug.Assert(stats.TotalTermFreq == -1);
}
else
{
Debug.Assert(stats.TotalTermFreq == lastPostingsConsumer.totalTermFreq);
sumTotalTermFreq += stats.TotalTermFreq;
}
@in.FinishTerm(text, stats);
}
public override void Finish(long sumTotalTermFreq, long sumDocFreq, int docCount)
{
Debug.Assert(state == TermsConsumerState.INITIAL || state == TermsConsumerState.START && lastPostingsConsumer.docFreq == 0);
state = TermsConsumerState.FINISHED;
Debug.Assert(docCount >= 0);
Debug.Assert(docCount == visitedDocs.Cardinality());
Debug.Assert(sumDocFreq >= docCount);
Debug.Assert(sumDocFreq == this.sumDocFreq);
if (fieldInfo.IndexOptions == IndexOptions.DOCS_ONLY)
{
Debug.Assert(sumTotalTermFreq == -1);
}
else
{
Debug.Assert(sumTotalTermFreq >= sumDocFreq);
Debug.Assert(sumTotalTermFreq == this.sumTotalTermFreq);
}
@in.Finish(sumTotalTermFreq, sumDocFreq, docCount);
}
public override void EncodeTerm(long[] empty, DataOutput @out, FieldInfo fieldInfo, BlockTermState state, bool absolute)
{
StandardTermState state_ = (StandardTermState)state;
if (absolute)
{
lastState = emptyState;
}
@out.WriteVInt64(state_.FreqStart - lastState.FreqStart);
if (state_.SkipOffset != -1)
{
Debug.Assert(state_.SkipOffset > 0);
@out.WriteVInt64(state_.SkipOffset);
}
if (indexOptions.CompareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0)
{
@out.WriteVInt64(state_.ProxStart - lastState.ProxStart);
}
lastState = state_;
}