public virtual void TestAdvance()
{
Term allTerm = new Term(FIELD, "all");
TermQuery termQuery = new TermQuery(allTerm);
Weight weight = indexSearcher.CreateNormalizedWeight(termQuery);
Assert.IsTrue(indexSearcher.TopReaderContext is AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext)indexSearcher.TopReaderContext;
Scorer ts = weight.GetScorer(context, (context.AtomicReader).LiveDocs);
Assert.IsTrue(ts.Advance(3) != DocIdSetIterator.NO_MORE_DOCS, "Didn't skip");
// The next doc should be doc 5
Assert.IsTrue(ts.DocID == 5, "doc should be number 5");
}
public virtual void TestSkipToFirsttimeHit()
{
DisjunctionMaxQuery dq = new DisjunctionMaxQuery(0.0f);
dq.Add(Tq("dek", "albino"));
dq.Add(Tq("dek", "DOES_NOT_EXIST"));
Assert.IsTrue(s.TopReaderContext is AtomicReaderContext);
QueryUtils.Check(Random, dq, s);
Weight dw = s.CreateNormalizedWeight(dq);
AtomicReaderContext context = (AtomicReaderContext)s.TopReaderContext;
Scorer ds = dw.GetScorer(context, (context.AtomicReader).LiveDocs);
Assert.IsTrue(ds.Advance(3) != DocIdSetIterator.NO_MORE_DOCS, "firsttime skipTo found no match");
Assert.AreEqual("d4", r.Document(ds.DocID).Get("id"), "found wrong docid");
}
public override Scorer FilteredScorer(AtomicReaderContext context, Weight weight, DocIdSet docIdSet)
{
DocIdSetIterator filterIter = docIdSet.GetIterator();
if (filterIter == null)
{
// this means the filter does not accept any documents.
return(null);
}
int firstFilterDoc = filterIter.NextDoc();
if (firstFilterDoc == DocIdSetIterator.NO_MORE_DOCS)
{
return(null);
}
IBits filterAcceptDocs = docIdSet.Bits;
// force if RA is requested
bool useRandomAccess = filterAcceptDocs != null && UseRandomAccess(filterAcceptDocs, firstFilterDoc);
if (useRandomAccess)
{
// if we are using random access, we return the inner scorer, just with other acceptDocs
return(weight.GetScorer(context, filterAcceptDocs));
}
else
{
Debug.Assert(firstFilterDoc > -1);
// we are gonna advance() this scorer, so we set inorder=true/toplevel=false
// we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
Scorer scorer = weight.GetScorer(context, null);
// TODO once we have way to figure out if we use RA or LeapFrog we can remove this scorer
return((scorer == null) ? null : new PrimaryAdvancedLeapFrogScorer(weight, firstFilterDoc, filterIter, scorer));
}
}
/// <summary>
/// Alternate scorer skipTo(),skipTo(),next(),next(),skipTo(),skipTo(), etc
/// and ensure a hitcollector receives same docs and scores.
/// </summary>
/// <param name="q"></param>
/// <param name="s"></param>
/// <param name="similarity">Generally, should always be <see cref="LuceneTestCase.Similarity"/>.</param>
// LUCENENET specific
// Removes dependency on <see cref="LuceneTestCase.ClassEnv.Similarity"/>
public static void CheckSkipTo(Query q, IndexSearcher s, Similarity similarity)
{
//System.out.println("Checking "+q);
IList <AtomicReaderContext> readerContextArray = s.TopReaderContext.Leaves;
if (s.CreateNormalizedWeight(q).ScoresDocsOutOfOrder) // in this case order of skipTo() might differ from that of next().
{
return;
}
const int skip_op = 0;
const int next_op = 1;
int[][] orders = new int[][] { new int[] { next_op }, new int[] { skip_op }, new int[] { skip_op, next_op }, new int[] { next_op, skip_op }, new int[] { skip_op, skip_op, next_op, next_op }, new int[] { next_op, next_op, skip_op, skip_op }, new int[] { skip_op, skip_op, skip_op, next_op, next_op } };
for (int k = 0; k < orders.Length; k++)
{
int[] order = orders[k];
// System.out.print("Order:");for (int i = 0; i < order.Length; i++)
// System.out.print(order[i]==skip_op ? " skip()":" next()");
// System.out.println();
int[] opidx = new int[] { 0 };
int[] lastDoc = new int[] { -1 };
// FUTURE: ensure scorer.Doc()==-1
const float maxDiff = 1e-5f;
AtomicReader[] lastReader = new AtomicReader[] { null };
s.Search(q, new CollectorAnonymousInnerClassHelper(q, s, readerContextArray, skip_op, order, opidx, lastDoc, maxDiff, lastReader, similarity));
if (lastReader[0] != null)
{
// confirm that skipping beyond the last doc, on the
// previous reader, hits NO_MORE_DOCS
AtomicReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.NewSearcher(previousReader, false, similarity);
indexSearcher.Similarity = s.Similarity;
Weight w = indexSearcher.CreateNormalizedWeight(q);
AtomicReaderContext ctx = (AtomicReaderContext)previousReader.Context;
Scorer scorer = w.GetScorer(ctx, ((AtomicReader)ctx.Reader).LiveDocs);
if (scorer != null)
{
bool more = scorer.Advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.IsFalse(more, "query's last doc was " + lastDoc[0] + " but skipTo(" + (lastDoc[0] + 1) + ") got to " + scorer.DocID);
}
}
}
}
public virtual void TestNext()
{
Term allTerm = new Term(FIELD, "all");
TermQuery termQuery = new TermQuery(allTerm);
Weight weight = IndexSearcher.CreateNormalizedWeight(termQuery);
Assert.IsTrue(IndexSearcher.TopReaderContext is AtomicReaderContext);
AtomicReaderContext context = (AtomicReaderContext)IndexSearcher.TopReaderContext;
Scorer ts = weight.GetScorer(context, (context.AtomicReader).LiveDocs);
Assert.IsTrue(ts.NextDoc() != DocIdSetIterator.NO_MORE_DOCS, "next did not return a doc");
Assert.IsTrue(ts.GetScore() == 1.6931472f, "score is not correct");
Assert.IsTrue(ts.NextDoc() != DocIdSetIterator.NO_MORE_DOCS, "next did not return a doc");
Assert.IsTrue(ts.GetScore() == 1.6931472f, "score is not correct");
Assert.IsTrue(ts.NextDoc() == DocIdSetIterator.NO_MORE_DOCS, "next returned a doc and it should not have");
}
public virtual void TestSkipToFirsttimeMiss()
{
DisjunctionMaxQuery dq = new DisjunctionMaxQuery(0.0f);
dq.Add(Tq("id", "d1"));
dq.Add(Tq("dek", "DOES_NOT_EXIST"));
QueryUtils.Check(Random(), dq, s, Similarity);
Assert.IsTrue(s.TopReaderContext is AtomicReaderContext);
Weight dw = s.CreateNormalizedWeight(dq);
AtomicReaderContext context = (AtomicReaderContext)s.TopReaderContext;
Scorer ds = dw.GetScorer(context, (context.AtomicReader).LiveDocs);
bool skipOk = ds.Advance(3) != DocIdSetIterator.NO_MORE_DOCS;
if (skipOk)
{
Assert.Fail("firsttime skipTo found a match? ... " + r.Document(ds.DocID).Get("id"));
}
}
public virtual void Collect(int doc)
{
float score = sc.GetScore();
lastDoc[0] = doc;
try
{
if (scorer == null)
{
Weight w = s.CreateNormalizedWeight(q);
AtomicReaderContext context = readerContextArray[leafPtr];
scorer = w.GetScorer(context, (context.AtomicReader).LiveDocs);
}
int op = order[(opidx[0]++) % order.Length];
// System.out.println(op==skip_op ?
// "skip("+(sdoc[0]+1)+")":"next()");
bool more = op == skip_op?scorer.Advance(scorer.DocID + 1) != DocIdSetIterator.NO_MORE_DOCS : scorer.NextDoc() != DocIdSetIterator.NO_MORE_DOCS;
int scorerDoc = scorer.DocID;
float scorerScore = scorer.GetScore();
float scorerScore2 = scorer.GetScore();
float scoreDiff = Math.Abs(score - scorerScore);
float scorerDiff = Math.Abs(scorerScore2 - scorerScore);
if (!more || doc != scorerDoc || scoreDiff > maxDiff || scorerDiff > maxDiff)
{
StringBuilder sbord = new StringBuilder();
for (int i = 0; i < order.Length; i++)
{
sbord.Append(order[i] == skip_op ? " skip()" : " next()");
}
throw new Exception("ERROR matching docs:" + "\n\t" + (doc != scorerDoc ? "--> " : "") + "doc=" + doc + ", scorerDoc=" + scorerDoc + "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more + "\n\t" + (scoreDiff > maxDiff ? "--> " : "") + "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff + " maxDiff=" + maxDiff + "\n\t" + (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2=" + scorerScore2 + " scorerDiff=" + scorerDiff + "\n\thitCollector.Doc=" + doc + " score=" + score + "\n\t Scorer=" + scorer + "\n\t Query=" + q + " " + q.GetType().Name + "\n\t Searcher=" + s + "\n\t Order=" + sbord + "\n\t Op=" + (op == skip_op ? " skip()" : " next()"));
}
}
catch (IOException e)
{
throw new Exception(e.ToString(), e);
}
}
public virtual void SetNextReader(AtomicReaderContext context)
{
// confirm that skipping beyond the last doc, on the
// previous reader, hits NO_MORE_DOCS
if (lastReader[0] != null)
{
AtomicReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.NewSearcher(previousReader, similarity);
indexSearcher.Similarity = s.Similarity;
Weight w = indexSearcher.CreateNormalizedWeight(q);
Scorer scorer = w.GetScorer((AtomicReaderContext)indexSearcher.TopReaderContext, previousReader.LiveDocs);
if (scorer != null)
{
bool more = scorer.Advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.IsFalse(more, "query's last doc was " + lastDoc[0] + " but skipTo(" + (lastDoc[0] + 1) + ") got to " + scorer.DocID);
}
leafPtr++;
}
lastReader[0] = (AtomicReader)context.Reader;
lastDoc[0] = -1;
liveDocs = ((AtomicReader)context.Reader).LiveDocs;
}
/// <summary>
/// Check that first skip on just created scorers always goes to the right doc.</summary>
/// <param name="luceneTestCase"> The current test instance. </param>
/// <param name="q"></param>
/// <param name="s"></param>
// LUCENENET specific
// Removes dependency on <see cref="LuceneTestCase.ClassEnv.Similarity"/>
public static void CheckFirstSkipTo(LuceneTestCase luceneTestCase, Query q, IndexSearcher s)
#endif
{
//System.out.println("checkFirstSkipTo: "+q);
const float maxDiff = 1e-3f;
int[] lastDoc = new int[] { -1 };
AtomicReader[] lastReader = new AtomicReader[] { null };
IList <AtomicReaderContext> context = s.TopReaderContext.Leaves;
s.Search(q, new CollectorAnonymousInnerClassHelper2(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
luceneTestCase,
#endif
q, s, maxDiff, lastDoc, lastReader, context));
if (lastReader[0] != null)
{
// confirm that skipping beyond the last doc, on the
// previous reader, hits NO_MORE_DOCS
AtomicReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.NewSearcher(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
luceneTestCase,
#endif
previousReader);
indexSearcher.Similarity = s.Similarity;
Weight w = indexSearcher.CreateNormalizedWeight(q);
Scorer scorer = w.GetScorer((AtomicReaderContext)indexSearcher.TopReaderContext, previousReader.LiveDocs);
if (scorer != null)
{
bool more = scorer.Advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.IsFalse(more, "query's last doc was " + lastDoc[0] + " but skipTo(" + (lastDoc[0] + 1) + ") got to " + scorer.DocID);
}
}
}
[AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/269")] // LUCENENET TODO: this test fails on x86 on .NET Framework in Release mode only
#endif
public virtual void TestBS2DisjunctionNextVsAdvance()
{
Directory d = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, d);
int numDocs = AtLeast(300);
for (int docUpto = 0; docUpto < numDocs; docUpto++)
{
string contents = "a";
if (Random.Next(20) <= 16)
{
contents += " b";
}
if (Random.Next(20) <= 8)
{
contents += " c";
}
if (Random.Next(20) <= 4)
{
contents += " d";
}
if (Random.Next(20) <= 2)
{
contents += " e";
}
if (Random.Next(20) <= 1)
{
contents += " f";
}
Document doc = new Document();
doc.Add(new TextField("field", contents, Field.Store.NO));
w.AddDocument(doc);
}
w.ForceMerge(1);
IndexReader r = w.GetReader();
IndexSearcher s = NewSearcher(r);
w.Dispose();
for (int iter = 0; iter < 10 * RandomMultiplier; iter++)
{
if (Verbose)
{
Console.WriteLine("iter=" + iter);
}
IList <string> terms = new List <string> {
"a", "b", "c", "d", "e", "f"
};
int numTerms = TestUtil.NextInt32(Random, 1, terms.Count);
while (terms.Count > numTerms)
{
terms.RemoveAt(Random.Next(terms.Count));
}
if (Verbose)
{
Console.WriteLine(" terms=" + terms);
}
BooleanQuery q = new BooleanQuery();
foreach (string term in terms)
{
q.Add(new BooleanClause(new TermQuery(new Term("field", term)), Occur.SHOULD));
}
Weight weight = s.CreateNormalizedWeight(q);
Scorer scorer = weight.GetScorer(s.m_leafContexts[0], null);
// First pass: just use .NextDoc() to gather all hits
IList <ScoreDoc> hits = new List <ScoreDoc>();
while (scorer.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
hits.Add(new ScoreDoc(scorer.DocID, scorer.GetScore()));
}
if (Verbose)
{
Console.WriteLine(" " + hits.Count + " hits");
}
// Now, randomly next/advance through the list and
// verify exact match:
for (int iter2 = 0; iter2 < 10; iter2++)
{
weight = s.CreateNormalizedWeight(q);
scorer = weight.GetScorer(s.m_leafContexts[0], null);
if (Verbose)
{
Console.WriteLine(" iter2=" + iter2);
}
int upto = -1;
while (upto < hits.Count)
{
int nextUpto;
int nextDoc;
int left = hits.Count - upto;
if (left == 1 || Random.nextBoolean())
{
// next
nextUpto = 1 + upto;
nextDoc = scorer.NextDoc();
}
else
{
// advance
int inc = TestUtil.NextInt32(Random, 1, left - 1);
nextUpto = inc + upto;
nextDoc = scorer.Advance(hits[nextUpto].Doc);
}
if (nextUpto == hits.Count)
{
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, nextDoc);
}
else
{
ScoreDoc hit = hits[nextUpto];
Assert.AreEqual(hit.Doc, nextDoc);
// Test for precise float equality:
Assert.IsTrue(hit.Score == scorer.GetScore(), "doc " + hit.Doc + " has wrong score: expected=" + hit.Score + " actual=" + scorer.GetScore());
}
upto = nextUpto;
}
}
}
r.Dispose();
d.Dispose();
}
public override Explanation Explain(AtomicReaderContext context, int doc)
{
int minShouldMatch = outerInstance.MinimumNumberShouldMatch;
ComplexExplanation sumExpl = new ComplexExplanation();
sumExpl.Description = "sum of:";
int coord = 0;
float sum = 0.0f;
bool fail = false;
int shouldMatchCount = 0;
IEnumerator <BooleanClause> cIter = outerInstance.clauses.GetEnumerator();
for (IEnumerator <Weight> wIter = m_weights.GetEnumerator(); wIter.MoveNext();)
{
Weight w = wIter.Current;
cIter.MoveNext();
BooleanClause c = cIter.Current;
if (w.GetScorer(context, context.AtomicReader.LiveDocs) == null)
{
if (c.IsRequired)
{
fail = true;
Explanation r = new Explanation(0.0f, "no match on required clause (" + c.Query.ToString() + ")");
sumExpl.AddDetail(r);
}
continue;
}
Explanation e = w.Explain(context, doc);
if (e.IsMatch)
{
if (!c.IsProhibited)
{
sumExpl.AddDetail(e);
sum += e.Value;
coord++;
}
else
{
Explanation r = new Explanation(0.0f, "match on prohibited clause (" + c.Query.ToString() + ")");
r.AddDetail(e);
sumExpl.AddDetail(r);
fail = true;
}
if (c.Occur == Occur_e.SHOULD)
{
shouldMatchCount++;
}
}
else if (c.IsRequired)
{
Explanation r = new Explanation(0.0f, "no match on required clause (" + c.Query.ToString() + ")");
r.AddDetail(e);
sumExpl.AddDetail(r);
fail = true;
}
}
if (fail)
{
sumExpl.Match = false;
sumExpl.Value = 0.0f;
sumExpl.Description = "Failure to meet condition(s) of required/prohibited clause(s)";
return(sumExpl);
}
else if (shouldMatchCount < minShouldMatch)
{
sumExpl.Match = false;
sumExpl.Value = 0.0f;
sumExpl.Description = "Failure to match minimum number " + "of optional clauses: " + minShouldMatch;
return(sumExpl);
}
sumExpl.Match = 0 < coord ? true : false;
sumExpl.Value = sum;
float coordFactor = disableCoord ? 1.0f : Coord(coord, m_maxCoord);
if (coordFactor == 1.0f)
{
return(sumExpl); // eliminate wrapper
}
else
{
ComplexExplanation result = new ComplexExplanation(sumExpl.IsMatch, sum * coordFactor, "product of:");
result.AddDetail(sumExpl);
result.AddDetail(new Explanation(coordFactor, "coord(" + coord + "/" + m_maxCoord + ")"));
return(result);
}
}
public override TopDocs Rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int topN)
{
ScoreDoc[] hits = (ScoreDoc[])firstPassTopDocs.ScoreDocs.Clone();
Array.Sort(hits, new ComparerAnonymousInnerClassHelper(this));
IList <AtomicReaderContext> leaves = searcher.IndexReader.Leaves;
Weight weight = searcher.CreateNormalizedWeight(query);
// Now merge sort docIDs from hits, with reader's leaves:
int hitUpto = 0;
int readerUpto = -1;
int endDoc = 0;
int docBase = 0;
Scorer scorer = null;
while (hitUpto < hits.Length)
{
ScoreDoc hit = hits[hitUpto];
int docID = hit.Doc;
AtomicReaderContext readerContext = null;
while (docID >= endDoc)
{
readerUpto++;
readerContext = leaves[readerUpto];
endDoc = readerContext.DocBase + readerContext.Reader.MaxDoc;
}
if (readerContext != null)
{
// We advanced to another segment:
docBase = readerContext.DocBase;
scorer = weight.GetScorer(readerContext, null);
}
int targetDoc = docID - docBase;
int actualDoc = scorer.DocID;
if (actualDoc < targetDoc)
{
actualDoc = scorer.Advance(targetDoc);
}
if (actualDoc == targetDoc)
{
// Query did match this doc:
hit.Score = Combine(hit.Score, true, scorer.GetScore());
}
else
{
// Query did not match this doc:
Debug.Assert(actualDoc > targetDoc);
hit.Score = Combine(hit.Score, false, 0.0f);
}
hitUpto++;
}
// TODO: we should do a partial sort (of only topN)
// instead, but typically the number of hits is
// smallish:
Array.Sort(hits, new ComparerAnonymousInnerClassHelper2(this));
if (topN < hits.Length)
{
ScoreDoc[] subset = new ScoreDoc[topN];
Array.Copy(hits, 0, subset, 0, topN);
hits = subset;
}
return(new TopDocs(firstPassTopDocs.TotalHits, hits, hits[0].Score));
}
public override TopDocs Rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int topN)
{
ScoreDoc[] hits = (ScoreDoc[])firstPassTopDocs.ScoreDocs.Clone();
Array.Sort(hits, Comparer <ScoreDoc> .Create((a, b) => a.Doc - b.Doc));
IList <AtomicReaderContext> leaves = searcher.IndexReader.Leaves;
Weight weight = searcher.CreateNormalizedWeight(query);
// Now merge sort docIDs from hits, with reader's leaves:
int hitUpto = 0;
int readerUpto = -1;
int endDoc = 0;
int docBase = 0;
Scorer scorer = null;
while (hitUpto < hits.Length)
{
ScoreDoc hit = hits[hitUpto];
int docID = hit.Doc;
AtomicReaderContext readerContext = null;
while (docID >= endDoc)
{
readerUpto++;
readerContext = leaves[readerUpto];
endDoc = readerContext.DocBase + readerContext.Reader.MaxDoc;
}
if (readerContext != null)
{
// We advanced to another segment:
docBase = readerContext.DocBase;
scorer = weight.GetScorer(readerContext, null);
}
int targetDoc = docID - docBase;
int actualDoc = scorer.DocID;
if (actualDoc < targetDoc)
{
actualDoc = scorer.Advance(targetDoc);
}
if (actualDoc == targetDoc)
{
// Query did match this doc:
hit.Score = Combine(hit.Score, true, scorer.GetScore());
}
else
{
// Query did not match this doc:
if (Debugging.AssertsEnabled)
{
Debugging.Assert(actualDoc > targetDoc);
}
hit.Score = Combine(hit.Score, false, 0.0f);
}
hitUpto++;
}
// TODO: we should do a partial sort (of only topN)
// instead, but typically the number of hits is
// smallish:
Array.Sort(hits, Comparer <ScoreDoc> .Create((a, b) =>
{
// Sort by score descending, then docID ascending:
// LUCENENET specific - compare bits rather than using equality operators to prevent these comparisons from failing in x86 in .NET Framework with optimizations enabled
if (NumericUtils.SingleToSortableInt32(a.Score) > NumericUtils.SingleToSortableInt32(b.Score))
{
return(-1);
}
else if (NumericUtils.SingleToSortableInt32(a.Score) < NumericUtils.SingleToSortableInt32(b.Score))
{
return(1);
}
else
{
// this subtraction can't overflow int
// because docIDs are >= 0:
return(a.Doc - b.Doc);
}
}));
if (topN < hits.Length)
{
ScoreDoc[] subset = new ScoreDoc[topN];
Array.Copy(hits, 0, subset, 0, topN);
hits = subset;
}
return(new TopDocs(firstPassTopDocs.TotalHits, hits, hits[0].Score));
}
public override DocIdSetIterator GetIterator()
{
return(weight.GetScorer(privateContext, acceptDocs));
}