public virtual void VerifyNrHits(Query q, int expected)
{
// bs1
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
if (expected != h.Length)
{
PrintHits(TestName, h, s);
}
Assert.AreEqual(expected, h.Length, "result count");
//System.out.println("TEST: now check");
// bs2
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
s.Search(q, collector);
ScoreDoc[] h2 = collector.GetTopDocs().ScoreDocs;
if (expected != h2.Length)
{
PrintHits(TestName, h2, s);
}
Assert.AreEqual(expected, h2.Length, "result count (bs2)");
QueryUtils.Check(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, q, s);
}
public virtual void TestBooleanQuery()
{
TermQuery aQuery = new TermQuery(new Term("f", "a"));
TermQuery dQuery = new TermQuery(new Term("f", "d"));
TermQuery cQuery = new TermQuery(new Term("f", "c"));
TermQuery yQuery = new TermQuery(new Term("f", "y"));
BooleanQuery query = new BooleanQuery();
BooleanQuery inner = new BooleanQuery();
inner.Add(cQuery, Occur.SHOULD);
inner.Add(yQuery, Occur.MUST_NOT);
query.Add(inner, Occur.MUST);
query.Add(aQuery, Occur.MUST);
query.Add(dQuery, Occur.MUST);
// Only needed in Java6; Java7+ has a @SafeVarargs annotated Arrays#asList()!
// see http://docs.oracle.com/javase/7/docs/api/java/lang/SafeVarargs.html
IEnumerable <ISet <string> > occurList = new ISet <string>[] { new JCG.HashSet <string> {
"MUST"
}, new JCG.HashSet <string> {
"MUST", "SHOULD"
} };
foreach (var occur in occurList)
{
var c = new CountingCollector(TopScoreDocCollector.Create(10, true), occur);
s.Search(query, null, c);
int maxDocs = s.IndexReader.MaxDoc;
Assert.AreEqual(maxDocs, c.DocCounts.Count);
bool includeOptional = occur.Contains("SHOULD");
for (int i = 0; i < maxDocs; i++)
{
IDictionary <Query, float?> doc0 = c.DocCounts[i];
Assert.AreEqual(includeOptional ? 5 : 4, doc0.Count);
Assert.AreEqual(1.0F, doc0[aQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
Assert.AreEqual(4.0F, doc0[dQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
if (includeOptional)
{
Assert.AreEqual(3.0F, doc0[cQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
}
IDictionary <Query, float?> doc1 = c.DocCounts[++i];
Assert.AreEqual(includeOptional ? 5 : 4, doc1.Count);
Assert.AreEqual(1.0F, doc1[aQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
Assert.AreEqual(1.0F, doc1[dQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
if (includeOptional)
{
Assert.AreEqual(1.0F, doc1[cQuery].GetValueOrDefault(), FLOAT_TOLERANCE);
}
}
}
}
// inherit javadoc
public override TopDocs Search(Weight weight, Filter filter, int nDocs, IState state)
{
if (nDocs <= 0)
{
throw new System.ArgumentException("nDocs must be > 0");
}
nDocs = Math.Min(nDocs, reader.MaxDoc);
TopScoreDocCollector collector = TopScoreDocCollector.Create(nDocs, !weight.GetScoresDocsOutOfOrder());
Search(weight, filter, collector, state);
return(collector.TopDocs());
}
public virtual void TestNegativeScores()
{
// The Top*Collectors previously filtered out documents with <= scores. this
// behavior has changed. this test checks that if PositiveOnlyScoresFilter
// wraps one of these collectors, documents with <= 0 scores are indeed
// filtered.
int numPositiveScores = 0;
for (int i = 0; i < Scores.Length; i++)
{
if (Scores[i] > 0)
{
++numPositiveScores;
}
}
Directory directory = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, directory);
writer.Commit();
IndexReader ir = writer.GetReader();
writer.Dispose();
IndexSearcher searcher = NewSearcher(ir);
Weight fake = (new TermQuery(new Term("fake", "weight"))).CreateWeight(searcher);
Scorer s = new SimpleScorer(fake);
TopDocsCollector <ScoreDoc> tdc = TopScoreDocCollector.Create(Scores.Length, true);
ICollector c = new PositiveScoresOnlyCollector(tdc);
c.SetScorer(s);
while (s.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
c.Collect(0);
}
TopDocs td = tdc.GetTopDocs();
ScoreDoc[] sd = td.ScoreDocs;
Assert.AreEqual(numPositiveScores, td.TotalHits);
for (int i = 0; i < sd.Length; i++)
{
Assert.IsTrue(sd[i].Score > 0, "only positive scores should return: " + sd[i].Score);
}
ir.Dispose();
directory.Dispose();
}
public virtual void QueriesTest(Query query, int[] expDocNrs)
{
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, false);
Searcher.Search(query, null, collector);
ScoreDoc[] hits1 = collector.GetTopDocs().ScoreDocs;
collector = TopScoreDocCollector.Create(1000, true);
Searcher.Search(query, null, collector);
ScoreDoc[] hits2 = collector.GetTopDocs().ScoreDocs;
Assert.AreEqual(MulFactor * collector.TotalHits, BigSearcher.Search(query, 1).TotalHits);
CheckHits.CheckHitsQuery(query, hits1, hits2, expDocNrs);
}
/// <summary>
/// Expert: Low-level search implementation. Finds the top <code>n</code>
/// hits for <code>query</code>.
///
/// <p>Applications should usually call <seealso cref="IndexSearcher#search(Query,int)"/> or
/// <seealso cref="IndexSearcher#search(Query,Filter,int)"/> instead. </summary>
/// <exception cref="BooleanQuery.TooManyClauses"> If a query would exceed
/// <seealso cref="BooleanQuery#getMaxClauseCount()"/> clauses. </exception>
protected internal virtual TopDocs Search(IList <AtomicReaderContext> leaves, Weight weight, ScoreDoc after, int nDocs)
{
// single thread
int limit = Reader.MaxDoc;
if (limit == 0)
{
limit = 1;
}
nDocs = Math.Min(nDocs, limit);
TopScoreDocCollector collector = TopScoreDocCollector.Create(nDocs, after, !weight.ScoresDocsOutOfOrder());
Search(leaves, weight, collector);
return(collector.TopDocs());
}
public virtual void TestOutOfOrderCollection()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
for (int i = 0; i < 10; i++)
{
writer.AddDocument(new Document());
}
bool[] inOrder = new bool[] { false, true };
string[] actualTSDCClass = new string[] { "OutOfOrderTopScoreDocCollector", "InOrderTopScoreDocCollector" };
BooleanQuery bq = new BooleanQuery();
// Add a Query with SHOULD, since bw.Scorer() returns BooleanScorer2
// which delegates to BS if there are no mandatory clauses.
bq.Add(new MatchAllDocsQuery(), Occur.SHOULD);
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.MinimumNumberShouldMatch = 1;
IndexReader reader = writer.GetReader();
IndexSearcher searcher = NewSearcher(reader);
for (int i = 0; i < inOrder.Length; i++)
{
TopDocsCollector <ScoreDoc> tdc = TopScoreDocCollector.Create(3, inOrder[i]);
Assert.AreEqual("Lucene.Net.Search.TopScoreDocCollector+" + actualTSDCClass[i], tdc.GetType().FullName);
searcher.Search(new MatchAllDocsQuery(), tdc);
ScoreDoc[] sd = tdc.GetTopDocs().ScoreDocs;
Assert.AreEqual(3, sd.Length);
for (int j = 0; j < sd.Length; j++)
{
Assert.AreEqual(j, sd[j].Doc, "expected doc Id " + j + " found " + sd[j].Doc);
}
}
writer.Dispose();
reader.Dispose();
dir.Dispose();
}
public virtual void TestTermQuery()
{
TermQuery q = new TermQuery(new Term("f", "d"));
CountingCollector c = new CountingCollector(TopScoreDocCollector.Create(10, true));
s.Search(q, null, c);
int maxDocs = s.IndexReader.MaxDoc;
Assert.AreEqual(maxDocs, c.DocCounts.Count);
for (int i = 0; i < maxDocs; i++)
{
IDictionary <Query, float?> doc0 = c.DocCounts[i];
Assert.AreEqual(1, doc0.Count);
Assert.AreEqual(4.0F, doc0[q], FLOAT_TOLERANCE);
IDictionary <Query, float?> doc1 = c.DocCounts[++i];
Assert.AreEqual(1, doc1.Count);
Assert.AreEqual(1.0F, doc1[q], FLOAT_TOLERANCE);
}
}
public virtual void QueriesTest(System.String queryText, int[] expDocNrs)
{
//System.out.println();
//System.out.println("Query: " + queryText);
Query query1 = MakeQuery(queryText);
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, false);
searcher.Search(query1, null, collector, null);
ScoreDoc[] hits1 = collector.TopDocs().ScoreDocs;
Query query2 = MakeQuery(queryText); // there should be no need to parse again...
collector = TopScoreDocCollector.Create(1000, true);
searcher.Search(query2, null, collector, null);
ScoreDoc[] hits2 = collector.TopDocs().ScoreDocs;
Assert.AreEqual(mulFactor * collector.internalTotalHits, bigSearcher.Search(query1, 1, null).TotalHits);
CheckHits.CheckHitsQuery(query2, hits1, hits2, expDocNrs);
}
public virtual void TestOutOfOrderCollection()
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, null, MaxFieldLength.UNLIMITED);
for (int i = 0; i < 10; i++)
{
writer.AddDocument(new Document());
}
writer.Commit();
writer.Close();
bool[] inOrder = new bool[] { false, true };
System.String[] actualTSDCClass = new System.String[]
{ "OutOfOrderTopScoreDocCollector", "InOrderTopScoreDocCollector" };
BooleanQuery bq = new BooleanQuery();
// Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
// which delegates to BS if there are no mandatory clauses.
bq.Add(new MatchAllDocsQuery(), Occur.SHOULD);
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.MinimumNumberShouldMatch = 1;
IndexSearcher searcher = new IndexSearcher(dir, true);
for (int i = 0; i < inOrder.Length; i++)
{
TopDocsCollector <ScoreDoc> tdc = TopScoreDocCollector.Create(3, inOrder[i]);
Assert.AreEqual("Lucene.Net.Search.TopScoreDocCollector+" + actualTSDCClass[i], tdc.GetType().FullName);
searcher.Search(new MatchAllDocsQuery(), tdc);
ScoreDoc[] sd = tdc.TopDocs().ScoreDocs;
Assert.AreEqual(3, sd.Length);
for (int j = 0; j < sd.Length; j++)
{
Assert.AreEqual(j, sd[j].Doc, "expected doc Id " + j + " found " + sd[j].Doc);
}
}
}
public virtual void TestBooleanSpanQuery()
{
bool failed = false;
int hits = 0;
Directory directory = NewDirectory();
Analyzer indexerAnalyzer = new MockAnalyzer(Random);
IndexWriterConfig config = new IndexWriterConfig(TEST_VERSION_CURRENT, indexerAnalyzer);
IndexWriter writer = new IndexWriter(directory, config);
string FIELD = "content";
Document d = new Document();
d.Add(new TextField(FIELD, "clockwork orange", Field.Store.YES));
writer.AddDocument(d);
writer.Dispose();
IndexReader indexReader = DirectoryReader.Open(directory);
IndexSearcher searcher = NewSearcher(indexReader);
BooleanQuery query = new BooleanQuery();
SpanQuery sq1 = new SpanTermQuery(new Term(FIELD, "clockwork"));
SpanQuery sq2 = new SpanTermQuery(new Term(FIELD, "clckwork"));
query.Add(sq1, Occur.SHOULD);
query.Add(sq2, Occur.SHOULD);
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
searcher.Search(query, collector);
hits = collector.GetTopDocs().ScoreDocs.Length;
foreach (ScoreDoc scoreDoc in collector.GetTopDocs().ScoreDocs)
{
Console.WriteLine(scoreDoc.Doc);
}
indexReader.Dispose();
Assert.AreEqual(failed, false, "Bug in boolean query composed of span queries");
Assert.AreEqual(hits, 1, "Bug in boolean query composed of span queries");
directory.Dispose();
}
public virtual void VerifyNrHits(Query q, int expected)
{
// bs1
ScoreDoc[] h = s.Search(q, null, 1000).ScoreDocs;
if (expected != h.Length)
{
PrintHits(TestName, h, s);
}
Assert.AreEqual(expected, h.Length, "result count");
//System.out.println("TEST: now check");
// bs2
TopScoreDocCollector collector = TopScoreDocCollector.Create(1000, true);
s.Search(q, collector);
ScoreDoc[] h2 = collector.GetTopDocs().ScoreDocs;
if (expected != h2.Length)
{
PrintHits(TestName, h2, s);
}
Assert.AreEqual(expected, h2.Length, "result count (bs2)");
QueryUtils.Check(Random(), q, s, Similarity);
}
internal virtual void TestSort(bool useFrom)
{
IndexReader reader = null;
Directory dir = null;
int numDocs = AtLeast(1000);
//final int numDocs = AtLeast(50);
string[] tokens = new string[] { "a", "b", "c", "d", "e" };
if (VERBOSE)
{
Console.WriteLine("TEST: make index");
}
{
dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir);
// w.setDoRandomForceMerge(false);
// w.w.getConfig().SetMaxBufferedDocs(AtLeast(100));
string[] content = new string[AtLeast(20)];
for (int contentIDX = 0; contentIDX < content.Length; contentIDX++)
{
StringBuilder sb = new StringBuilder();
int numTokens = TestUtil.NextInt(Random(), 1, 10);
for (int tokenIDX = 0; tokenIDX < numTokens; tokenIDX++)
{
sb.Append(tokens[Random().Next(tokens.Length)]).Append(' ');
}
content[contentIDX] = sb.ToString();
}
for (int docIDX = 0; docIDX < numDocs; docIDX++)
{
Document doc = new Document();
doc.Add(NewStringField("string", TestUtil.RandomRealisticUnicodeString(Random()), Field.Store.NO));
doc.Add(NewTextField("text", content[Random().Next(content.Length)], Field.Store.NO));
doc.Add(new FloatField("float", (float)Random().NextDouble(), Field.Store.NO));
int intValue;
if (Random().Next(100) == 17)
{
intValue = int.MinValue;
}
else if (Random().Next(100) == 17)
{
intValue = int.MaxValue;
}
else
{
intValue = Random().Next();
}
doc.Add(new IntField("int", intValue, Field.Store.NO));
if (VERBOSE)
{
Console.WriteLine(" doc=" + doc);
}
w.AddDocument(doc);
}
reader = w.Reader;
w.Dispose();
}
// NOTE: sometimes reader has just one segment, which is
// important to test
IndexSearcher searcher = NewSearcher(reader);
IndexReaderContext ctx = searcher.TopReaderContext;
ShardSearcher[] subSearchers;
int[] docStarts;
if (ctx is AtomicReaderContext)
{
subSearchers = new ShardSearcher[1];
docStarts = new int[1];
subSearchers[0] = new ShardSearcher((AtomicReaderContext)ctx, ctx);
docStarts[0] = 0;
}
else
{
CompositeReaderContext compCTX = (CompositeReaderContext)ctx;
int size = compCTX.Leaves.Count;
subSearchers = new ShardSearcher[size];
docStarts = new int[size];
int docBase = 0;
for (int searcherIDX = 0; searcherIDX < subSearchers.Length; searcherIDX++)
{
AtomicReaderContext leave = compCTX.Leaves[searcherIDX];
subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX);
docStarts[searcherIDX] = docBase;
docBase += leave.Reader.MaxDoc;
}
}
IList <SortField> sortFields = new List <SortField>();
sortFields.Add(new SortField("string", SortField.Type_e.STRING, true));
sortFields.Add(new SortField("string", SortField.Type_e.STRING, false));
sortFields.Add(new SortField("int", SortField.Type_e.INT, true));
sortFields.Add(new SortField("int", SortField.Type_e.INT, false));
sortFields.Add(new SortField("float", SortField.Type_e.FLOAT, true));
sortFields.Add(new SortField("float", SortField.Type_e.FLOAT, false));
sortFields.Add(new SortField(null, SortField.Type_e.SCORE, true));
sortFields.Add(new SortField(null, SortField.Type_e.SCORE, false));
sortFields.Add(new SortField(null, SortField.Type_e.DOC, true));
sortFields.Add(new SortField(null, SortField.Type_e.DOC, false));
for (int iter = 0; iter < 1000 * RANDOM_MULTIPLIER; iter++)
{
// TODO: custom FieldComp...
Query query = new TermQuery(new Term("text", tokens[Random().Next(tokens.Length)]));
Sort sort;
if (Random().Next(10) == 4)
{
// Sort by score
sort = null;
}
else
{
SortField[] randomSortFields = new SortField[TestUtil.NextInt(Random(), 1, 3)];
for (int sortIDX = 0; sortIDX < randomSortFields.Length; sortIDX++)
{
randomSortFields[sortIDX] = sortFields[Random().Next(sortFields.Count)];
}
sort = new Sort(randomSortFields);
}
int numHits = TestUtil.NextInt(Random(), 1, numDocs + 5);
//final int numHits = 5;
if (VERBOSE)
{
Console.WriteLine("TEST: search query=" + query + " sort=" + sort + " numHits=" + numHits);
}
int from = -1;
int size = -1;
// First search on whole index:
TopDocs topHits;
if (sort == null)
{
if (useFrom)
{
TopScoreDocCollector c = TopScoreDocCollector.Create(numHits, Random().NextBoolean());
searcher.Search(query, c);
from = TestUtil.NextInt(Random(), 0, numHits - 1);
size = numHits - from;
TopDocs tempTopHits = c.TopDocs();
if (from < tempTopHits.ScoreDocs.Length)
{
// Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
// than TopDocs#merge currently has
ScoreDoc[] newScoreDocs = new ScoreDoc[Math.Min(size, tempTopHits.ScoreDocs.Length - from)];
Array.Copy(tempTopHits.ScoreDocs, from, newScoreDocs, 0, newScoreDocs.Length);
tempTopHits.ScoreDocs = newScoreDocs;
topHits = tempTopHits;
}
else
{
topHits = new TopDocs(tempTopHits.TotalHits, new ScoreDoc[0], tempTopHits.MaxScore);
}
}
else
{
topHits = searcher.Search(query, numHits);
}
}
else
{
TopFieldCollector c = TopFieldCollector.Create(sort, numHits, true, true, true, Random().NextBoolean());
searcher.Search(query, c);
if (useFrom)
{
from = TestUtil.NextInt(Random(), 0, numHits - 1);
size = numHits - from;
TopDocs tempTopHits = c.TopDocs();
if (from < tempTopHits.ScoreDocs.Length)
{
// Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
// than TopDocs#merge currently has
ScoreDoc[] newScoreDocs = new ScoreDoc[Math.Min(size, tempTopHits.ScoreDocs.Length - from)];
Array.Copy(tempTopHits.ScoreDocs, from, newScoreDocs, 0, newScoreDocs.Length);
tempTopHits.ScoreDocs = newScoreDocs;
topHits = tempTopHits;
}
else
{
topHits = new TopDocs(tempTopHits.TotalHits, new ScoreDoc[0], tempTopHits.MaxScore);
}
}
else
{
topHits = c.TopDocs(0, numHits);
}
}
if (VERBOSE)
{
if (useFrom)
{
Console.WriteLine("from=" + from + " size=" + size);
}
Console.WriteLine(" top search: " + topHits.TotalHits + " totalHits; hits=" + (topHits.ScoreDocs == null ? "null" : topHits.ScoreDocs.Length + " maxScore=" + topHits.MaxScore));
if (topHits.ScoreDocs != null)
{
for (int hitIDX = 0; hitIDX < topHits.ScoreDocs.Length; hitIDX++)
{
ScoreDoc sd = topHits.ScoreDocs[hitIDX];
Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score);
}
}
}
// ... then all shards:
Weight w = searcher.CreateNormalizedWeight(query);
TopDocs[] shardHits = new TopDocs[subSearchers.Length];
for (int shardIDX = 0; shardIDX < subSearchers.Length; shardIDX++)
{
TopDocs subHits;
ShardSearcher subSearcher = subSearchers[shardIDX];
if (sort == null)
{
subHits = subSearcher.Search(w, numHits);
}
else
{
TopFieldCollector c = TopFieldCollector.Create(sort, numHits, true, true, true, Random().NextBoolean());
subSearcher.Search(w, c);
subHits = c.TopDocs(0, numHits);
}
shardHits[shardIDX] = subHits;
if (VERBOSE)
{
Console.WriteLine(" shard=" + shardIDX + " " + subHits.TotalHits + " totalHits hits=" + (subHits.ScoreDocs == null ? "null" : subHits.ScoreDocs.Length.ToString()));
if (subHits.ScoreDocs != null)
{
foreach (ScoreDoc sd in subHits.ScoreDocs)
{
Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score);
}
}
}
}
// Merge:
TopDocs mergedHits;
if (useFrom)
{
mergedHits = TopDocs.Merge(sort, from, size, shardHits);
}
else
{
mergedHits = TopDocs.Merge(sort, numHits, shardHits);
}
if (mergedHits.ScoreDocs != null)
{
// Make sure the returned shards are correct:
for (int hitIDX = 0; hitIDX < mergedHits.ScoreDocs.Length; hitIDX++)
{
ScoreDoc sd = mergedHits.ScoreDocs[hitIDX];
Assert.AreEqual(ReaderUtil.SubIndex(sd.Doc, docStarts), sd.ShardIndex, "doc=" + sd.Doc + " wrong shard");
}
}
TestUtil.AssertEquals(topHits, mergedHits);
}
reader.Dispose();
dir.Dispose();
}
internal MyCollector()
{
Collector = TopScoreDocCollector.Create(10, true);
}