public void TestRandom()
{
Random random = Random;
int numberOfRuns = TestUtil.NextInt32(random, 3, 6);
for (int indexIter = 0; indexIter < numberOfRuns; indexIter++)
{
bool multipleFacetsPerDocument = random.nextBoolean();
IndexContext context = CreateIndexContext(multipleFacetsPerDocument);
IndexSearcher searcher = NewSearcher(context.indexReader);
if (Verbose)
{
Console.WriteLine("TEST: searcher=" + searcher);
}
for (int searchIter = 0; searchIter < 100; searchIter++)
{
if (Verbose)
{
Console.WriteLine("TEST: searchIter=" + searchIter);
}
bool useDv = !multipleFacetsPerDocument && context.useDV && random.nextBoolean();
string searchTerm = context.contentStrings[random.nextInt(context.contentStrings.Length)];
int limit = random.nextInt(context.facetValues.size());
int offset = random.nextInt(context.facetValues.size() - limit);
int size = offset + limit;
int minCount = random.nextBoolean() ? 0 : random.nextInt(1 + context.facetWithMostGroups / 10);
bool orderByCount = random.nextBoolean();
string randomStr = GetFromSet(context.facetValues, random.nextInt(context.facetValues.size()));
string facetPrefix;
if (randomStr == null)
{
facetPrefix = null;
}
else
{
int codePointLen = randomStr.CodePointCount(0, randomStr.Length);
int randomLen = random.nextInt(codePointLen);
if (codePointLen == randomLen - 1)
{
facetPrefix = null;
}
else
{
int end = randomStr.OffsetByCodePoints(0, randomLen);
facetPrefix = random.nextBoolean() ? null : randomStr.Substring(end);
}
}
GroupedFacetResult expectedFacetResult = CreateExpectedFacetResult(searchTerm, context, offset, limit, minCount, orderByCount, facetPrefix);
AbstractGroupFacetCollector groupFacetCollector = CreateRandomCollector(useDv ? "group_dv" : "group", useDv ? "facet_dv" : "facet", facetPrefix, multipleFacetsPerDocument);
searcher.Search(new TermQuery(new Term("content", searchTerm)), groupFacetCollector);
TermGroupFacetCollector.GroupedFacetResult actualFacetResult = groupFacetCollector.MergeSegmentResults(size, minCount, orderByCount);
IList <TermGroupFacetCollector.FacetEntry> expectedFacetEntries = expectedFacetResult.GetFacetEntries();
IList <TermGroupFacetCollector.FacetEntry> actualFacetEntries = actualFacetResult.GetFacetEntries(offset, limit);
if (Verbose)
{
Console.WriteLine("Use DV: " + useDv);
Console.WriteLine("Collector: " + groupFacetCollector.GetType().Name);
Console.WriteLine("Num group: " + context.numGroups);
Console.WriteLine("Num doc: " + context.numDocs);
Console.WriteLine("Index iter: " + indexIter);
Console.WriteLine("multipleFacetsPerDocument: " + multipleFacetsPerDocument);
Console.WriteLine("Search iter: " + searchIter);
Console.WriteLine("Search term: " + searchTerm);
Console.WriteLine("Min count: " + minCount);
Console.WriteLine("Facet offset: " + offset);
Console.WriteLine("Facet limit: " + limit);
Console.WriteLine("Facet prefix: " + facetPrefix);
Console.WriteLine("Order by count: " + orderByCount);
Console.WriteLine("\n=== Expected: \n");
Console.WriteLine("Total count " + expectedFacetResult.TotalCount);
Console.WriteLine("Total missing count " + expectedFacetResult.TotalMissingCount);
int counter = 0;
foreach (TermGroupFacetCollector.FacetEntry expectedFacetEntry in expectedFacetEntries)
{
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"{0}. Expected facet value {1} with count {2}",
counter++, expectedFacetEntry.Value.Utf8ToString(), expectedFacetEntry.Count
)
);
}
Console.WriteLine("\n=== Actual: \n");
Console.WriteLine("Total count " + actualFacetResult.TotalCount);
Console.WriteLine("Total missing count " + actualFacetResult.TotalMissingCount);
counter = 0;
foreach (TermGroupFacetCollector.FacetEntry actualFacetEntry in actualFacetEntries)
{
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"{0}. Actual facet value {1} with count {2}",
counter++, actualFacetEntry.Value.Utf8ToString(), actualFacetEntry.Count
)
);
}
Console.WriteLine("\n===================================================================================");
}
assertEquals(expectedFacetResult.TotalCount, actualFacetResult.TotalCount);
assertEquals(expectedFacetResult.TotalMissingCount, actualFacetResult.TotalMissingCount);
assertEquals(expectedFacetEntries.size(), actualFacetEntries.size());
for (int i = 0; i < expectedFacetEntries.size(); i++)
{
TermGroupFacetCollector.FacetEntry expectedFacetEntry = expectedFacetEntries[i];
TermGroupFacetCollector.FacetEntry actualFacetEntry = actualFacetEntries[i];
assertEquals("i=" + i + ": " + expectedFacetEntry.Value.Utf8ToString() + " != " + actualFacetEntry.Value.Utf8ToString(), expectedFacetEntry.Value, actualFacetEntry.Value);
assertEquals("i=" + i + ": " + expectedFacetEntry.Count + " != " + actualFacetEntry.Count, expectedFacetEntry.Count, actualFacetEntry.Count);
}
}
context.indexReader.Dispose();
context.dir.Dispose();
}
}
public void TestSimple()
{
string groupField = "hotel";
FieldType customType = new FieldType();
customType.IsStored = true;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy()));
bool canUseDV = !"Lucene3x".Equals(w.IndexWriter.Config.Codec.Name, StringComparison.Ordinal);
bool useDv = canUseDV && Random.nextBoolean();
// 0
Document doc = new Document();
AddField(doc, groupField, "a", useDv);
AddField(doc, "airport", "ams", useDv);
AddField(doc, "duration", "5", useDv);
w.AddDocument(doc);
// 1
doc = new Document();
AddField(doc, groupField, "a", useDv);
AddField(doc, "airport", "dus", useDv);
AddField(doc, "duration", "10", useDv);
w.AddDocument(doc);
// 2
doc = new Document();
AddField(doc, groupField, "b", useDv);
AddField(doc, "airport", "ams", useDv);
AddField(doc, "duration", "10", useDv);
w.AddDocument(doc);
w.Commit(); // To ensure a second segment
// 3
doc = new Document();
AddField(doc, groupField, "b", useDv);
AddField(doc, "airport", "ams", useDv);
AddField(doc, "duration", "5", useDv);
w.AddDocument(doc);
// 4
doc = new Document();
AddField(doc, groupField, "b", useDv);
AddField(doc, "airport", "ams", useDv);
AddField(doc, "duration", "5", useDv);
w.AddDocument(doc);
IndexSearcher indexSearcher = NewSearcher(w.GetReader());
IList <TermGroupFacetCollector.FacetEntry> entries = null;
AbstractGroupFacetCollector groupedAirportFacetCollector = null;
TermGroupFacetCollector.GroupedFacetResult airportResult = null;
foreach (int limit in new int[] { 2, 10, 100, int.MaxValue })
{
// any of these limits is plenty for the data we have
groupedAirportFacetCollector = CreateRandomCollector
(useDv ? "hotel_dv" : "hotel",
useDv ? "airport_dv" : "airport", null, false);
indexSearcher.Search(new MatchAllDocsQuery(), groupedAirportFacetCollector);
int maxOffset = 5;
airportResult = groupedAirportFacetCollector.MergeSegmentResults
(int.MaxValue == limit ? limit : maxOffset + limit, 0, false);
assertEquals(3, airportResult.TotalCount);
assertEquals(0, airportResult.TotalMissingCount);
entries = airportResult.GetFacetEntries(maxOffset, limit);
assertEquals(0, entries.size());
entries = airportResult.GetFacetEntries(0, limit);
assertEquals(2, entries.size());
assertEquals("ams", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("dus", entries[1].Value.Utf8ToString());
assertEquals(1, entries[1].Count);
entries = airportResult.GetFacetEntries(1, limit);
assertEquals(1, entries.size());
assertEquals("dus", entries[0].Value.Utf8ToString());
assertEquals(1, entries[0].Count);
}
AbstractGroupFacetCollector groupedDurationFacetCollector = CreateRandomCollector(useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", null, false);
indexSearcher.Search(new MatchAllDocsQuery(), groupedDurationFacetCollector);
TermGroupFacetCollector.GroupedFacetResult durationResult = groupedDurationFacetCollector.MergeSegmentResults(10, 0, false);
assertEquals(4, durationResult.TotalCount);
assertEquals(0, durationResult.TotalMissingCount);
entries = durationResult.GetFacetEntries(0, 10);
assertEquals(2, entries.size());
assertEquals("10", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("5", entries[1].Value.Utf8ToString());
assertEquals(2, entries[1].Count);
// 5
doc = new Document();
AddField(doc, groupField, "b", useDv);
// missing airport
if (useDv)
{
AddField(doc, "airport", "", useDv);
}
AddField(doc, "duration", "5", useDv);
w.AddDocument(doc);
// 6
doc = new Document();
AddField(doc, groupField, "b", useDv);
AddField(doc, "airport", "bru", useDv);
AddField(doc, "duration", "10", useDv);
w.AddDocument(doc);
// 7
doc = new Document();
AddField(doc, groupField, "b", useDv);
AddField(doc, "airport", "bru", useDv);
AddField(doc, "duration", "15", useDv);
w.AddDocument(doc);
// 8
doc = new Document();
AddField(doc, groupField, "a", useDv);
AddField(doc, "airport", "bru", useDv);
AddField(doc, "duration", "10", useDv);
w.AddDocument(doc);
indexSearcher.IndexReader.Dispose();
indexSearcher = NewSearcher(w.GetReader());
groupedAirportFacetCollector = CreateRandomCollector(useDv ? "hotel_dv" : "hotel", useDv ? "airport_dv" : "airport", null, !useDv);
indexSearcher.Search(new MatchAllDocsQuery(), groupedAirportFacetCollector);
airportResult = groupedAirportFacetCollector.MergeSegmentResults(3, 0, true);
entries = airportResult.GetFacetEntries(1, 2);
assertEquals(2, entries.size());
if (useDv)
{
assertEquals(6, airportResult.TotalCount);
assertEquals(0, airportResult.TotalMissingCount);
assertEquals("bru", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("", entries[1].Value.Utf8ToString());
assertEquals(1, entries[1].Count);
}
else
{
assertEquals(5, airportResult.TotalCount);
assertEquals(1, airportResult.TotalMissingCount);
assertEquals("bru", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("dus", entries[1].Value.Utf8ToString());
assertEquals(1, entries[1].Count);
}
groupedDurationFacetCollector = CreateRandomCollector(useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", null, false);
indexSearcher.Search(new MatchAllDocsQuery(), groupedDurationFacetCollector);
durationResult = groupedDurationFacetCollector.MergeSegmentResults(10, 2, true);
assertEquals(5, durationResult.TotalCount);
assertEquals(0, durationResult.TotalMissingCount);
entries = durationResult.GetFacetEntries(1, 1);
assertEquals(1, entries.size());
assertEquals("5", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
// 9
doc = new Document();
AddField(doc, groupField, "c", useDv);
AddField(doc, "airport", "bru", useDv);
AddField(doc, "duration", "15", useDv);
w.AddDocument(doc);
// 10
doc = new Document();
AddField(doc, groupField, "c", useDv);
AddField(doc, "airport", "dus", useDv);
AddField(doc, "duration", "10", useDv);
w.AddDocument(doc);
indexSearcher.IndexReader.Dispose();
indexSearcher = NewSearcher(w.GetReader());
groupedAirportFacetCollector = CreateRandomCollector(useDv ? "hotel_dv" : "hotel", useDv ? "airport_dv" : "airport", null, false);
indexSearcher.Search(new MatchAllDocsQuery(), groupedAirportFacetCollector);
airportResult = groupedAirportFacetCollector.MergeSegmentResults(10, 0, false);
entries = airportResult.GetFacetEntries(0, 10);
if (useDv)
{
assertEquals(8, airportResult.TotalCount);
assertEquals(0, airportResult.TotalMissingCount);
assertEquals(4, entries.size());
assertEquals("", entries[0].Value.Utf8ToString());
assertEquals(1, entries[0].Count);
assertEquals("ams", entries[1].Value.Utf8ToString());
assertEquals(2, entries[1].Count);
assertEquals("bru", entries[2].Value.Utf8ToString());
assertEquals(3, entries[2].Count);
assertEquals("dus", entries[3].Value.Utf8ToString());
assertEquals(2, entries[3].Count);
}
else
{
assertEquals(7, airportResult.TotalCount);
assertEquals(1, airportResult.TotalMissingCount);
assertEquals(3, entries.size());
assertEquals("ams", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("bru", entries[1].Value.Utf8ToString());
assertEquals(3, entries[1].Count);
assertEquals("dus", entries[2].Value.Utf8ToString());
assertEquals(2, entries[2].Count);
}
groupedDurationFacetCollector = CreateRandomCollector(useDv ? "hotel_dv" : "hotel", useDv ? "duration_dv" : "duration", "1", false);
indexSearcher.Search(new MatchAllDocsQuery(), groupedDurationFacetCollector);
durationResult = groupedDurationFacetCollector.MergeSegmentResults(10, 0, true);
assertEquals(5, durationResult.TotalCount);
assertEquals(0, durationResult.TotalMissingCount);
entries = durationResult.GetFacetEntries(0, 10);
assertEquals(2, entries.size());
assertEquals("10", entries[0].Value.Utf8ToString());
assertEquals(3, entries[0].Count);
assertEquals("15", entries[1].Value.Utf8ToString());
assertEquals(2, entries[1].Count);
w.Dispose();
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}
public void TestMVGroupedFacetingWithDeletes()
{
string groupField = "hotel";
FieldType customType = new FieldType();
customType.IsStored = (true);
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
bool useDv = false;
// Cannot assert this since we use NoMergePolicy:
w.DoRandomForceMergeAssert = (false);
// 0
Document doc = new Document();
doc.Add(new StringField("x", "x", Field.Store.NO));
w.AddDocument(doc);
// 1
doc = new Document();
AddField(doc, groupField, "a", useDv);
doc.Add(new StringField("airport", "ams", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
w.DeleteDocuments(new TermQuery(new Term("airport", "ams")));
// 2
doc = new Document();
AddField(doc, groupField, "a", useDv);
doc.Add(new StringField("airport", "ams", Field.Store.NO));
w.AddDocument(doc);
// 3
doc = new Document();
AddField(doc, groupField, "a", useDv);
doc.Add(new StringField("airport", "dus", Field.Store.NO));
w.AddDocument(doc);
// 4
doc = new Document();
AddField(doc, groupField, "b", useDv);
doc.Add(new StringField("airport", "ams", Field.Store.NO));
w.AddDocument(doc);
// 5
doc = new Document();
AddField(doc, groupField, "b", useDv);
doc.Add(new StringField("airport", "ams", Field.Store.NO));
w.AddDocument(doc);
// 6
doc = new Document();
AddField(doc, groupField, "b", useDv);
doc.Add(new StringField("airport", "ams", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
// 7
doc = new Document();
doc.Add(new StringField("x", "x", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
w.Dispose();
IndexSearcher indexSearcher = NewSearcher(DirectoryReader.Open(dir));
AbstractGroupFacetCollector groupedAirportFacetCollector = CreateRandomCollector(groupField, "airport", null, true);
indexSearcher.Search(new MatchAllDocsQuery(), groupedAirportFacetCollector);
TermGroupFacetCollector.GroupedFacetResult airportResult = groupedAirportFacetCollector.MergeSegmentResults(10, 0, false);
assertEquals(3, airportResult.TotalCount);
assertEquals(1, airportResult.TotalMissingCount);
IList <TermGroupFacetCollector.FacetEntry> entries = airportResult.GetFacetEntries(0, 10);
assertEquals(2, entries.size());
assertEquals("ams", entries[0].Value.Utf8ToString());
assertEquals(2, entries[0].Count);
assertEquals("dus", entries[1].Value.Utf8ToString());
assertEquals(1, entries[1].Count);
indexSearcher.IndexReader.Dispose();
dir.Dispose();
}