public virtual void TestRandomFloats()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir);
int numDocs = AtLeast(1000);
float[] values = new float[numDocs];
float minValue = float.PositiveInfinity;
float maxValue = float.NegativeInfinity;
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
float v = Random().NextFloat();
values[i] = v;
doc.Add(new FloatDocValuesField("field", v));
doc.Add(new FloatField("field", v, Field.Store.NO));
w.AddDocument(doc);
minValue = Math.Min(minValue, v);
maxValue = Math.Max(maxValue, v);
}
IndexReader r = w.Reader;
IndexSearcher s = NewSearcher(r);
FacetsConfig config = new FacetsConfig();
int numIters = AtLeast(10);
for (int iter = 0; iter < numIters; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
int numRange = TestUtil.NextInt(Random(), 1, 5);
DoubleRange[] ranges = new DoubleRange[numRange];
int[] expectedCounts = new int[numRange];
float minAcceptedValue = float.PositiveInfinity;
float maxAcceptedValue = float.NegativeInfinity;
if (VERBOSE)
{
Console.WriteLine("TEST: " + numRange + " ranges");
}
for (int rangeID = 0; rangeID < numRange; rangeID++)
{
double min;
if (rangeID > 0 && Random().Next(10) == 7)
{
// Use an existing boundary:
DoubleRange prevRange = ranges[Random().Next(rangeID)];
if (Random().NextBoolean())
{
min = prevRange.Min;
}
else
{
min = prevRange.Max;
}
}
else
{
min = Random().NextDouble();
}
double max;
if (rangeID > 0 && Random().Next(10) == 7)
{
// Use an existing boundary:
DoubleRange prevRange = ranges[Random().Next(rangeID)];
if (Random().NextBoolean())
{
max = prevRange.Min;
}
else
{
max = prevRange.Max;
}
}
else
{
max = Random().NextDouble();
}
if (min > max)
{
double x = min;
min = max;
max = x;
}
// Must truncate to float precision so that the
// drill-down counts (which use NRQ.newFloatRange)
// are correct:
min = (float)min;
max = (float)max;
bool minIncl;
bool maxIncl;
if (min == max)
{
minIncl = true;
maxIncl = true;
}
else
{
minIncl = Random().NextBoolean();
maxIncl = Random().NextBoolean();
}
ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl);
if (VERBOSE)
{
Console.WriteLine("TEST: range " + rangeID + ": " + ranges[rangeID]);
}
// Do "slow but hopefully correct" computation of
// expected count:
for (int i = 0; i < numDocs; i++)
{
bool accept = true;
if (minIncl)
{
accept &= values[i] >= min;
}
else
{
accept &= values[i] > min;
}
if (maxIncl)
{
accept &= values[i] <= max;
}
else
{
accept &= values[i] < max;
}
if (VERBOSE)
{
Console.WriteLine("TEST: check doc=" + i + " val=" + values[i] + " accept=" + accept);
}
if (accept)
{
expectedCounts[rangeID]++;
minAcceptedValue = Math.Min(minAcceptedValue, values[i]);
maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]);
}
}
}
FacetsCollector sfc = new FacetsCollector();
s.Search(new MatchAllDocsQuery(), sfc);
Filter fastMatchFilter;
if (Random().NextBoolean())
{
if (Random().NextBoolean())
{
fastMatchFilter = NumericRangeFilter.NewFloatRange("field", minValue, maxValue, true, true);
}
else
{
fastMatchFilter = NumericRangeFilter.NewFloatRange("field", minAcceptedValue, maxAcceptedValue, true, true);
}
}
else
{
fastMatchFilter = null;
}
ValueSource vs = new FloatFieldSource("field");
Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges);
FacetResult result = facets.GetTopChildren(10, "field");
Assert.AreEqual(numRange, result.LabelValues.Length);
for (int rangeID = 0; rangeID < numRange; rangeID++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: verify range " + rangeID + " expectedCount=" + expectedCounts[rangeID]);
}
LabelAndValue subNode = result.LabelValues[rangeID];
Assert.AreEqual("r" + rangeID, subNode.label);
Assert.AreEqual(expectedCounts[rangeID], (int)subNode.value);
DoubleRange range = ranges[rangeID];
// Test drill-down:
DrillDownQuery ddq = new DrillDownQuery(config);
if (Random().NextBoolean())
{
if (Random().NextBoolean())
{
ddq.Add("field", NumericRangeFilter.NewFloatRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive));
}
else
{
ddq.Add("field", NumericRangeQuery.NewFloatRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive));
}
}
else
{
ddq.Add("field", range.GetFilter(fastMatchFilter, vs));
}
Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits);
}
}
IOUtils.Close(w, r, dir);
}
public virtual void TestRandom()
{
string[] tokens = GetRandomTokens(10);
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), indexDir);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
int numDocs = AtLeast(1000);
int numDims = TestUtil.NextInt(Random(), 1, 7);
IList<TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
foreach (TestDoc testDoc in testDocs)
{
Document doc = new Document();
doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
testDoc.value = Random().NextFloat();
doc.Add(new FloatDocValuesField("value", testDoc.value));
for (int j = 0; j < numDims; j++)
{
if (testDoc.dims[j] != null)
{
doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
}
}
w.AddDocument(config.Build(tw, doc));
}
// NRT open
IndexSearcher searcher = NewSearcher(w.Reader);
// NRT open
var tr = new DirectoryTaxonomyReader(tw);
ValueSource values = new FloatFieldSource("value");
int iters = AtLeast(100);
for (int iter = 0; iter < iters; iter++)
{
string searchToken = tokens[Random().Next(tokens.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter content=" + searchToken);
}
FacetsCollector fc = new FacetsCollector();
FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
Facets facets = new TaxonomyFacetSumValueSource(tr, config, fc, values);
// Slow, yet hopefully bug-free, faceting:
var expectedValues = new List<Dictionary<string, float?>>();
for (int i = 0; i < numDims; i++)
{
expectedValues[i] = new Dictionary<string, float?>();
}
foreach (TestDoc doc in testDocs)
{
if (doc.content.Equals(searchToken))
{
for (int j = 0; j < numDims; j++)
{
if (doc.dims[j] != null)
{
float? v = expectedValues[j][doc.dims[j]];
if (v == null)
{
expectedValues[j][doc.dims[j]] = doc.value;
}
else
{
expectedValues[j][doc.dims[j]] = (float)v + doc.value;
}
}
}
}
}
IList<FacetResult> expected = new List<FacetResult>();
for (int i = 0; i < numDims; i++)
{
IList<LabelAndValue> labelValues = new List<LabelAndValue>();
float totValue = 0;
foreach (KeyValuePair<string, float?> ent in expectedValues[i])
{
labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
totValue += ent.Value.Value;
}
SortLabelValues(labelValues);
if (totValue > 0)
{
expected.Add(new FacetResult("dim" + i, new string[0], totValue, labelValues.ToArray(), labelValues.Count));
}
}
// Sort by highest value, tie break by value:
SortFacetResults(expected);
IList<FacetResult> actual = facets.GetAllDims(10);
// Messy: fixup ties
SortTies(actual);
if (VERBOSE)
{
Console.WriteLine("expected=\n" + expected.ToString());
Console.WriteLine("actual=\n" + actual.ToString());
}
AssertFloatValuesEquals(expected, actual);
}
IOUtils.Close(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
}
