public virtual void TestRandomFloats() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter(Random(), dir); int numDocs = AtLeast(1000); float[] values = new float[numDocs]; float minValue = float.PositiveInfinity; float maxValue = float.NegativeInfinity; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); float v = Random().NextFloat(); values[i] = v; doc.Add(new FloatDocValuesField("field", v)); doc.Add(new FloatField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.Reader; IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt(Random(), 1, 5); DoubleRange[] ranges = new DoubleRange[numRange]; int[] expectedCounts = new int[numRange]; float minAcceptedValue = float.PositiveInfinity; float maxAcceptedValue = float.NegativeInfinity; if (VERBOSE) { Console.WriteLine("TEST: " + numRange + " ranges"); } for (int rangeID = 0; rangeID < numRange; rangeID++) { double min; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { min = prevRange.Min; } else { min = prevRange.Max; } } else { min = Random().NextDouble(); } double max; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { max = prevRange.Min; } else { max = prevRange.Max; } } else { max = Random().NextDouble(); } if (min > max) { double x = min; min = max; max = x; } // Must truncate to float precision so that the // drill-down counts (which use NRQ.newFloatRange) // are correct: min = (float)min; max = (float)max; bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random().NextBoolean(); maxIncl = Random().NextBoolean(); } ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); if (VERBOSE) { Console.WriteLine("TEST: range " + rangeID + ": " + ranges[rangeID]); } // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (VERBOSE) { Console.WriteLine("TEST: check doc=" + i + " val=" + values[i] + " accept=" + accept); } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random().NextBoolean()) { if (Random().NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewFloatRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewFloatRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new FloatFieldSource("field"); Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine("TEST: verify range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.value); DoubleRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random().NextBoolean()) { if (Random().NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewFloatRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewFloatRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Close(w, r, dir); }
public DrillSidewaysAnonymousInnerClassHelper2(TestRangeFacetCounts testRangeFacetCounts, IndexSearcher indexSearcher, FacetsConfig facetsConfig, TaxonomyReader org, ValueSource valueSource, DoubleRange[] doubleRanges, Filter filter) : base(indexSearcher, facetsConfig, org) { this.outerInstance = outerInstance; this.vs = valueSource; this.ranges = doubleRanges; this.fastMatchFilter = filter; }
public virtual void TestCustomDoublesValueSource() { Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir); Document doc = new Document(); writer.AddDocument(doc); writer.AddDocument(doc); writer.AddDocument(doc); // Test wants 3 docs in one segment: writer.ForceMerge(1); var vs = new ValueSourceAnonymousInnerClassHelper(this, doc); FacetsConfig config = new FacetsConfig(); FacetsCollector fc = new FacetsCollector(); IndexReader r = writer.Reader; IndexSearcher s = NewSearcher(r); s.Search(new MatchAllDocsQuery(), fc); DoubleRange[] ranges = new DoubleRange[] { new DoubleRange("< 1", 0.0, true, 1.0, false), new DoubleRange("< 2", 0.0, true, 2.0, false), new DoubleRange("< 5", 0.0, true, 5.0, false), new DoubleRange("< 10", 0.0, true, 10.0, false), new DoubleRange("< 20", 0.0, true, 20.0, false), new DoubleRange("< 50", 0.0, true, 50.0, false) }; Filter fastMatchFilter; AtomicBoolean filterWasUsed = new AtomicBoolean(); if (Random().NextBoolean()) { // Sort of silly: fastMatchFilter = new CachingWrapperFilterAnonymousInnerClassHelper(this, new QueryWrapperFilter(new MatchAllDocsQuery()), filterWasUsed); } else { fastMatchFilter = null; } if (VERBOSE) { Console.WriteLine("TEST: fastMatchFilter=" + fastMatchFilter); } Facets facets = new DoubleRangeFacetCounts("field", vs, fc, fastMatchFilter, ranges); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", facets.GetTopChildren(10, "field").ToString()); Assert.True(fastMatchFilter == null || filterWasUsed.Get()); DrillDownQuery ddq = new DrillDownQuery(config); ddq.Add("field", ranges[1].GetFilter(fastMatchFilter, vs)); // Test simple drill-down: Assert.AreEqual(1, s.Search(ddq, 10).TotalHits); // Test drill-sideways after drill-down DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper2(this, s, config, (TaxonomyReader)null, vs, ranges, fastMatchFilter); DrillSidewaysResult dsr = ds.Search(ddq, 10); Assert.AreEqual(1, dsr.Hits.TotalHits); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); IOUtils.Close(r, writer, dir); }
private void Count(ValueSource valueSource, IEnumerable <MatchingDocs> matchingDocs) { DoubleRange[] ranges = (DoubleRange[])this.m_ranges; Int64Range[] longRanges = new Int64Range[ranges.Length]; for (int i = 0; i < ranges.Length; i++) { DoubleRange range = ranges[i]; longRanges[i] = new Int64Range(range.Label, NumericUtils.DoubleToSortableInt64(range.minIncl), true, NumericUtils.DoubleToSortableInt64(range.maxIncl), true); } Int64RangeCounter counter = new Int64RangeCounter(longRanges); int missingCount = 0; foreach (MatchingDocs hits in matchingDocs) { FunctionValues fv = valueSource.GetValues(Collections.EmptyMap <string, object>(), hits.Context); m_totCount += hits.TotalHits; IBits bits; if (m_fastMatchFilter != null) { DocIdSet dis = m_fastMatchFilter.GetDocIdSet(hits.Context, null); if (dis == null) { // No documents match continue; } bits = dis.Bits; if (bits == null) { throw new ArgumentException("fastMatchFilter does not implement DocIdSet.bits"); } } else { bits = null; } DocIdSetIterator docs = hits.Bits.GetIterator(); int doc; while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (bits != null && bits.Get(doc) == false) { doc++; continue; } // Skip missing docs: if (fv.Exists(doc)) { counter.Add(NumericUtils.DoubleToSortableInt64(fv.DoubleVal(doc))); } else { missingCount++; } } } missingCount += counter.FillCounts(m_counts); m_totCount -= missingCount; }
/// <summary> /// Create {@code RangeFacetCounts}, using the provided /// <seealso cref="ValueSource"/>, and using the provided Filter as /// a fastmatch: only documents passing the filter are /// checked for the matching ranges. The filter must be /// random access (implement <seealso cref="DocIdSet#bits"/>). /// </summary> public DoubleRangeFacetCounts(string field, ValueSource valueSource, FacetsCollector hits, Filter fastMatchFilter, DoubleRange[] ranges) : base(field, ranges, fastMatchFilter) { Count(valueSource, hits.GetMatchingDocs); }
public virtual void TestRandomDoubles() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); int numDocs = AtLeast(1000); double[] values = new double[numDocs]; double minValue = double.PositiveInfinity; double maxValue = double.NegativeInfinity; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); double v = Random().NextDouble(); values[i] = v; doc.Add(new DoubleDocValuesField("field", v)); doc.Add(new DoubleField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.Reader; IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt(Random(), 1, 5); DoubleRange[] ranges = new DoubleRange[numRange]; int[] expectedCounts = new int[numRange]; double minAcceptedValue = double.PositiveInfinity; double maxAcceptedValue = double.NegativeInfinity; for (int rangeID = 0; rangeID < numRange; rangeID++) { double min; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { min = prevRange.Min; } else { min = prevRange.Max; } } else { min = Random().NextDouble(); } double max; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { max = prevRange.Min; } else { max = prevRange.Max; } } else { max = Random().NextDouble(); } if (min > max) { double x = min; min = max; max = x; } bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random().NextBoolean(); maxIncl = Random().NextBoolean(); } ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random().NextBoolean()) { if (Random().NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewDoubleRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new DoubleFieldSource("field"); Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine(" range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.Label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.Value); DoubleRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random().NextBoolean()) { if (Random().NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewDoubleRange("field", range.Min, range.Max, range.MinInclusive, range.MaxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Dispose(w, r, dir); }
public virtual void TestCustomDoublesValueSource() { Directory dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); Document doc = new Document(); writer.AddDocument(doc); writer.AddDocument(doc); writer.AddDocument(doc); // Test wants 3 docs in one segment: writer.ForceMerge(1); var vs = new ValueSourceAnonymousInnerClassHelper(this, doc); FacetsConfig config = new FacetsConfig(); FacetsCollector fc = new FacetsCollector(); IndexReader r = writer.Reader; IndexSearcher s = NewSearcher(r); s.Search(new MatchAllDocsQuery(), fc); DoubleRange[] ranges = new DoubleRange[] { new DoubleRange("< 1", 0.0, true, 1.0, false), new DoubleRange("< 2", 0.0, true, 2.0, false), new DoubleRange("< 5", 0.0, true, 5.0, false), new DoubleRange("< 10", 0.0, true, 10.0, false), new DoubleRange("< 20", 0.0, true, 20.0, false), new DoubleRange("< 50", 0.0, true, 50.0, false) }; Filter fastMatchFilter; AtomicBoolean filterWasUsed = new AtomicBoolean(); if (Random().NextBoolean()) { // Sort of silly: fastMatchFilter = new CachingWrapperFilterAnonymousInnerClassHelper(this, new QueryWrapperFilter(new MatchAllDocsQuery()), filterWasUsed); } else { fastMatchFilter = null; } if (VERBOSE) { Console.WriteLine("TEST: fastMatchFilter=" + fastMatchFilter); } Facets facets = new DoubleRangeFacetCounts("field", vs, fc, fastMatchFilter, ranges); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", facets.GetTopChildren(10, "field").ToString()); Assert.True(fastMatchFilter == null || filterWasUsed.Get()); DrillDownQuery ddq = new DrillDownQuery(config); ddq.Add("field", ranges[1].GetFilter(fastMatchFilter, vs)); // Test simple drill-down: Assert.AreEqual(1, s.Search(ddq, 10).TotalHits); // Test drill-sideways after drill-down DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper2(this, s, config, (TaxonomyReader)null, vs, ranges, fastMatchFilter); DrillSidewaysResult dsr = ds.Search(ddq, 10); Assert.AreEqual(1, dsr.Hits.TotalHits); Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", dsr.Facets.GetTopChildren(10, "field").ToString()); IOUtils.Dispose(r, writer, dir); }
public FilterAnonymousClass(DoubleRange outerInstance, Filter fastMatchFilter, ValueSource valueSource) { this.outerInstance = outerInstance; this.fastMatchFilter = fastMatchFilter; this.valueSource = valueSource; }
public virtual void TestRandomFloats() { Directory dir = NewDirectory(); RandomIndexWriter w = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); int numDocs = AtLeast(1000); float[] values = new float[numDocs]; float minValue = float.PositiveInfinity; float maxValue = float.NegativeInfinity; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); float v = Random.NextSingle(); values[i] = v; doc.Add(new SingleDocValuesField("field", v)); doc.Add(new SingleField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.GetReader(); IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt32(Random, 1, 5); DoubleRange[] ranges = new DoubleRange[numRange]; int[] expectedCounts = new int[numRange]; float minAcceptedValue = float.PositiveInfinity; float maxAcceptedValue = float.NegativeInfinity; if (VERBOSE) { Console.WriteLine("TEST: " + numRange + " ranges"); } for (int rangeID = 0; rangeID < numRange; rangeID++) { double min; if (rangeID > 0 && Random.Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random.Next(rangeID)]; if (Random.NextBoolean()) { min = prevRange.Min; } else { min = prevRange.Max; } } else { min = Random.NextDouble(); } double max; if (rangeID > 0 && Random.Next(10) == 7) { // Use an existing boundary: DoubleRange prevRange = ranges[Random.Next(rangeID)]; if (Random.NextBoolean()) { max = prevRange.Min; } else { max = prevRange.Max; } } else { max = Random.NextDouble(); } if (min > max) { double x = min; min = max; max = x; } // Must truncate to float precision so that the // drill-down counts (which use NRQ.newFloatRange) // are correct: min = (float)min; max = (float)max; bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random.NextBoolean(); maxIncl = Random.NextBoolean(); } ranges[rangeID] = new DoubleRange("r" + rangeID, min, minIncl, max, maxIncl); if (VERBOSE) { Console.WriteLine("TEST: range " + rangeID + ": " + ranges[rangeID]); } // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (VERBOSE) { Console.WriteLine("TEST: check doc=" + i + " val=" + values[i] + " accept=" + accept); } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random.NextBoolean()) { if (Random.NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewSingleRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewSingleRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new SingleFieldSource("field"); Facets facets = new DoubleRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine("TEST: verify range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.Label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.Value); DoubleRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random.NextBoolean()) { if (Random.NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewSingleRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewSingleRange("field", (float)range.Min, (float)range.Max, range.MinInclusive, range.MaxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Dispose(w, r, dir); }
public FilterAnonymousInnerClassHelper(DoubleRange outerInstance, Filter fastMatchFilter, ValueSource valueSource) { this.outerInstance = outerInstance; this.fastMatchFilter = fastMatchFilter; this.valueSource = valueSource; }