/// <summary> /// Recursively assigns range outputs to each node. </summary> internal void addOutputs(int index, LongRange range) { if (start >= range.minIncl && end <= range.maxIncl) { // Our range is fully included in the incoming // range; add to our output list: if (outputs == null) { outputs = new List <int?>(); } outputs.Add(index); } else if (left != null) { Debug.Assert(right != null); // Recurse: left.addOutputs(index, range); right.addOutputs(index, range); } }
public FilterAnonymousInnerClassHelper(LongRange outerInstance, Filter fastMatchFilter, ValueSource valueSource) { this.outerInstance = outerInstance; this.fastMatchFilter = fastMatchFilter; this.valueSource = valueSource; }
public LongRangeCounter(LongRange[] ranges) { // Maps all range inclusive endpoints to int flags; 1 // = start of interval, 2 = end of interval. We need to // track the start vs end case separately because if a // given point is both, then it must be its own // elementary interval: IDictionary<long?, int?> endsMap = new Dictionary<long?, int?>(); endsMap[long.MinValue] = 1; endsMap[long.MaxValue] = 2; foreach (LongRange range in ranges) { int? cur; if (!endsMap.TryGetValue(range.minIncl,out cur)) { endsMap[range.minIncl] = 1; } else { endsMap[range.minIncl] = (int)cur | 1; } if (!endsMap.TryGetValue(range.maxIncl,out cur)) { endsMap[range.maxIncl] = 2; } else { endsMap[range.maxIncl] = (int)cur | 2; } } var endsList = new List<long?>(endsMap.Keys); endsList.Sort(); // Build elementaryIntervals (a 1D Venn diagram): IList<InclusiveRange> elementaryIntervals = new List<InclusiveRange>(); int upto0 = 1; long v = endsList[0].HasValue ? endsList[0].Value : 0; long prev; if (endsMap[v] == 3) { elementaryIntervals.Add(new InclusiveRange(v, v)); prev = v + 1; } else { prev = v; } while (upto0 < endsList.Count) { v = endsList[upto0].HasValue ? endsList[upto0].Value : 0; int flags = endsMap[v].HasValue ? endsMap[v].Value : 0; //System.out.println(" v=" + v + " flags=" + flags); if (flags == 3) { // This point is both an end and a start; we need to // separate it: if (v > prev) { elementaryIntervals.Add(new InclusiveRange(prev, v - 1)); } elementaryIntervals.Add(new InclusiveRange(v, v)); prev = v + 1; } else if (flags == 1) { // This point is only the start of an interval; // attach it to next interval: if (v > prev) { elementaryIntervals.Add(new InclusiveRange(prev, v - 1)); } prev = v; } else { Debug.Assert(flags == 2); // This point is only the end of an interval; attach // it to last interval: elementaryIntervals.Add(new InclusiveRange(prev, v)); prev = v + 1; } //System.out.println(" ints=" + elementaryIntervals); upto0++; } // Build binary tree on top of intervals: root = Split(0, elementaryIntervals.Count, elementaryIntervals); // Set outputs, so we know which range to output for // each node in the tree: for (int i = 0; i < ranges.Length; i++) { root.addOutputs(i, ranges[i]); } // Set boundaries (ends of each elementary interval): boundaries = new long[elementaryIntervals.Count]; for (int i = 0; i < boundaries.Length; i++) { boundaries[i] = elementaryIntervals[i].end; } leafCounts = new int[boundaries.Length]; //System.out.println("ranges: " + Arrays.toString(ranges)); //System.out.println("intervals: " + elementaryIntervals); //System.out.println("boundaries: " + Arrays.toString(boundaries)); //System.out.println("root:\n" + root); }
/// <summary> /// Recursively assigns range outputs to each node. </summary> internal void addOutputs(int index, LongRange range) { if (start >= range.minIncl && end <= range.maxIncl) { // Our range is fully included in the incoming // range; add to our output list: if (outputs == null) { outputs = new List<int?>(); } outputs.Add(index); } else if (left != null) { Debug.Assert(right != null); // Recurse: left.addOutputs(index, range); right.addOutputs(index, range); } }
public virtual void TestRandomLongs() { Directory dir = NewDirectory(); var w = new RandomIndexWriter(Random(), dir); int numDocs = AtLeast(1000); if (VERBOSE) { Console.WriteLine("TEST: numDocs=" + numDocs); } long[] values = new long[numDocs]; long minValue = long.MaxValue; long maxValue = long.MinValue; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); long v = Random().NextLong(); values[i] = v; doc.Add(new NumericDocValuesField("field", v)); doc.Add(new LongField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.Reader; IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt(Random(), 1, 100); LongRange[] ranges = new LongRange[numRange]; int[] expectedCounts = new int[numRange]; long minAcceptedValue = long.MaxValue; long maxAcceptedValue = long.MinValue; for (int rangeID = 0; rangeID < numRange; rangeID++) { long min; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: LongRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { min = prevRange.min; } else { min = prevRange.max; } } else { min = Random().NextLong(); } long max; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: LongRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { max = prevRange.min; } else { max = prevRange.max; } } else { max = Random().NextLong(); } if (min > max) { long x = min; min = max; max = x; } bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random().NextBoolean(); maxIncl = Random().NextBoolean(); } ranges[rangeID] = new LongRange("r" + rangeID, min, minIncl, max, maxIncl); if (VERBOSE) { Console.WriteLine(" range " + rangeID + ": " + ranges[rangeID]); } // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random().NextBoolean()) { if (Random().NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewLongRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewLongRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new LongFieldSource("field"); Facets facets = new LongRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine(" range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.value); LongRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random().NextBoolean()) { if (Random().NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Close(w, r, dir); }
private void Count(ValueSource valueSource, IEnumerable<MatchingDocs> matchingDocs) { DoubleRange[] ranges = (DoubleRange[])this.Ranges; LongRange[] longRanges = new LongRange[ranges.Length]; for (int i = 0; i < ranges.Length; i++) { DoubleRange range = ranges[i]; longRanges[i] = new LongRange(range.Label, NumericUtils.DoubleToSortableLong(range.minIncl), true, NumericUtils.DoubleToSortableLong(range.maxIncl), true); } LongRangeCounter counter = new LongRangeCounter(longRanges); int missingCount = 0; foreach (MatchingDocs hits in matchingDocs) { FunctionValues fv = valueSource.GetValues(new Dictionary<string,object>(), hits.context); TotCount += hits.totalHits; Bits bits; if (FastMatchFilter != null) { DocIdSet dis = FastMatchFilter.GetDocIdSet(hits.context, null); if (dis == null) { // No documents match continue; } bits = dis.GetBits(); if (bits == null) { throw new System.ArgumentException("fastMatchFilter does not implement DocIdSet.bits"); } } else { bits = null; } DocIdSetIterator docs = hits.bits.GetIterator(); int doc; while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (bits != null && bits.Get(doc) == false) { doc++; continue; } // Skip missing docs: if (fv.Exists(doc)) { counter.add(NumericUtils.DoubleToSortableLong(fv.DoubleVal(doc))); } else { missingCount++; } } } missingCount += counter.fillCounts(Counts); TotCount -= missingCount; }
public virtual void TestRandomLongs() { Directory dir = NewDirectory(); var w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); int numDocs = AtLeast(1000); if (VERBOSE) { Console.WriteLine("TEST: numDocs=" + numDocs); } long[] values = new long[numDocs]; long minValue = long.MaxValue; long maxValue = long.MinValue; for (int i = 0; i < numDocs; i++) { Document doc = new Document(); long v = Random().NextLong(); values[i] = v; doc.Add(new NumericDocValuesField("field", v)); doc.Add(new LongField("field", v, Field.Store.NO)); w.AddDocument(doc); minValue = Math.Min(minValue, v); maxValue = Math.Max(maxValue, v); } IndexReader r = w.Reader; IndexSearcher s = NewSearcher(r); FacetsConfig config = new FacetsConfig(); int numIters = AtLeast(10); for (int iter = 0; iter < numIters; iter++) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } int numRange = TestUtil.NextInt(Random(), 1, 100); LongRange[] ranges = new LongRange[numRange]; int[] expectedCounts = new int[numRange]; long minAcceptedValue = long.MaxValue; long maxAcceptedValue = long.MinValue; for (int rangeID = 0; rangeID < numRange; rangeID++) { long min; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: LongRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { min = prevRange.min; } else { min = prevRange.max; } } else { min = Random().NextLong(); } long max; if (rangeID > 0 && Random().Next(10) == 7) { // Use an existing boundary: LongRange prevRange = ranges[Random().Next(rangeID)]; if (Random().NextBoolean()) { max = prevRange.min; } else { max = prevRange.max; } } else { max = Random().NextLong(); } if (min > max) { long x = min; min = max; max = x; } bool minIncl; bool maxIncl; if (min == max) { minIncl = true; maxIncl = true; } else { minIncl = Random().NextBoolean(); maxIncl = Random().NextBoolean(); } ranges[rangeID] = new LongRange("r" + rangeID, min, minIncl, max, maxIncl); if (VERBOSE) { Console.WriteLine(" range " + rangeID + ": " + ranges[rangeID]); } // Do "slow but hopefully correct" computation of // expected count: for (int i = 0; i < numDocs; i++) { bool accept = true; if (minIncl) { accept &= values[i] >= min; } else { accept &= values[i] > min; } if (maxIncl) { accept &= values[i] <= max; } else { accept &= values[i] < max; } if (accept) { expectedCounts[rangeID]++; minAcceptedValue = Math.Min(minAcceptedValue, values[i]); maxAcceptedValue = Math.Max(maxAcceptedValue, values[i]); } } } FacetsCollector sfc = new FacetsCollector(); s.Search(new MatchAllDocsQuery(), sfc); Filter fastMatchFilter; if (Random().NextBoolean()) { if (Random().NextBoolean()) { fastMatchFilter = NumericRangeFilter.NewLongRange("field", minValue, maxValue, true, true); } else { fastMatchFilter = NumericRangeFilter.NewLongRange("field", minAcceptedValue, maxAcceptedValue, true, true); } } else { fastMatchFilter = null; } ValueSource vs = new LongFieldSource("field"); Facets facets = new LongRangeFacetCounts("field", vs, sfc, fastMatchFilter, ranges); FacetResult result = facets.GetTopChildren(10, "field"); Assert.AreEqual(numRange, result.LabelValues.Length); for (int rangeID = 0; rangeID < numRange; rangeID++) { if (VERBOSE) { Console.WriteLine(" range " + rangeID + " expectedCount=" + expectedCounts[rangeID]); } LabelAndValue subNode = result.LabelValues[rangeID]; Assert.AreEqual("r" + rangeID, subNode.label); Assert.AreEqual(expectedCounts[rangeID], (int)subNode.value); LongRange range = ranges[rangeID]; // Test drill-down: DrillDownQuery ddq = new DrillDownQuery(config); if (Random().NextBoolean()) { if (Random().NextBoolean()) { ddq.Add("field", NumericRangeFilter.NewLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive)); } else { ddq.Add("field", NumericRangeQuery.NewLongRange("field", range.min, range.max, range.minInclusive, range.maxInclusive)); } } else { ddq.Add("field", range.GetFilter(fastMatchFilter, vs)); } Assert.AreEqual(expectedCounts[rangeID], s.Search(ddq, 10).TotalHits); } } IOUtils.Close(w, r, dir); }
private void Count(ValueSource valueSource, IEnumerable <MatchingDocs> matchingDocs) { DoubleRange[] ranges = (DoubleRange[])this.Ranges; LongRange[] longRanges = new LongRange[ranges.Length]; for (int i = 0; i < ranges.Length; i++) { DoubleRange range = ranges[i]; longRanges[i] = new LongRange(range.Label, NumericUtils.DoubleToSortableLong(range.minIncl), true, NumericUtils.DoubleToSortableLong(range.maxIncl), true); } LongRangeCounter counter = new LongRangeCounter(longRanges); int missingCount = 0; foreach (MatchingDocs hits in matchingDocs) { FunctionValues fv = valueSource.GetValues(new Dictionary <string, object>(), hits.context); TotCount += hits.totalHits; Bits bits; if (FastMatchFilter != null) { DocIdSet dis = FastMatchFilter.GetDocIdSet(hits.context, null); if (dis == null) { // No documents match continue; } bits = dis.GetBits(); if (bits == null) { throw new System.ArgumentException("fastMatchFilter does not implement DocIdSet.bits"); } } else { bits = null; } DocIdSetIterator docs = hits.bits.GetIterator(); int doc; while ((doc = docs.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (bits != null && bits.Get(doc) == false) { doc++; continue; } // Skip missing docs: if (fv.Exists(doc)) { counter.add(NumericUtils.DoubleToSortableLong(fv.DoubleVal(doc))); } else { missingCount++; } } } missingCount += counter.fillCounts(Counts); TotCount -= missingCount; }