private void ApplyAggregation(Facet facet, FacetValue value, List<int> docsInQuery, IndexReader indexReader, int docBase) { var sortOptionsForFacet = GetSortOptionsForFacet(facet.AggregationField); switch (sortOptionsForFacet) { case SortOptions.String: case SortOptions.StringVal: case SortOptions.Byte: case SortOptions.Short: case SortOptions.Custom: case SortOptions.None: throw new InvalidOperationException(string.Format("Cannot perform numeric aggregation on index field '{0}'. You must set the Sort mode of the field to Int, Float, Long or Double.", TryTrimRangeSuffix(facet.AggregationField))); case SortOptions.Int: int[] ints = FieldCache_Fields.DEFAULT.GetInts(indexReader, facet.AggregationField); foreach (var doc in docsInQuery) { var currentVal = ints[doc - docBase]; if (facet.Aggregation.HasFlag(FacetAggregation.Max)) { value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Min)) { value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Sum)) { value.Sum = currentVal + (value.Sum ?? 0d); } if (facet.Aggregation.HasFlag(FacetAggregation.Average)) { value.Average = currentVal + (value.Average ?? 0d); } } break; case SortOptions.Float: var floats = FieldCache_Fields.DEFAULT.GetFloats(indexReader, facet.AggregationField); foreach (var doc in docsInQuery) { var currentVal = floats[doc - docBase]; if (facet.Aggregation.HasFlag(FacetAggregation.Max)) { value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Min)) { value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Sum)) { value.Sum = currentVal + (value.Sum ?? 0d); } if (facet.Aggregation.HasFlag(FacetAggregation.Average)) { value.Average = currentVal + (value.Average ?? 0d); } } break; case SortOptions.Long: var longs = FieldCache_Fields.DEFAULT.GetLongs(indexReader, facet.AggregationField); foreach (var doc in docsInQuery) { var currentVal = longs[doc - docBase]; if (facet.Aggregation.HasFlag(FacetAggregation.Max)) { value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Min)) { value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Sum)) { value.Sum = currentVal + (value.Sum ?? 0d); } if (facet.Aggregation.HasFlag(FacetAggregation.Average)) { value.Average = currentVal + (value.Average ?? 0d); } } break; case SortOptions.Double: var doubles = FieldCache_Fields.DEFAULT.GetDoubles(indexReader, facet.AggregationField); foreach (var doc in docsInQuery) { var currentVal = doubles[doc - docBase]; if (facet.Aggregation.HasFlag(FacetAggregation.Max)) { value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Min)) { value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Sum)) { value.Sum = currentVal + (value.Sum ?? 0d); } if (facet.Aggregation.HasFlag(FacetAggregation.Average)) { value.Average = currentVal + (value.Average ?? 0d); } } break; default: throw new ArgumentOutOfRangeException("Cannot understand " + sortOptionsForFacet); } }
private void ApplyFacetValueHit(FacetValue facetValue, Facet value, int docId, ParsedRange parsedRange) { facetValue.Hits++; if (value.Aggregation == FacetAggregation.Count || value.Aggregation == FacetAggregation.None) { return; } FacetValueState set; if (matches.TryGetValue(facetValue, out set) == false) { matches[facetValue] = set = new FacetValueState { Docs = new HashSet<int>(), Facet = value, Range = parsedRange }; } set.Docs.Add(docId); }
public void Execute() { ValidateFacets(); var facetsByName = new Dictionary<string, Dictionary<string, FacetValue>>(); bool isDistinct = IndexQuery.IsDistinct; if (isDistinct) { _fieldsCrc = IndexQuery.FieldsToFetch.Aggregate<string, uint>(0, (current, field) => Crc.Value(field, current)); } _currentState = Database.IndexStorage.GetCurrentStateHolder(Index); using (_currentState) { var currentIndexSearcher = _currentState.IndexSearcher; var baseQuery = Database.IndexStorage.GetDocumentQuery(Index, IndexQuery, Database.IndexQueryTriggers); var returnedReaders = GetQueryMatchingDocuments(currentIndexSearcher, baseQuery); foreach (var facet in Facets.Values) { if(facet.Mode != FacetMode.Default) continue; Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>> distinctItems = null; HashSet<IndexSearcherHolder.StringCollectionValue> alreadySeen = null; if(isDistinct) distinctItems = new Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>>(); foreach (var readerFacetInfo in returnedReaders) { var termsForField = IndexedTerms.GetTermsAndDocumenstFor(readerFacetInfo.Reader, readerFacetInfo.DocBase, facet.Name); Dictionary<string, FacetValue> facetValues; if (facetsByName.TryGetValue(facet.DisplayName, out facetValues) == false) { facetsByName[facet.DisplayName] = facetValues = new Dictionary<string, FacetValue>(); } foreach (var kvp in termsForField) { if (isDistinct) { if (distinctItems.TryGetValue(kvp.Key, out alreadySeen) == false) { alreadySeen = new HashSet<IndexSearcherHolder.StringCollectionValue>(); distinctItems[kvp.Key] = alreadySeen; } } var needToApplyAggregation = (facet.Aggregation == FacetAggregation.None || facet.Aggregation == FacetAggregation.Count) == false; var intersectedDocuments = GetIntersectedDocuments(kvp.Value, readerFacetInfo.Results, alreadySeen, needToApplyAggregation); var intersectCount = intersectedDocuments.Count; if (intersectCount == 0) continue; FacetValue facetValue; if (facetValues.TryGetValue(kvp.Key, out facetValue) == false) { facetValue = new FacetValue { Range = GetRangeName(facet.Name, kvp.Key) }; facetValues.Add(kvp.Key, facetValue); } facetValue.Hits += intersectCount; facetValue.Count = facetValue.Hits; if (needToApplyAggregation) { ApplyAggregation(facet, facetValue, intersectedDocuments.Documents, readerFacetInfo.Reader, readerFacetInfo.DocBase); } } } } foreach (var range in Ranges) { var facet = Facets[range.Key]; var needToApplyAggregation = (facet.Aggregation == FacetAggregation.None || facet.Aggregation == FacetAggregation.Count) == false; Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>> distinctItems = null; HashSet<IndexSearcherHolder.StringCollectionValue> alreadySeen = null; if (isDistinct) distinctItems = new Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>>(); foreach (var readerFacetInfo in returnedReaders) { var termsForField = IndexedTerms.GetTermsAndDocumenstFor(readerFacetInfo.Reader, readerFacetInfo.DocBase, facet.Name); if (isDistinct) { if (distinctItems.TryGetValue(range.Key, out alreadySeen) == false) { alreadySeen = new HashSet<IndexSearcherHolder.StringCollectionValue>(); distinctItems[range.Key] = alreadySeen; } } var facetResult = Results.Results[range.Key]; var ranges = range.Value; foreach (var kvp in termsForField) { for (int i = 0; i < ranges.Count; i++) { var parsedRange = ranges[i]; if (parsedRange.IsMatch(kvp.Key)) { var facetValue = facetResult.Values[i]; var intersectedDocuments = GetIntersectedDocuments(kvp.Value, readerFacetInfo.Results, alreadySeen, needToApplyAggregation); var intersectCount = intersectedDocuments.Count; if (intersectCount == 0) continue; facetValue.Hits += intersectCount; facetValue.Count = facetValue.Hits; if (needToApplyAggregation) { ApplyAggregation(facet, facetValue, intersectedDocuments.Documents, readerFacetInfo.Reader, readerFacetInfo.DocBase); } } } } } } UpdateFacetResults(facetsByName); CompleteFacetCalculationsStage(); } }
private void ApplyAggregation(Facet facet, FacetValue value, double currentVal) { if (facet.Aggregation.HasFlag(FacetAggregation.Max)) { value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Min)) { value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal); } if (facet.Aggregation.HasFlag(FacetAggregation.Sum)) { value.Sum = currentVal + (value.Sum ?? 0d); } if (facet.Aggregation.HasFlag(FacetAggregation.Average)) { value.Average = currentVal + (value.Average ?? 0d); } }
public void Execute() { ValidateFacets(); //We only want to run the base query once, so we capture all of the facet-ing terms then run the query // once through the collector and pull out all of the terms in one shot var allCollector = new GatherAllCollector(); var facetsByName = new Dictionary<string, Dictionary<string, FacetValue>>(); using (var currentState = Database.IndexStorage.GetCurrentStateHolder(Index)) { var currentIndexSearcher = currentState.IndexSearcher; var baseQuery = Database.IndexStorage.GetDocumentQuery(Index, IndexQuery, Database.IndexQueryTriggers); currentIndexSearcher.Search(baseQuery, allCollector); var fields = Facets.Values.Select(x => x.Name) .Concat(Ranges.Select(x => x.Key)); var fieldsToRead = new HashSet<string>(fields); IndexedTerms.ReadEntriesForFields(currentState, fieldsToRead, allCollector.Documents, (term, doc) => { var facets = Facets.Values.Where(facet => facet.Name == term.Field); foreach (var facet in facets) { switch (facet.Mode) { case FacetMode.Default: var facetValues = facetsByName.GetOrAdd(facet.DisplayName); FacetValue existing; if (facetValues.TryGetValue(term.Text, out existing) == false) { existing = new FacetValue { Range = GetRangeName(term) }; facetValues[term.Text] = existing; } ApplyFacetValueHit(existing, facet, doc, null); break; case FacetMode.Ranges: List<ParsedRange> list; if (Ranges.TryGetValue(term.Field, out list)) { for (int i = 0; i < list.Count; i++) { var parsedRange = list[i]; if (parsedRange.IsMatch(term.Text)) { var facetValue = Results.Results[term.Field].Values[i]; ApplyFacetValueHit(facetValue, facet, doc, parsedRange); } } } break; default: throw new ArgumentOutOfRangeException(); } } }); UpdateFacetResults(facetsByName); CompleteFacetCalculationsStage1(currentState); CompleteFacetCalculationsStage2(); } }
private void CheckFacetCount(int expectedCount, FacetValue facets) { if (expectedCount > 0) { Assert.NotNull(facets); Assert.Equal(expectedCount, facets.Count); } }
private void HandleFacets(string field, string value, Dictionary<string, Dictionary<string, FacetValue>> facetsByName, int doc) { var facets = Facets.Values.Where(facet => facet.Name == field); foreach (var facet in facets) { switch (facet.Mode) { case FacetMode.Default: var facetValues = facetsByName.GetOrAdd(facet.DisplayName); FacetValue existing; if (facetValues.TryGetValue(value, out existing) == false) { existing = new FacetValue { Range = GetRangeName(field, value) }; facetValues[value] = existing; } ApplyFacetValueHit(existing, facet, doc, null); break; case FacetMode.Ranges: List<ParsedRange> list; if (Ranges.TryGetValue(field, out list)) { for (int i = 0; i < list.Count; i++) { var parsedRange = list[i]; if (parsedRange.IsMatch(value)) { var facetValue = Results.Results[field].Values[i]; ApplyFacetValueHit(facetValue, facet, doc, parsedRange); } } } break; default: throw new ArgumentOutOfRangeException(); } } }
private void ApplyFacetValueHit(FacetValue facetValue, Facet value, int docId, ParsedRange parsedRange, IndexReader indexReader) { facetValue.Hits++; if ( IndexQuery.IsDistinct == false && (value.Aggregation == FacetAggregation.Count || value.Aggregation == FacetAggregation.None) ) { return; } FacetValueState set; if (matches.TryGetValue(facetValue, out set) == false) { matches[facetValue] = set = new FacetValueState { Docs = new HashSet<int>(), Facet = value, Range = parsedRange }; } if (IndexQuery.IsDistinct) { if(IndexQuery.FieldsToFetch.Length == 0) throw new InvalidOperationException("Cannot process distinct facet query without specifying which fields to distinct upon."); if (set.AlreadySeen == null) set.AlreadySeen = new HashSet<StringCollectionValue>(); var document = indexReader.Document(docId); var fields = new List<string>(); foreach (var fieldName in IndexQuery.FieldsToFetch) { foreach (var field in document.GetFields(fieldName)) { if (field.StringValue == null) continue; fields.Add(field.StringValue); } } if (fields.Count == 0) throw new InvalidOperationException("Cannot apply distinct facet on [" + string.Join(", ", IndexQuery.FieldsToFetch) + "], did you forget to store them in the index? "); if (set.AlreadySeen.Add(new StringCollectionValue(fields)) == false) { facetValue.Hits--;// already seen, cancel this return; } } set.Docs.Add(docId); }