private void ApplyAggregation(Facet facet, FacetValue value, List<int> docsInQuery, IndexReader indexReader, int docBase)
{
var sortOptionsForFacet = GetSortOptionsForFacet(facet.AggregationField);
switch (sortOptionsForFacet)
{
case SortOptions.String:
case SortOptions.StringVal:
case SortOptions.Byte:
case SortOptions.Short:
case SortOptions.Custom:
case SortOptions.None:
throw new InvalidOperationException(string.Format("Cannot perform numeric aggregation on index field '{0}'. You must set the Sort mode of the field to Int, Float, Long or Double.", TryTrimRangeSuffix(facet.AggregationField)));
case SortOptions.Int:
int[] ints = FieldCache_Fields.DEFAULT.GetInts(indexReader, facet.AggregationField);
foreach (var doc in docsInQuery)
{
var currentVal = ints[doc - docBase];
if (facet.Aggregation.HasFlag(FacetAggregation.Max))
{
value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Min))
{
value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Sum))
{
value.Sum = currentVal + (value.Sum ?? 0d);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Average))
{
value.Average = currentVal + (value.Average ?? 0d);
}
}
break;
case SortOptions.Float:
var floats = FieldCache_Fields.DEFAULT.GetFloats(indexReader, facet.AggregationField);
foreach (var doc in docsInQuery)
{
var currentVal = floats[doc - docBase];
if (facet.Aggregation.HasFlag(FacetAggregation.Max))
{
value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Min))
{
value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Sum))
{
value.Sum = currentVal + (value.Sum ?? 0d);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Average))
{
value.Average = currentVal + (value.Average ?? 0d);
}
}
break;
case SortOptions.Long:
var longs = FieldCache_Fields.DEFAULT.GetLongs(indexReader, facet.AggregationField);
foreach (var doc in docsInQuery)
{
var currentVal = longs[doc - docBase];
if (facet.Aggregation.HasFlag(FacetAggregation.Max))
{
value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Min))
{
value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Sum))
{
value.Sum = currentVal + (value.Sum ?? 0d);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Average))
{
value.Average = currentVal + (value.Average ?? 0d);
}
}
break;
case SortOptions.Double:
var doubles = FieldCache_Fields.DEFAULT.GetDoubles(indexReader, facet.AggregationField);
foreach (var doc in docsInQuery)
{
var currentVal = doubles[doc - docBase];
if (facet.Aggregation.HasFlag(FacetAggregation.Max))
{
value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Min))
{
value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Sum))
{
value.Sum = currentVal + (value.Sum ?? 0d);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Average))
{
value.Average = currentVal + (value.Average ?? 0d);
}
}
break;
default:
throw new ArgumentOutOfRangeException("Cannot understand " + sortOptionsForFacet);
}
}
private void ApplyFacetValueHit(FacetValue facetValue, Facet value, int docId, ParsedRange parsedRange)
{
facetValue.Hits++;
if (value.Aggregation == FacetAggregation.Count || value.Aggregation == FacetAggregation.None)
{
return;
}
FacetValueState set;
if (matches.TryGetValue(facetValue, out set) == false)
{
matches[facetValue] = set = new FacetValueState
{
Docs = new HashSet<int>(),
Facet = value,
Range = parsedRange
};
}
set.Docs.Add(docId);
}
public void Execute()
{
ValidateFacets();
var facetsByName = new Dictionary<string, Dictionary<string, FacetValue>>();
bool isDistinct = IndexQuery.IsDistinct;
if (isDistinct)
{
_fieldsCrc = IndexQuery.FieldsToFetch.Aggregate<string, uint>(0, (current, field) => Crc.Value(field, current));
}
_currentState = Database.IndexStorage.GetCurrentStateHolder(Index);
using (_currentState)
{
var currentIndexSearcher = _currentState.IndexSearcher;
var baseQuery = Database.IndexStorage.GetDocumentQuery(Index, IndexQuery, Database.IndexQueryTriggers);
var returnedReaders = GetQueryMatchingDocuments(currentIndexSearcher, baseQuery);
foreach (var facet in Facets.Values)
{
if(facet.Mode != FacetMode.Default)
continue;
Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>> distinctItems = null;
HashSet<IndexSearcherHolder.StringCollectionValue> alreadySeen = null;
if(isDistinct)
distinctItems = new Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>>();
foreach (var readerFacetInfo in returnedReaders)
{
var termsForField = IndexedTerms.GetTermsAndDocumenstFor(readerFacetInfo.Reader, readerFacetInfo.DocBase, facet.Name);
Dictionary<string, FacetValue> facetValues;
if (facetsByName.TryGetValue(facet.DisplayName, out facetValues) == false)
{
facetsByName[facet.DisplayName] = facetValues = new Dictionary<string, FacetValue>();
}
foreach (var kvp in termsForField)
{
if (isDistinct)
{
if (distinctItems.TryGetValue(kvp.Key, out alreadySeen) == false)
{
alreadySeen = new HashSet<IndexSearcherHolder.StringCollectionValue>();
distinctItems[kvp.Key] = alreadySeen;
}
}
var needToApplyAggregation = (facet.Aggregation == FacetAggregation.None || facet.Aggregation == FacetAggregation.Count) == false;
var intersectedDocuments = GetIntersectedDocuments(kvp.Value, readerFacetInfo.Results, alreadySeen, needToApplyAggregation);
var intersectCount = intersectedDocuments.Count;
if (intersectCount == 0)
continue;
FacetValue facetValue;
if (facetValues.TryGetValue(kvp.Key, out facetValue) == false)
{
facetValue = new FacetValue
{
Range = GetRangeName(facet.Name, kvp.Key)
};
facetValues.Add(kvp.Key, facetValue);
}
facetValue.Hits += intersectCount;
facetValue.Count = facetValue.Hits;
if (needToApplyAggregation)
{
ApplyAggregation(facet, facetValue, intersectedDocuments.Documents, readerFacetInfo.Reader, readerFacetInfo.DocBase);
}
}
}
}
foreach (var range in Ranges)
{
var facet = Facets[range.Key];
var needToApplyAggregation = (facet.Aggregation == FacetAggregation.None || facet.Aggregation == FacetAggregation.Count) == false;
Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>> distinctItems = null;
HashSet<IndexSearcherHolder.StringCollectionValue> alreadySeen = null;
if (isDistinct)
distinctItems = new Dictionary<string, HashSet<IndexSearcherHolder.StringCollectionValue>>();
foreach (var readerFacetInfo in returnedReaders)
{
var termsForField = IndexedTerms.GetTermsAndDocumenstFor(readerFacetInfo.Reader, readerFacetInfo.DocBase, facet.Name);
if (isDistinct)
{
if (distinctItems.TryGetValue(range.Key, out alreadySeen) == false)
{
alreadySeen = new HashSet<IndexSearcherHolder.StringCollectionValue>();
distinctItems[range.Key] = alreadySeen;
}
}
var facetResult = Results.Results[range.Key];
var ranges = range.Value;
foreach (var kvp in termsForField)
{
for (int i = 0; i < ranges.Count; i++)
{
var parsedRange = ranges[i];
if (parsedRange.IsMatch(kvp.Key))
{
var facetValue = facetResult.Values[i];
var intersectedDocuments = GetIntersectedDocuments(kvp.Value, readerFacetInfo.Results, alreadySeen, needToApplyAggregation);
var intersectCount = intersectedDocuments.Count;
if (intersectCount == 0)
continue;
facetValue.Hits += intersectCount;
facetValue.Count = facetValue.Hits;
if (needToApplyAggregation)
{
ApplyAggregation(facet, facetValue, intersectedDocuments.Documents, readerFacetInfo.Reader, readerFacetInfo.DocBase);
}
}
}
}
}
}
UpdateFacetResults(facetsByName);
CompleteFacetCalculationsStage();
}
}
private void ApplyAggregation(Facet facet, FacetValue value, double currentVal)
{
if (facet.Aggregation.HasFlag(FacetAggregation.Max))
{
value.Max = Math.Max(value.Max ?? Double.MinValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Min))
{
value.Min = Math.Min(value.Min ?? Double.MaxValue, currentVal);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Sum))
{
value.Sum = currentVal + (value.Sum ?? 0d);
}
if (facet.Aggregation.HasFlag(FacetAggregation.Average))
{
value.Average = currentVal + (value.Average ?? 0d);
}
}
public void Execute()
{
ValidateFacets();
//We only want to run the base query once, so we capture all of the facet-ing terms then run the query
// once through the collector and pull out all of the terms in one shot
var allCollector = new GatherAllCollector();
var facetsByName = new Dictionary<string, Dictionary<string, FacetValue>>();
using (var currentState = Database.IndexStorage.GetCurrentStateHolder(Index))
{
var currentIndexSearcher = currentState.IndexSearcher;
var baseQuery = Database.IndexStorage.GetDocumentQuery(Index, IndexQuery, Database.IndexQueryTriggers);
currentIndexSearcher.Search(baseQuery, allCollector);
var fields = Facets.Values.Select(x => x.Name)
.Concat(Ranges.Select(x => x.Key));
var fieldsToRead = new HashSet<string>(fields);
IndexedTerms.ReadEntriesForFields(currentState,
fieldsToRead,
allCollector.Documents,
(term, doc) =>
{
var facets = Facets.Values.Where(facet => facet.Name == term.Field);
foreach (var facet in facets)
{
switch (facet.Mode)
{
case FacetMode.Default:
var facetValues = facetsByName.GetOrAdd(facet.DisplayName);
FacetValue existing;
if (facetValues.TryGetValue(term.Text, out existing) == false)
{
existing = new FacetValue
{
Range = GetRangeName(term)
};
facetValues[term.Text] = existing;
}
ApplyFacetValueHit(existing, facet, doc, null);
break;
case FacetMode.Ranges:
List<ParsedRange> list;
if (Ranges.TryGetValue(term.Field, out list))
{
for (int i = 0; i < list.Count; i++)
{
var parsedRange = list[i];
if (parsedRange.IsMatch(term.Text))
{
var facetValue = Results.Results[term.Field].Values[i];
ApplyFacetValueHit(facetValue, facet, doc, parsedRange);
}
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
});
UpdateFacetResults(facetsByName);
CompleteFacetCalculationsStage1(currentState);
CompleteFacetCalculationsStage2();
}
}
private void CheckFacetCount(int expectedCount, FacetValue facets)
{
if (expectedCount > 0)
{
Assert.NotNull(facets);
Assert.Equal(expectedCount, facets.Count);
}
}
private void HandleFacets(string field, string value, Dictionary<string, Dictionary<string, FacetValue>> facetsByName, int doc)
{
var facets = Facets.Values.Where(facet => facet.Name == field);
foreach (var facet in facets)
{
switch (facet.Mode)
{
case FacetMode.Default:
var facetValues = facetsByName.GetOrAdd(facet.DisplayName);
FacetValue existing;
if (facetValues.TryGetValue(value, out existing) == false)
{
existing = new FacetValue
{
Range = GetRangeName(field, value)
};
facetValues[value] = existing;
}
ApplyFacetValueHit(existing, facet, doc, null);
break;
case FacetMode.Ranges:
List<ParsedRange> list;
if (Ranges.TryGetValue(field, out list))
{
for (int i = 0; i < list.Count; i++)
{
var parsedRange = list[i];
if (parsedRange.IsMatch(value))
{
var facetValue = Results.Results[field].Values[i];
ApplyFacetValueHit(facetValue, facet, doc, parsedRange);
}
}
}
break;
default:
throw new ArgumentOutOfRangeException();
}
}
}
private void ApplyFacetValueHit(FacetValue facetValue, Facet value, int docId, ParsedRange parsedRange, IndexReader indexReader)
{
facetValue.Hits++;
if (
IndexQuery.IsDistinct == false &&
(value.Aggregation == FacetAggregation.Count || value.Aggregation == FacetAggregation.None)
)
{
return;
}
FacetValueState set;
if (matches.TryGetValue(facetValue, out set) == false)
{
matches[facetValue] = set = new FacetValueState
{
Docs = new HashSet<int>(),
Facet = value,
Range = parsedRange
};
}
if (IndexQuery.IsDistinct)
{
if(IndexQuery.FieldsToFetch.Length == 0)
throw new InvalidOperationException("Cannot process distinct facet query without specifying which fields to distinct upon.");
if (set.AlreadySeen == null)
set.AlreadySeen = new HashSet<StringCollectionValue>();
var document = indexReader.Document(docId);
var fields = new List<string>();
foreach (var fieldName in IndexQuery.FieldsToFetch)
{
foreach (var field in document.GetFields(fieldName))
{
if (field.StringValue == null)
continue;
fields.Add(field.StringValue);
}
}
if (fields.Count == 0)
throw new InvalidOperationException("Cannot apply distinct facet on [" + string.Join(", ", IndexQuery.FieldsToFetch) +
"], did you forget to store them in the index? ");
if (set.AlreadySeen.Add(new StringCollectionValue(fields)) == false)
{
facetValue.Hits--;// already seen, cancel this
return;
}
}
set.Docs.Add(docId);
}