public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(IndexQueryServerSide query)
{
var map = DynamicQueryMapping.Create(query);
var explanations = new List <DynamicQueryToIndexMatcher.Explanation>();
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore);
dynamicQueryToIndex.Match(map, explanations);
return(explanations);
}
public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(IndexQueryServerSide query, DocumentsOperationContext docsContext, out string indexName)
{
var map = DynamicQueryMapping.Create(query);
var explanations = new List <DynamicQueryToIndexMatcher.Explanation>();
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore);
var match = dynamicQueryToIndex.Match(map, docsContext, explanations);
indexName = match.IndexName;
return(explanations);
}
public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(string dynamicIndexName, IndexQueryServerSide query)
{
var collection = dynamicIndexName.Substring(DynamicIndexPrefix.Length);
var map = DynamicQueryMapping.Create(collection, query);
var explanations = new List <DynamicQueryToIndexMatcher.Explanation>();
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore);
dynamicQueryToIndex.Match(map, explanations);
return(explanations);
}
public static DynamicQueryMapping Create(Index index)
{
if (index.Type.IsAuto() == false)
{
throw new InvalidOperationException();
}
var mapping = new DynamicQueryMapping
{
ForCollection = index.Collections.First(),
MapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal)
};
var definition = (AutoIndexDefinitionBase)index.Definition;
foreach (var field in definition.MapFields)
{
var autoField = (AutoIndexField)field.Value;
mapping.MapFields[field.Key] = DynamicQueryMappingItem.Create(
new QueryFieldName(autoField.Name, autoField.HasQuotedName),
autoField.Aggregation,
isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search),
isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact),
hasHighlighting: autoField.Indexing.HasFlag(AutoFieldIndexing.Highlighting),
hasSuggestions: autoField.HasSuggestions,
spatial: autoField.Spatial);
}
if (index.Type.IsMapReduce())
{
mapping.IsGroupBy = true;
var mapReduceDefinition = (AutoMapReduceIndexDefinition)definition;
mapping.GroupByFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal);
foreach (var field in mapReduceDefinition.GroupByFields)
{
var autoField = field.Value;
mapping.GroupByFields[field.Key] = DynamicQueryMappingItem.CreateGroupBy(
new QueryFieldName(autoField.Name, autoField.HasQuotedName),
autoField.GroupByArrayBehavior,
isSpecifiedInWhere: true,
isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search),
isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact));
}
}
return(mapping);
}
public async Task <(Index Index, bool HasCreatedAutoIndex)> CreateAutoIndexIfNeeded(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout,
DocumentsOperationContext docsContext, CancellationToken token)
{
var map = DynamicQueryMapping.Create(query);
bool hasCreatedAutoIndex = false;
if (TryMatchExistingIndexToQuery(map, docsContext, out var index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistException("Could not find index for a given query.");
}
var definition = map.CreateAutoIndexDefinition();
index = await _indexStore.CreateIndex(definition, RaftIdGenerator.NewId());
hasCreatedAutoIndex = true;
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout ?? TimeSpan.FromSeconds(15);
}
var t = CleanupSupersededAutoIndexes(index, map, RaftIdGenerator.NewId(), token)
.ContinueWith(task =>
{
if (task.Exception != null)
{
if (token.IsCancellationRequested)
{
return;
}
if (_indexStore.Logger.IsInfoEnabled)
{
_indexStore.Logger.Info("Failed to delete superseded indexes for index " + index.Name);
}
}
}, token);
if (query.WaitForNonStaleResults &&
Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan ==
TimeSpan.Zero)
{
await t; // this is used in testing, mainly
}
}
return(index, hasCreatedAutoIndex);
}
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null)
{
var bestComplete = DynamicQueryMatchResult.Failure;
foreach (var index in _indexStore.GetIndexesForCollection(query.ForCollection))
{
if (query.IsGroupBy)
{
if (index.Type != IndexType.AutoMapReduce)
{
continue;
}
}
else if (index.Type != IndexType.AutoMap)
{
continue;
}
var auto = (AutoIndexDefinitionBase)index.Definition;
var result = ConsiderUsageOfIndex(query, auto, explanations);
if (result.MatchType < bestComplete.MatchType)
{
explanations?.Add(new Explanation(result.IndexName, "A better match was available"));
continue;
}
const string notWidestMsg = "Wasn't the widest / most up to date index matching this query";
if (result.LastMappedEtag <= bestComplete.LastMappedEtag &&
result.NumberOfMappedFields <= bestComplete.NumberOfMappedFields)
{
explanations?.Add(new Explanation(bestComplete.IndexName, notWidestMsg));
continue;
}
if (explanations != null &&
bestComplete.MatchType != DynamicQueryMatchType.Failure)
{
explanations.Add(new Explanation(bestComplete.IndexName, notWidestMsg));
}
bestComplete = result;
}
return(bestComplete);
}
private bool AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations)
{
var indexName = definition.Name;
foreach (var mapField in query.MapFields)
{
if (definition.ContainsField(mapField.Name) == false)
{
Debug.Assert(currentBestState == DynamicQueryMatchType.Partial);
continue;
}
var field = definition.GetField(mapField.Name);
if (field.MapReduceOperation != mapField.MapReduceOperation)
{
explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.MapReduceOperation} operation defined, while query required {mapField.MapReduceOperation}"));
return(false);
}
}
if (query.GroupByFields.All(definition.ContainsGroupByField) == false)
{
if (explanations != null)
{
var missingFields = query.GroupByFields.Where(x => definition.ContainsGroupByField(x) == false);
explanations?.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}"));
}
return(false);
}
if (query.GroupByFields.Length != definition.GroupByFields.Count)
{
if (explanations != null)
{
var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Contains(x.Key) == false);
explanations?.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}"));
}
return(false);
}
return(true);
}
private bool TryMatchExistingIndexToQuery(DynamicQueryMapping map, out Index index)
{
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore);
var matchResult = dynamicQueryToIndex.Match(map);
switch (matchResult.MatchType)
{
case DynamicQueryMatchType.Complete:
case DynamicQueryMatchType.CompleteButIdle:
index = _indexStore.GetIndex(matchResult.IndexName);
if (index == null)
{
// the auto index was deleted
break;
}
return(true);
case DynamicQueryMatchType.Partial:
// At this point, we found an index that has some fields we need and
// isn't incompatible with anything else we're asking for
// We need to clone that other index
// We need to add all our requested indexes information to our cloned index
// We can then use our new index instead
var currentIndex = _indexStore.GetIndex(matchResult.IndexName);
if (currentIndex != null)
{
if (map.SupersededIndexes == null)
{
map.SupersededIndexes = new List <Index>();
}
map.SupersededIndexes.Add(currentIndex);
map.ExtendMappingBasedOn((AutoIndexDefinitionBase)currentIndex.Definition);
}
break;
}
index = null;
return(false);
}
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null)
{
var definitions = _indexStore.GetIndexesForCollection(query.ForCollection)
.Where(x => x.Type.IsAuto() && (query.IsGroupBy ? x.Type.IsMapReduce() : x.Type.IsMap()))
.Select(x => x.Definition as AutoIndexDefinitionBase)
.ToList();
if (definitions.Count == 0)
{
return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure));
}
var results = definitions.Select(definition => ConsiderUsageOfIndex(query, definition, explanations))
.Where(result => result.MatchType != DynamicQueryMatchType.Failure)
.GroupBy(x => x.MatchType)
.ToDictionary(x => x.Key, x => x.ToArray());
if (results.TryGetValue(DynamicQueryMatchType.Complete, out DynamicQueryMatchResult[] matchResults) && matchResults.Length > 0)
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null)
{
var definitions = _indexStore.GetIndexesForCollection(query.ForCollection)
.Where(x => query.IsMapReduce ? x.Type.IsMapReduce() : x.Type.IsMap())
.Select(x => x.Definition)
.ToList();
if (definitions.Count == 0)
{
return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure));
}
var results = definitions.Select(definition => ConsiderUsageOfIndex(query, definition, explanations))
.Where(result => result.MatchType != DynamicQueryMatchType.Failure)
.GroupBy(x => x.MatchType)
.ToDictionary(x => x.Key, x => x.ToArray());
DynamicQueryMatchResult[] matchResults;
if (results.TryGetValue(DynamicQueryMatchType.Complete, out matchResults) && matchResults.Length > 0)
{
var prioritizedResults = matchResults
.OrderByDescending(x => x.LastMappedEtag)
.ThenByDescending(x => x.NumberOfMappedFields)
.ToArray();
if (explanations != null)
{
for (var i = 1; i < prioritizedResults.Length; i++)
{
explanations.Add(new Explanation(prioritizedResults[i].IndexName, "Wasn't the widest / most unstable index matching this query"));
}
}
return(prioritizedResults[0]);
}
if (results.TryGetValue(DynamicQueryMatchType.Partial, out matchResults) && matchResults.Length > 0)
{
return(matchResults.OrderByDescending(x => x.NumberOfMappedFields).First());
}
return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure));
}
private Index MatchIndex(string dynamicIndexName, IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, out string collection)
{
collection = dynamicIndexName.Length == DynamicIndex.Length
? Constants.Indexing.AllDocumentsCollection
: dynamicIndexName.Substring(DynamicIndexPrefix.Length);
var map = DynamicQueryMapping.Create(collection, query);
if (map.MapFields.Length == 0 && map.SortDescriptors.Length == 0 && map.GroupByFields.Length == 0)
{
return(null); // use collection query
}
Index index;
if (TryMatchExistingIndexToQuery(map, out index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistsException("Could not find index for a given query.");
}
var definition = map.CreateAutoIndexDefinition();
var id = _indexStore.CreateIndex(definition);
index = _indexStore.GetIndex(id);
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results
}
}
EnsureValidQuery(query, map);
return(index);
}
public static DynamicQueryMapping Create(string entityName, IndexQueryServerSide query)
{
var result = new DynamicQueryMapping
{
ForCollection = entityName,
};
IEnumerable <DynamicQueryMappingItem> dynamicMapFields;
string[] numericFields;
if (query.DynamicMapReduceFields == null)
{
// auto map query
var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query); // TODO arek - not sure if we really need a Tuple<string, string> here
if (query.SortedFields != null)
{
foreach (var sortedField in query.SortedFields)
{
var field = sortedField.Field;
if (field == Constants.Indexing.Fields.IndexFieldScoreName)
{
continue;
}
if (field.StartsWith(Constants.Indexing.Fields.RandomFieldName) ||
field.StartsWith(Constants.Indexing.Fields.CustomSortFieldName))
{
continue;
}
if (InvariantCompare.IsPrefix(field, Constants.Indexing.Fields.AlphaNumericFieldName, CompareOptions.None))
{
field = SortFieldHelper.ExtractName(field);
}
if (InvariantCompare.IsSuffix(field, Constants.Indexing.Fields.RangeFieldSuffix, CompareOptions.None))
{
field = field.Substring(0, field.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length);
}
fields.Add(Tuple.Create(SimpleQueryParser.TranslateField(field), field));
}
}
dynamicMapFields = fields.Select(x => new DynamicQueryMappingItem(x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix) ? x.Item1.Substring(0, x.Item1.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length) : x.Item1, FieldMapReduceOperation.None));
numericFields = fields.Where(x => x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix)).Select(x => x.Item1).Distinct().ToArray();
}
else
{
// dynamic map-reduce query
result.IsMapReduce = true;
dynamicMapFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy == false).Select(x => new DynamicQueryMappingItem(x.Name, x.OperationType));
result.GroupByFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy).Select(x => x.Name).ToArray();
numericFields = null;
}
result.MapFields = dynamicMapFields
.Where(x => x.Name != Constants.Indexing.Fields.DocumentIdFieldName)
.OrderByDescending(x => x.Name.Length)
.ToArray();
result.SortDescriptors = GetSortInfo(query.SortedFields, numericFields);
result.HighlightedFields = query.HighlightedFields.EmptyIfNull().Select(x => x.Field).ToArray();
return(result);
}
private async Task <Index> MatchIndex(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, DocumentsOperationContext docsContext,
CancellationToken token)
{
Index index;
if (query.Metadata.AutoIndexName != null)
{
index = _indexStore.GetIndex(query.Metadata.AutoIndexName);
if (index != null)
{
return(index);
}
}
var map = DynamicQueryMapping.Create(query);
if (TryMatchExistingIndexToQuery(map, docsContext, out index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistException("Could not find index for a given query.");
}
var definition = map.CreateAutoIndexDefinition();
index = await _indexStore.CreateIndex(definition);
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
if (customStalenessWaitTimeout.HasValue)
{
query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout.Value;
}
else
{
query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results
}
}
var t = CleanupSupercededAutoIndexes(index, map, token)
.ContinueWith(task =>
{
if (task.Exception != null)
{
if (token.IsCancellationRequested)
{
return;
}
if (_indexStore.Logger.IsInfoEnabled)
{
_indexStore.Logger.Info("Failed to delete superceded indexes for index " + index.Name);
}
}
});
if (query.WaitForNonStaleResults &&
Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan ==
TimeSpan.Zero)
{
await t; // this is used in testing, mainly
}
}
return(index);
}
private async Task CleanupSupercededAutoIndexes(Index index, DynamicQueryMapping map, CancellationToken token)
{
if (map.SupersededIndexes == null || map.SupersededIndexes.Count == 0)
{
return;
}
// this is meant to remove superceded indexes immediately when they are of no use
// however, they'll also be cleaned by the idle timer, so we don't worry too much
// about this being in memory only operation
while (token.IsCancellationRequested == false)
{
AsyncManualResetEvent.FrozenAwaiter indexingBatchCompleted;
try
{
indexingBatchCompleted = index.GetIndexingBatchAwaiter();
}
catch (ObjectDisposedException)
{
break;
}
var maxSupercededEtag = 0L;
foreach (var supercededIndex in map.SupersededIndexes)
{
try
{
var etag = supercededIndex.GetLastMappedEtagFor(map.ForCollection);
maxSupercededEtag = Math.Max(etag, maxSupercededEtag);
}
catch (OperationCanceledException)
{
// the superceded index was already deleted
}
}
long currentEtag;
try
{
currentEtag = index.GetLastMappedEtagFor(map.ForCollection);
}
catch (OperationCanceledException)
{
// the index was already disposed by something else
break;
}
if (currentEtag >= maxSupercededEtag)
{
// we'll give it a few seconds to drain any pending queries,
// and because it make it easier to demonstrate how we auto
// clear the old auto indexes.
var timeout = Database.Configuration.Indexing.TimeBeforeDeletionOfSupersededAutoIndex.AsTimeSpan;
if (timeout != TimeSpan.Zero)
{
await TimeoutManager.WaitFor(
timeout
).ConfigureAwait(false);
}
foreach (var supercededIndex in map.SupersededIndexes)
{
try
{
await _indexStore.DeleteIndex(supercededIndex.Name);
}
catch (IndexDoesNotExistException)
{
}
}
break;
}
if (await indexingBatchCompleted.WaitAsync() == false)
{
break;
}
}
}
private DynamicQueryMatchResult ConsiderUsageOfIndex(DynamicQueryMapping query, IndexDefinitionBase definition, List <Explanation> explanations = null)
{
var collection = query.ForCollection;
var indexName = definition.Name;
if (definition.Collections.Contains(collection, StringComparer.OrdinalIgnoreCase) == false)
{
if (definition.Collections.Length == 0)
{
explanations?.Add(new Explanation(indexName, "Query is specific for collection, but the index searches across all of them, may result in a different type being returned."));
}
else
{
explanations?.Add(new Explanation(indexName, $"Index does not apply to collection '{collection}'"));
}
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
else
{
if (definition.Collections.Length > 1) // we only allow indexes with a single entity name
{
explanations?.Add(new Explanation(indexName, "Index contains more than a single entity name, may result in a different type being returned."));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
}
var index = _indexStore.GetIndex(definition.Name);
var state = index.State;
var stats = index.GetStats();
if (state == IndexState.Error || state == IndexState.Disabled || stats.IsInvalidIndex)
{
explanations?.Add(new Explanation(indexName, $"Cannot do dynamic queries on disabled index or index with errors (index name = {indexName})"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
var currentBestState = DynamicQueryMatchType.Complete;
foreach (var field in query.MapFields)
{
IndexField indexField;
if (definition.TryGetField(index.Type.IsAuto() ? field.Name : field.NormalizedName, out indexField))
{
if (string.IsNullOrWhiteSpace(indexField.Analyzer) == false)
{
explanations?.Add(new Explanation(indexName, $"The following field have a custom analyzer: {indexField.Name}"));
currentBestState = DynamicQueryMatchType.Partial;
}
if (indexField.Indexing != FieldIndexing.Default)
{
explanations?.Add(new Explanation(indexName, $"The following field is not using default indexing: {indexField.Name}"));
currentBestState = DynamicQueryMatchType.Partial;
}
}
else
{
explanations?.Add(new Explanation(indexName, $"The following field is missing: {indexField.Name}"));
currentBestState = DynamicQueryMatchType.Partial;
}
}
//TODO arek: ignore highlighting for now
foreach (var sortInfo in query.SortDescriptors) // with matching sort options
{
var sortFieldName = index.Type.IsAuto() ? sortInfo.Name : sortInfo.NormalizedName;
if (sortFieldName.StartsWith(Constants.Indexing.Fields.AlphaNumericFieldName) ||
sortFieldName.StartsWith(Constants.Indexing.Fields.RandomFieldName) ||
sortFieldName.StartsWith(Constants.Indexing.Fields.CustomSortFieldName))
{
sortFieldName = SortFieldHelper.ExtractName(sortFieldName);
}
if (sortFieldName.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix))
{
sortFieldName = sortFieldName.Substring(0, sortFieldName.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length);
}
IndexField indexField = null;
// if the field is not in the output, then we can't sort on it.
if (definition.ContainsField(sortFieldName) == false)
{
if (query.IsMapReduce == false)
{
explanations?.Add(new Explanation(indexName, $"Rejected because index does not contains field '{sortFieldName}' which we need to sort on"));
currentBestState = DynamicQueryMatchType.Partial;
continue;
}
// for map-reduce queries try to get field from group by fields as well
var autoMapReduceIndexDefinition = definition as AutoMapReduceIndexDefinition;
if (autoMapReduceIndexDefinition != null)
{
if (autoMapReduceIndexDefinition.GroupByFields.TryGetValue(sortFieldName, out indexField) == false)
{
explanations?.Add(new Explanation(indexName, $"Rejected because index does not contains field '{sortFieldName}' which we need to sort on"));
currentBestState = DynamicQueryMatchType.Partial;
continue;
}
}
else
{
var mapReduceIndexDefinition = definition as MapReduceIndexDefinition;
if (mapReduceIndexDefinition != null)
{
throw new NotImplementedException("TODO arek");
}
}
}
else
{
indexField = definition.GetField(sortFieldName);
}
Debug.Assert(indexField != null);
if (sortInfo.FieldType != indexField.SortOption)
{
if (indexField.SortOption == null)
{
switch (sortInfo.FieldType) // if field is not sorted, we check if we asked for the default sorting
{
case SortOptions.String:
case SortOptions.None:
continue;
}
}
explanations?.Add(new Explanation(indexName,
$"The specified sort type ({sortInfo.FieldType}) is different than the one specified for field '{sortFieldName}' ({indexField.SortOption})"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
}
if (currentBestState == DynamicQueryMatchType.Complete && state == IndexState.Idle)
{
currentBestState = DynamicQueryMatchType.Partial;
explanations?.Add(new Explanation(indexName, $"The index (name = {indexName}) is disabled or abandoned. The preference is for active indexes - making a partial match"));
}
if (currentBestState != DynamicQueryMatchType.Failure && query.IsMapReduce)
{
if (AssertMapReduceFields(query, (AutoMapReduceIndexDefinition)definition, currentBestState, explanations) == false)
{
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
}
if (currentBestState == DynamicQueryMatchType.Partial && index.Type.IsStatic()) // we cannot support this because we might extend fields from static index into auto index
{
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
return(new DynamicQueryMatchResult(indexName, currentBestState)
{
LastMappedEtag = index.GetLastMappedEtagFor(collection),
NumberOfMappedFields = definition.MapFields.Count
});
}
public static DynamicQueryMapping Create(IndexQueryServerSide query)
{
var result = new DynamicQueryMapping
{
ForCollection = query.Metadata.CollectionName
};
var mapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal);
foreach (var field in query.Metadata.IndexFieldNames)
{
if (field == Constants.Documents.Indexing.Fields.DocumentIdFieldName)
{
continue;
}
mapFields[field] = DynamicQueryMappingItem.Create(field, AggregationOperation.None, query.Metadata.WhereFields);
}
if (query.Metadata.OrderBy != null)
{
foreach (var field in query.Metadata.OrderBy)
{
if (field.OrderingType == OrderByFieldType.Random)
{
continue;
}
if (field.OrderingType == OrderByFieldType.Score)
{
continue;
}
if (field.Name == Constants.Documents.Indexing.Fields.DocumentIdFieldName)
{
continue;
}
var fieldName = field.Name;
#if FEATURE_CUSTOM_SORTING
if (fieldName.Value.StartsWith(Constants.Documents.Indexing.Fields.CustomSortFieldName))
{
continue;
}
#endif
if (mapFields.ContainsKey(field.Name))
{
continue;
}
mapFields.Add(field.Name, DynamicQueryMappingItem.Create(fieldName, field.AggregationOperation));
}
}
if (query.Metadata.IsGroupBy)
{
result.IsGroupBy = true;
result.GroupByFields = CreateGroupByFields(query, mapFields);
foreach (var field in mapFields)
{
if (field.Value.AggregationOperation == AggregationOperation.None)
{
throw new InvalidQueryException($"Field '{field.Key}' isn't neither an aggregation operation nor part of the group by key", query.Metadata.QueryText,
query.QueryParameters);
}
}
}
if (query.Metadata.HasHighlightings)
{
foreach (var highlighting in query.Metadata.Highlightings)
{
var fieldName = highlighting.Field;
if (mapFields.TryGetValue(fieldName, out var value))
{
value.HasHighlighting = true;
}
else
{
mapFields[fieldName] = DynamicQueryMappingItem.Create(fieldName, AggregationOperation.None, isFullTextSearch: true, isExactSearch: false, hasHighlighting: true, hasSuggestions: false, spatial: null);
}
}
}
if (query.Metadata.HasSuggest)
{
foreach (var selectField in query.Metadata.SelectFields)
{
var fieldName = selectField.Name;
if (mapFields.TryGetValue(fieldName, out var value))
{
value.HasSuggestions = true;
}
else
{
mapFields[fieldName] = DynamicQueryMappingItem.Create(fieldName, AggregationOperation.None, isFullTextSearch: false, isExactSearch: false, hasHighlighting: false, hasSuggestions: true, spatial: null);
}
}
}
result.MapFields = mapFields;
return(result);
}
public static DynamicQueryMapping Create(IndexQueryServerSide query)
{
var result = new DynamicQueryMapping
{
ForCollection = query.Metadata.CollectionName
};
var mapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal);
foreach (var field in query.Metadata.IndexFieldNames)
{
if (field == Constants.Documents.Indexing.Fields.DocumentIdFieldName)
{
continue;
}
mapFields[field] = DynamicQueryMappingItem.Create(field, AggregationOperation.None, query.Metadata.WhereFields);
}
if (query.Metadata.OrderBy != null)
{
foreach (var field in query.Metadata.OrderBy)
{
if (field.OrderingType == OrderByFieldType.Random)
{
continue;
}
if (field.OrderingType == OrderByFieldType.Score)
{
continue;
}
var fieldName = field.Name;
if (fieldName.Value.StartsWith(Constants.Documents.Indexing.Fields.CustomSortFieldName))
{
continue;
}
if (mapFields.ContainsKey(field.Name))
{
continue;
}
mapFields.Add(field.Name, DynamicQueryMappingItem.Create(fieldName, field.AggregationOperation));
}
}
if (query.Metadata.IsGroupBy)
{
result.IsGroupBy = true;
result.GroupByFields = CreateGroupByFields(query, mapFields);
foreach (var field in mapFields)
{
if (field.Value.AggregationOperation == AggregationOperation.None)
{
throw new InvalidQueryException($"Field '{field.Key}' isn't neither an aggregation operation nor part of the group by key", query.Metadata.QueryText,
query.QueryParameters);
}
}
}
result.MapFields = mapFields;
return(result);
}
private static DynamicQueryMatchType AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState,
List <Explanation> explanations)
{
var indexName = definition.Name;
foreach (var mapField in query.MapFields.Values)
{
if (definition.ContainsField(mapField.Name) == false)
{
Debug.Assert(currentBestState == DynamicQueryMatchType.Partial);
continue;
}
var field = definition.GetField(mapField.Name);
if (field.Aggregation != mapField.AggregationOperation)
{
explanations?.Add(new Explanation(indexName,
$"The following field {field.Name} has {field.Aggregation} operation defined, while query required {mapField.AggregationOperation}"));
return(DynamicQueryMatchType.Failure);
}
}
foreach (var groupByField in query.GroupByFields.Values)
{
if (definition.GroupByFields.TryGetValue(groupByField.Name, out var indexField))
{
if (groupByField.GroupByArrayBehavior != indexField.GroupByArrayBehavior)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field {indexField.Name} is grouping by '{indexField.GroupByArrayBehavior}', while the query needs to perform '{groupByField.GroupByArrayBehavior}' grouping"));
return(DynamicQueryMatchType.Failure);
}
if (groupByField.IsSpecifiedInWhere == false)
{
continue;
}
if (groupByField.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field is not searchable {indexField.Name}, while the query needs to perform search() on it"));
return(DynamicQueryMatchType.Partial);
}
if (groupByField.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field is not exactable {indexField.Name}, while the query needs to perform exact() on it"));
return(DynamicQueryMatchType.Partial);
}
}
else
{
if (explanations != null)
{
var missingFields = query.GroupByFields.Where(x => definition.GroupByFields.ContainsKey(x.Value.Name) == false);
explanations.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}"));
}
return(DynamicQueryMatchType.Failure);
}
}
if (query.GroupByFields.Count != definition.GroupByFields.Count)
{
if (explanations != null)
{
var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Select(y => y.Value.Name.Value).Contains(x.Key) == false);
explanations.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}"));
}
return(DynamicQueryMatchType.Failure);
}
return(DynamicQueryMatchType.Complete);
}
internal DynamicQueryMatchResult ConsiderUsageOfIndex(DynamicQueryMapping query, AutoIndexDefinitionBase definition, List <Explanation> explanations = null)
{
var collection = query.ForCollection;
var indexName = definition.Name;
if (definition.Collections.Contains(collection) == false)
{
explanations?.Add(new Explanation(indexName,
definition.Collections.Count == 0
? "Query is specific for collection, but the index searches across all of them, may result in a different type being returned."
: $"Index does not apply to collection '{collection}'"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
if (definition.Collections.Count > 1) // we only allow indexes with a single entity name
{
explanations?.Add(new Explanation(indexName, "Index contains more than a single entity name, may result in a different type being returned."));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
var index = _indexStore.GetIndex(definition.Name);
if (index == null)
{
return(new DynamicQueryMatchResult(definition.Name, DynamicQueryMatchType.Failure));
}
var state = index.State;
bool isInvalidStats;
try
{
isInvalidStats = index.IsInvalidIndex();
}
catch (OperationCanceledException)
{
return(new DynamicQueryMatchResult(definition.Name, DynamicQueryMatchType.Failure));
}
if (state == IndexState.Error || state == IndexState.Disabled || isInvalidStats)
{
explanations?.Add(new Explanation(indexName, $"Cannot do dynamic queries on disabled index or index with errors (index name = {indexName})"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
var currentBestState = DynamicQueryMatchType.Complete;
foreach (var field in query.MapFields.Values)
{
if (definition.TryGetField(field.Name, out var indexField))
{
if (field.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false)
{
explanations?.Add(new Explanation(indexName, $"The following field is not searchable {indexField.Name}, while the query needs to search() on it"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial));
}
if (field.HasHighlighting && indexField.Indexing.HasFlag(AutoFieldIndexing.Highlighting) == false)
{
explanations?.Add(new Explanation(indexName, $"The following field does not have highlighting {indexField.Name}, while the query needs to do highlight() on it"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial));
}
if (field.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false)
{
explanations?.Add(new Explanation(indexName, $"The following field is not exactable {indexField.Name}, while the query needs to perform exact() on it"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial));
}
if (field.Spatial != null)
{
if (field.Spatial.Equals(indexField.Spatial) == false)
{
explanations?.Add(new Explanation(indexName, $"The following field is not a spatial field {indexField.Name}, while the query needs to perform spatial() on it"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
}
if (field.HasSuggestions && indexField.HasSuggestions == false)
{
explanations?.Add(new Explanation(indexName, $"The following field does not have suggestions enabled {indexField.Name}, while the query needs to perform suggest() on it"));
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial));
}
}
else
{
explanations?.Add(new Explanation(indexName, $"The following field is missing: {field.Name}"));
currentBestState = DynamicQueryMatchType.Partial;
}
}
if (currentBestState == DynamicQueryMatchType.Complete && state == IndexState.Idle)
{
currentBestState = DynamicQueryMatchType.CompleteButIdle;
explanations?.Add(new Explanation(indexName, $"The index (name = {indexName}) is idle. The preference is for active indexes - making a complete match but marking the index is idle"));
}
if (currentBestState != DynamicQueryMatchType.Failure && query.IsGroupBy)
{
var bestMapReduceMatch = AssertMapReduceFields(query, (AutoMapReduceIndexDefinition)definition, currentBestState, explanations);
if (bestMapReduceMatch != DynamicQueryMatchType.Complete)
{
return(new DynamicQueryMatchResult(indexName, bestMapReduceMatch));
}
}
long lastMappedEtagFor;
try
{
lastMappedEtagFor = index.GetLastMappedEtagFor(collection);
}
catch (OperationCanceledException)
{
// the index was disposed while we were reading it, just ignore it
// probably dynamic index that was disposed by the auto cleaner
return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure));
}
return(new DynamicQueryMatchResult(indexName, currentBestState)
{
LastMappedEtag = lastMappedEtagFor,
NumberOfMappedFields = definition.MapFields.Count
});
}
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null)
{
var bestComplete = DynamicQueryMatchResult.Failure;
foreach (var index in _indexStore.GetIndexesForCollection(query.ForCollection))
{
if (query.IsGroupBy)
{
if (index.Type != IndexType.AutoMapReduce)
{
continue;
}
}
else if (index.Type != IndexType.AutoMap)
{
continue;
}
var auto = (AutoIndexDefinitionBaseServerSide)index.Definition;
var result = ConsiderUsageOfIndex(query, auto, explanations);
string reason = null;
bool hasBetterMatch = false;
if (BetterMatchAvailable(out var bothMatch))
{
hasBetterMatch = result.MatchType > bestComplete.MatchType;
reason = "A better match was available";
}
else if (result.LastMappedEtag != bestComplete.LastMappedEtag)
{
hasBetterMatch = result.LastMappedEtag > bestComplete.LastMappedEtag;
reason = "Wasn't the most up to date index matching this query";
}
else if (bothMatch && result.MatchType != bestComplete.MatchType)
{
// both indexes have reached the same etag but one of them is idle
hasBetterMatch = result.MatchType > bestComplete.MatchType;
reason = "The index is idle. The preference is for active indexes - making a complete match but marking the index as idle";
}
else if (result.NumberOfMappedFields != bestComplete.NumberOfMappedFields)
{
hasBetterMatch = result.NumberOfMappedFields > bestComplete.NumberOfMappedFields;
reason = "Wasn't the widest index matching this query";
}
if (explanations != null &&
bestComplete.MatchType != DynamicQueryMatchType.Failure &&
reason != null)
{
var indexName = hasBetterMatch ? bestComplete.IndexName : result.IndexName;
explanations.Add(new Explanation(indexName, reason));
}
if (hasBetterMatch)
{
bestComplete = result;
}
bool BetterMatchAvailable(out bool bothComplete)
{
bothComplete = false;
if (result.MatchType == bestComplete.MatchType)
{
return(false);
}
if (result.IsComplete() && bestComplete.IsComplete())
{
// both indexes are a perfect match for the query
// we'll choose the most up to date one and wake it up if needed
bothComplete = true;
return(false);
}
return(true);
}
}
return(bestComplete);
}
private async Task <(Index Index, string Collection)> MatchIndex(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound)
{
var collection = query.Metadata.CollectionName;
if (query.Metadata.DynamicIndexName != null)
{
var previousIndex = _indexStore.GetIndex(query.Metadata.DynamicIndexName);
if (previousIndex != null)
{
return(previousIndex, collection);
}
}
if (query.Metadata.IsCollectionQuery)
{
return(null, collection);
}
var map = DynamicQueryMapping.Create(query);
if (TryMatchExistingIndexToQuery(map, out Index index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistException("Could not find index for a given query.");
}
var definition = map.CreateAutoIndexDefinition();
var id = await _indexStore.CreateIndex(definition);
index = _indexStore.GetIndex(id);
var t = CleanupSupercededAutoIndexes(index, map)
.ContinueWith(task =>
{
if (task.Exception != null)
{
if (_token.Token.IsCancellationRequested)
{
return;
}
if (_indexStore.Logger.IsInfoEnabled)
{
_indexStore.Logger.Info("Failed to delete superceded indexes for index " + index.Name);
}
}
});
if (query.WaitForNonStaleResults &&
_configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan ==
TimeSpan.Zero)
{
await t; // this is used in testing, mainly
}
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results
}
}
return(index, collection);
}
public async Task <(Index Index, bool HasCreatedAutoIndex)> CreateAutoIndexIfNeeded(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, CancellationToken token)
{
var map = DynamicQueryMapping.Create(query);
bool hasCreatedAutoIndex = false;
Index index;
while (TryMatchExistingIndexToQuery(map, out index) == false)
{
if (createAutoIndexIfNoMatchIsFound == false)
{
throw new IndexDoesNotExistException("No Auto Index matching the given query was found.");
}
if (query.DisableAutoIndexCreation)
{
throw new InvalidOperationException("Creation of Auto Indexes was disabled and no Auto Index matching the given query was found.");
}
var definition = map.CreateAutoIndexDefinition();
index = await _indexStore.CreateIndex(definition, RaftIdGenerator.NewId());
if (index == null)
{
// the index was deleted, we'll try to find a better match (replaced by a wider index)
continue;
}
hasCreatedAutoIndex = true;
if (query.WaitForNonStaleResultsTimeout.HasValue == false)
{
query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout ?? TimeSpan.FromSeconds(15);
}
var t = CleanupSupersededAutoIndexes(index, map, RaftIdGenerator.NewId(), token)
.ContinueWith(task =>
{
if (task.Exception != null)
{
if (token.IsCancellationRequested)
{
return;
}
if (_indexStore.Logger.IsInfoEnabled)
{
_indexStore.Logger.Info("Failed to delete superseded indexes for index " + index.Name);
}
}
}, token);
if (query.WaitForNonStaleResults &&
Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan ==
TimeSpan.Zero)
{
await t; // this is used in testing, mainly
}
break;
}
return(index, hasCreatedAutoIndex);
}
