public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(IndexQueryServerSide query) { var map = DynamicQueryMapping.Create(query); var explanations = new List <DynamicQueryToIndexMatcher.Explanation>(); var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore); dynamicQueryToIndex.Match(map, explanations); return(explanations); }
public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(IndexQueryServerSide query, DocumentsOperationContext docsContext, out string indexName) { var map = DynamicQueryMapping.Create(query); var explanations = new List <DynamicQueryToIndexMatcher.Explanation>(); var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore); var match = dynamicQueryToIndex.Match(map, docsContext, explanations); indexName = match.IndexName; return(explanations); }
public List <DynamicQueryToIndexMatcher.Explanation> ExplainIndexSelection(string dynamicIndexName, IndexQueryServerSide query) { var collection = dynamicIndexName.Substring(DynamicIndexPrefix.Length); var map = DynamicQueryMapping.Create(collection, query); var explanations = new List <DynamicQueryToIndexMatcher.Explanation>(); var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore); dynamicQueryToIndex.Match(map, explanations); return(explanations); }
public static DynamicQueryMapping Create(Index index) { if (index.Type.IsAuto() == false) { throw new InvalidOperationException(); } var mapping = new DynamicQueryMapping { ForCollection = index.Collections.First(), MapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal) }; var definition = (AutoIndexDefinitionBase)index.Definition; foreach (var field in definition.MapFields) { var autoField = (AutoIndexField)field.Value; mapping.MapFields[field.Key] = DynamicQueryMappingItem.Create( new QueryFieldName(autoField.Name, autoField.HasQuotedName), autoField.Aggregation, isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search), isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact), hasHighlighting: autoField.Indexing.HasFlag(AutoFieldIndexing.Highlighting), hasSuggestions: autoField.HasSuggestions, spatial: autoField.Spatial); } if (index.Type.IsMapReduce()) { mapping.IsGroupBy = true; var mapReduceDefinition = (AutoMapReduceIndexDefinition)definition; mapping.GroupByFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal); foreach (var field in mapReduceDefinition.GroupByFields) { var autoField = field.Value; mapping.GroupByFields[field.Key] = DynamicQueryMappingItem.CreateGroupBy( new QueryFieldName(autoField.Name, autoField.HasQuotedName), autoField.GroupByArrayBehavior, isSpecifiedInWhere: true, isFullTextSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Search), isExactSearch: autoField.Indexing.HasFlag(AutoFieldIndexing.Exact)); } } return(mapping); }
public async Task <(Index Index, bool HasCreatedAutoIndex)> CreateAutoIndexIfNeeded(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, DocumentsOperationContext docsContext, CancellationToken token) { var map = DynamicQueryMapping.Create(query); bool hasCreatedAutoIndex = false; if (TryMatchExistingIndexToQuery(map, docsContext, out var index) == false) { if (createAutoIndexIfNoMatchIsFound == false) { throw new IndexDoesNotExistException("Could not find index for a given query."); } var definition = map.CreateAutoIndexDefinition(); index = await _indexStore.CreateIndex(definition, RaftIdGenerator.NewId()); hasCreatedAutoIndex = true; if (query.WaitForNonStaleResultsTimeout.HasValue == false) { query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout ?? TimeSpan.FromSeconds(15); } var t = CleanupSupersededAutoIndexes(index, map, RaftIdGenerator.NewId(), token) .ContinueWith(task => { if (task.Exception != null) { if (token.IsCancellationRequested) { return; } if (_indexStore.Logger.IsInfoEnabled) { _indexStore.Logger.Info("Failed to delete superseded indexes for index " + index.Name); } } }, token); if (query.WaitForNonStaleResults && Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan == TimeSpan.Zero) { await t; // this is used in testing, mainly } } return(index, hasCreatedAutoIndex); }
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null) { var bestComplete = DynamicQueryMatchResult.Failure; foreach (var index in _indexStore.GetIndexesForCollection(query.ForCollection)) { if (query.IsGroupBy) { if (index.Type != IndexType.AutoMapReduce) { continue; } } else if (index.Type != IndexType.AutoMap) { continue; } var auto = (AutoIndexDefinitionBase)index.Definition; var result = ConsiderUsageOfIndex(query, auto, explanations); if (result.MatchType < bestComplete.MatchType) { explanations?.Add(new Explanation(result.IndexName, "A better match was available")); continue; } const string notWidestMsg = "Wasn't the widest / most up to date index matching this query"; if (result.LastMappedEtag <= bestComplete.LastMappedEtag && result.NumberOfMappedFields <= bestComplete.NumberOfMappedFields) { explanations?.Add(new Explanation(bestComplete.IndexName, notWidestMsg)); continue; } if (explanations != null && bestComplete.MatchType != DynamicQueryMatchType.Failure) { explanations.Add(new Explanation(bestComplete.IndexName, notWidestMsg)); } bestComplete = result; } return(bestComplete); }
private bool AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations) { var indexName = definition.Name; foreach (var mapField in query.MapFields) { if (definition.ContainsField(mapField.Name) == false) { Debug.Assert(currentBestState == DynamicQueryMatchType.Partial); continue; } var field = definition.GetField(mapField.Name); if (field.MapReduceOperation != mapField.MapReduceOperation) { explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.MapReduceOperation} operation defined, while query required {mapField.MapReduceOperation}")); return(false); } } if (query.GroupByFields.All(definition.ContainsGroupByField) == false) { if (explanations != null) { var missingFields = query.GroupByFields.Where(x => definition.ContainsGroupByField(x) == false); explanations?.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}")); } return(false); } if (query.GroupByFields.Length != definition.GroupByFields.Count) { if (explanations != null) { var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Contains(x.Key) == false); explanations?.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}")); } return(false); } return(true); }
private bool TryMatchExistingIndexToQuery(DynamicQueryMapping map, out Index index) { var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(_indexStore); var matchResult = dynamicQueryToIndex.Match(map); switch (matchResult.MatchType) { case DynamicQueryMatchType.Complete: case DynamicQueryMatchType.CompleteButIdle: index = _indexStore.GetIndex(matchResult.IndexName); if (index == null) { // the auto index was deleted break; } return(true); case DynamicQueryMatchType.Partial: // At this point, we found an index that has some fields we need and // isn't incompatible with anything else we're asking for // We need to clone that other index // We need to add all our requested indexes information to our cloned index // We can then use our new index instead var currentIndex = _indexStore.GetIndex(matchResult.IndexName); if (currentIndex != null) { if (map.SupersededIndexes == null) { map.SupersededIndexes = new List <Index>(); } map.SupersededIndexes.Add(currentIndex); map.ExtendMappingBasedOn((AutoIndexDefinitionBase)currentIndex.Definition); } break; } index = null; return(false); }
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null) { var definitions = _indexStore.GetIndexesForCollection(query.ForCollection) .Where(x => x.Type.IsAuto() && (query.IsGroupBy ? x.Type.IsMapReduce() : x.Type.IsMap())) .Select(x => x.Definition as AutoIndexDefinitionBase) .ToList(); if (definitions.Count == 0) { return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure)); } var results = definitions.Select(definition => ConsiderUsageOfIndex(query, definition, explanations)) .Where(result => result.MatchType != DynamicQueryMatchType.Failure) .GroupBy(x => x.MatchType) .ToDictionary(x => x.Key, x => x.ToArray()); if (results.TryGetValue(DynamicQueryMatchType.Complete, out DynamicQueryMatchResult[] matchResults) && matchResults.Length > 0)
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null) { var definitions = _indexStore.GetIndexesForCollection(query.ForCollection) .Where(x => query.IsMapReduce ? x.Type.IsMapReduce() : x.Type.IsMap()) .Select(x => x.Definition) .ToList(); if (definitions.Count == 0) { return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure)); } var results = definitions.Select(definition => ConsiderUsageOfIndex(query, definition, explanations)) .Where(result => result.MatchType != DynamicQueryMatchType.Failure) .GroupBy(x => x.MatchType) .ToDictionary(x => x.Key, x => x.ToArray()); DynamicQueryMatchResult[] matchResults; if (results.TryGetValue(DynamicQueryMatchType.Complete, out matchResults) && matchResults.Length > 0) { var prioritizedResults = matchResults .OrderByDescending(x => x.LastMappedEtag) .ThenByDescending(x => x.NumberOfMappedFields) .ToArray(); if (explanations != null) { for (var i = 1; i < prioritizedResults.Length; i++) { explanations.Add(new Explanation(prioritizedResults[i].IndexName, "Wasn't the widest / most unstable index matching this query")); } } return(prioritizedResults[0]); } if (results.TryGetValue(DynamicQueryMatchType.Partial, out matchResults) && matchResults.Length > 0) { return(matchResults.OrderByDescending(x => x.NumberOfMappedFields).First()); } return(new DynamicQueryMatchResult(string.Empty, DynamicQueryMatchType.Failure)); }
private Index MatchIndex(string dynamicIndexName, IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, out string collection) { collection = dynamicIndexName.Length == DynamicIndex.Length ? Constants.Indexing.AllDocumentsCollection : dynamicIndexName.Substring(DynamicIndexPrefix.Length); var map = DynamicQueryMapping.Create(collection, query); if (map.MapFields.Length == 0 && map.SortDescriptors.Length == 0 && map.GroupByFields.Length == 0) { return(null); // use collection query } Index index; if (TryMatchExistingIndexToQuery(map, out index) == false) { if (createAutoIndexIfNoMatchIsFound == false) { throw new IndexDoesNotExistsException("Could not find index for a given query."); } var definition = map.CreateAutoIndexDefinition(); var id = _indexStore.CreateIndex(definition); index = _indexStore.GetIndex(id); if (query.WaitForNonStaleResultsTimeout.HasValue == false) { query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results } } EnsureValidQuery(query, map); return(index); }
public static DynamicQueryMapping Create(string entityName, IndexQueryServerSide query) { var result = new DynamicQueryMapping { ForCollection = entityName, }; IEnumerable <DynamicQueryMappingItem> dynamicMapFields; string[] numericFields; if (query.DynamicMapReduceFields == null) { // auto map query var fields = SimpleQueryParser.GetFieldsForDynamicQuery(query); // TODO arek - not sure if we really need a Tuple<string, string> here if (query.SortedFields != null) { foreach (var sortedField in query.SortedFields) { var field = sortedField.Field; if (field == Constants.Indexing.Fields.IndexFieldScoreName) { continue; } if (field.StartsWith(Constants.Indexing.Fields.RandomFieldName) || field.StartsWith(Constants.Indexing.Fields.CustomSortFieldName)) { continue; } if (InvariantCompare.IsPrefix(field, Constants.Indexing.Fields.AlphaNumericFieldName, CompareOptions.None)) { field = SortFieldHelper.ExtractName(field); } if (InvariantCompare.IsSuffix(field, Constants.Indexing.Fields.RangeFieldSuffix, CompareOptions.None)) { field = field.Substring(0, field.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length); } fields.Add(Tuple.Create(SimpleQueryParser.TranslateField(field), field)); } } dynamicMapFields = fields.Select(x => new DynamicQueryMappingItem(x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix) ? x.Item1.Substring(0, x.Item1.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length) : x.Item1, FieldMapReduceOperation.None)); numericFields = fields.Where(x => x.Item1.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix)).Select(x => x.Item1).Distinct().ToArray(); } else { // dynamic map-reduce query result.IsMapReduce = true; dynamicMapFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy == false).Select(x => new DynamicQueryMappingItem(x.Name, x.OperationType)); result.GroupByFields = query.DynamicMapReduceFields.Where(x => x.IsGroupBy).Select(x => x.Name).ToArray(); numericFields = null; } result.MapFields = dynamicMapFields .Where(x => x.Name != Constants.Indexing.Fields.DocumentIdFieldName) .OrderByDescending(x => x.Name.Length) .ToArray(); result.SortDescriptors = GetSortInfo(query.SortedFields, numericFields); result.HighlightedFields = query.HighlightedFields.EmptyIfNull().Select(x => x.Field).ToArray(); return(result); }
private async Task <Index> MatchIndex(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, DocumentsOperationContext docsContext, CancellationToken token) { Index index; if (query.Metadata.AutoIndexName != null) { index = _indexStore.GetIndex(query.Metadata.AutoIndexName); if (index != null) { return(index); } } var map = DynamicQueryMapping.Create(query); if (TryMatchExistingIndexToQuery(map, docsContext, out index) == false) { if (createAutoIndexIfNoMatchIsFound == false) { throw new IndexDoesNotExistException("Could not find index for a given query."); } var definition = map.CreateAutoIndexDefinition(); index = await _indexStore.CreateIndex(definition); if (query.WaitForNonStaleResultsTimeout.HasValue == false) { if (customStalenessWaitTimeout.HasValue) { query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout.Value; } else { query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results } } var t = CleanupSupercededAutoIndexes(index, map, token) .ContinueWith(task => { if (task.Exception != null) { if (token.IsCancellationRequested) { return; } if (_indexStore.Logger.IsInfoEnabled) { _indexStore.Logger.Info("Failed to delete superceded indexes for index " + index.Name); } } }); if (query.WaitForNonStaleResults && Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan == TimeSpan.Zero) { await t; // this is used in testing, mainly } } return(index); }
private async Task CleanupSupercededAutoIndexes(Index index, DynamicQueryMapping map, CancellationToken token) { if (map.SupersededIndexes == null || map.SupersededIndexes.Count == 0) { return; } // this is meant to remove superceded indexes immediately when they are of no use // however, they'll also be cleaned by the idle timer, so we don't worry too much // about this being in memory only operation while (token.IsCancellationRequested == false) { AsyncManualResetEvent.FrozenAwaiter indexingBatchCompleted; try { indexingBatchCompleted = index.GetIndexingBatchAwaiter(); } catch (ObjectDisposedException) { break; } var maxSupercededEtag = 0L; foreach (var supercededIndex in map.SupersededIndexes) { try { var etag = supercededIndex.GetLastMappedEtagFor(map.ForCollection); maxSupercededEtag = Math.Max(etag, maxSupercededEtag); } catch (OperationCanceledException) { // the superceded index was already deleted } } long currentEtag; try { currentEtag = index.GetLastMappedEtagFor(map.ForCollection); } catch (OperationCanceledException) { // the index was already disposed by something else break; } if (currentEtag >= maxSupercededEtag) { // we'll give it a few seconds to drain any pending queries, // and because it make it easier to demonstrate how we auto // clear the old auto indexes. var timeout = Database.Configuration.Indexing.TimeBeforeDeletionOfSupersededAutoIndex.AsTimeSpan; if (timeout != TimeSpan.Zero) { await TimeoutManager.WaitFor( timeout ).ConfigureAwait(false); } foreach (var supercededIndex in map.SupersededIndexes) { try { await _indexStore.DeleteIndex(supercededIndex.Name); } catch (IndexDoesNotExistException) { } } break; } if (await indexingBatchCompleted.WaitAsync() == false) { break; } } }
private DynamicQueryMatchResult ConsiderUsageOfIndex(DynamicQueryMapping query, IndexDefinitionBase definition, List <Explanation> explanations = null) { var collection = query.ForCollection; var indexName = definition.Name; if (definition.Collections.Contains(collection, StringComparer.OrdinalIgnoreCase) == false) { if (definition.Collections.Length == 0) { explanations?.Add(new Explanation(indexName, "Query is specific for collection, but the index searches across all of them, may result in a different type being returned.")); } else { explanations?.Add(new Explanation(indexName, $"Index does not apply to collection '{collection}'")); } return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } else { if (definition.Collections.Length > 1) // we only allow indexes with a single entity name { explanations?.Add(new Explanation(indexName, "Index contains more than a single entity name, may result in a different type being returned.")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } } var index = _indexStore.GetIndex(definition.Name); var state = index.State; var stats = index.GetStats(); if (state == IndexState.Error || state == IndexState.Disabled || stats.IsInvalidIndex) { explanations?.Add(new Explanation(indexName, $"Cannot do dynamic queries on disabled index or index with errors (index name = {indexName})")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } var currentBestState = DynamicQueryMatchType.Complete; foreach (var field in query.MapFields) { IndexField indexField; if (definition.TryGetField(index.Type.IsAuto() ? field.Name : field.NormalizedName, out indexField)) { if (string.IsNullOrWhiteSpace(indexField.Analyzer) == false) { explanations?.Add(new Explanation(indexName, $"The following field have a custom analyzer: {indexField.Name}")); currentBestState = DynamicQueryMatchType.Partial; } if (indexField.Indexing != FieldIndexing.Default) { explanations?.Add(new Explanation(indexName, $"The following field is not using default indexing: {indexField.Name}")); currentBestState = DynamicQueryMatchType.Partial; } } else { explanations?.Add(new Explanation(indexName, $"The following field is missing: {indexField.Name}")); currentBestState = DynamicQueryMatchType.Partial; } } //TODO arek: ignore highlighting for now foreach (var sortInfo in query.SortDescriptors) // with matching sort options { var sortFieldName = index.Type.IsAuto() ? sortInfo.Name : sortInfo.NormalizedName; if (sortFieldName.StartsWith(Constants.Indexing.Fields.AlphaNumericFieldName) || sortFieldName.StartsWith(Constants.Indexing.Fields.RandomFieldName) || sortFieldName.StartsWith(Constants.Indexing.Fields.CustomSortFieldName)) { sortFieldName = SortFieldHelper.ExtractName(sortFieldName); } if (sortFieldName.EndsWith(Constants.Indexing.Fields.RangeFieldSuffix)) { sortFieldName = sortFieldName.Substring(0, sortFieldName.Length - Constants.Indexing.Fields.RangeFieldSuffix.Length); } IndexField indexField = null; // if the field is not in the output, then we can't sort on it. if (definition.ContainsField(sortFieldName) == false) { if (query.IsMapReduce == false) { explanations?.Add(new Explanation(indexName, $"Rejected because index does not contains field '{sortFieldName}' which we need to sort on")); currentBestState = DynamicQueryMatchType.Partial; continue; } // for map-reduce queries try to get field from group by fields as well var autoMapReduceIndexDefinition = definition as AutoMapReduceIndexDefinition; if (autoMapReduceIndexDefinition != null) { if (autoMapReduceIndexDefinition.GroupByFields.TryGetValue(sortFieldName, out indexField) == false) { explanations?.Add(new Explanation(indexName, $"Rejected because index does not contains field '{sortFieldName}' which we need to sort on")); currentBestState = DynamicQueryMatchType.Partial; continue; } } else { var mapReduceIndexDefinition = definition as MapReduceIndexDefinition; if (mapReduceIndexDefinition != null) { throw new NotImplementedException("TODO arek"); } } } else { indexField = definition.GetField(sortFieldName); } Debug.Assert(indexField != null); if (sortInfo.FieldType != indexField.SortOption) { if (indexField.SortOption == null) { switch (sortInfo.FieldType) // if field is not sorted, we check if we asked for the default sorting { case SortOptions.String: case SortOptions.None: continue; } } explanations?.Add(new Explanation(indexName, $"The specified sort type ({sortInfo.FieldType}) is different than the one specified for field '{sortFieldName}' ({indexField.SortOption})")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } } if (currentBestState == DynamicQueryMatchType.Complete && state == IndexState.Idle) { currentBestState = DynamicQueryMatchType.Partial; explanations?.Add(new Explanation(indexName, $"The index (name = {indexName}) is disabled or abandoned. The preference is for active indexes - making a partial match")); } if (currentBestState != DynamicQueryMatchType.Failure && query.IsMapReduce) { if (AssertMapReduceFields(query, (AutoMapReduceIndexDefinition)definition, currentBestState, explanations) == false) { return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } } if (currentBestState == DynamicQueryMatchType.Partial && index.Type.IsStatic()) // we cannot support this because we might extend fields from static index into auto index { return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } return(new DynamicQueryMatchResult(indexName, currentBestState) { LastMappedEtag = index.GetLastMappedEtagFor(collection), NumberOfMappedFields = definition.MapFields.Count }); }
public static DynamicQueryMapping Create(IndexQueryServerSide query) { var result = new DynamicQueryMapping { ForCollection = query.Metadata.CollectionName }; var mapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal); foreach (var field in query.Metadata.IndexFieldNames) { if (field == Constants.Documents.Indexing.Fields.DocumentIdFieldName) { continue; } mapFields[field] = DynamicQueryMappingItem.Create(field, AggregationOperation.None, query.Metadata.WhereFields); } if (query.Metadata.OrderBy != null) { foreach (var field in query.Metadata.OrderBy) { if (field.OrderingType == OrderByFieldType.Random) { continue; } if (field.OrderingType == OrderByFieldType.Score) { continue; } if (field.Name == Constants.Documents.Indexing.Fields.DocumentIdFieldName) { continue; } var fieldName = field.Name; #if FEATURE_CUSTOM_SORTING if (fieldName.Value.StartsWith(Constants.Documents.Indexing.Fields.CustomSortFieldName)) { continue; } #endif if (mapFields.ContainsKey(field.Name)) { continue; } mapFields.Add(field.Name, DynamicQueryMappingItem.Create(fieldName, field.AggregationOperation)); } } if (query.Metadata.IsGroupBy) { result.IsGroupBy = true; result.GroupByFields = CreateGroupByFields(query, mapFields); foreach (var field in mapFields) { if (field.Value.AggregationOperation == AggregationOperation.None) { throw new InvalidQueryException($"Field '{field.Key}' isn't neither an aggregation operation nor part of the group by key", query.Metadata.QueryText, query.QueryParameters); } } } if (query.Metadata.HasHighlightings) { foreach (var highlighting in query.Metadata.Highlightings) { var fieldName = highlighting.Field; if (mapFields.TryGetValue(fieldName, out var value)) { value.HasHighlighting = true; } else { mapFields[fieldName] = DynamicQueryMappingItem.Create(fieldName, AggregationOperation.None, isFullTextSearch: true, isExactSearch: false, hasHighlighting: true, hasSuggestions: false, spatial: null); } } } if (query.Metadata.HasSuggest) { foreach (var selectField in query.Metadata.SelectFields) { var fieldName = selectField.Name; if (mapFields.TryGetValue(fieldName, out var value)) { value.HasSuggestions = true; } else { mapFields[fieldName] = DynamicQueryMappingItem.Create(fieldName, AggregationOperation.None, isFullTextSearch: false, isExactSearch: false, hasHighlighting: false, hasSuggestions: true, spatial: null); } } } result.MapFields = mapFields; return(result); }
public static DynamicQueryMapping Create(IndexQueryServerSide query) { var result = new DynamicQueryMapping { ForCollection = query.Metadata.CollectionName }; var mapFields = new Dictionary <string, DynamicQueryMappingItem>(StringComparer.Ordinal); foreach (var field in query.Metadata.IndexFieldNames) { if (field == Constants.Documents.Indexing.Fields.DocumentIdFieldName) { continue; } mapFields[field] = DynamicQueryMappingItem.Create(field, AggregationOperation.None, query.Metadata.WhereFields); } if (query.Metadata.OrderBy != null) { foreach (var field in query.Metadata.OrderBy) { if (field.OrderingType == OrderByFieldType.Random) { continue; } if (field.OrderingType == OrderByFieldType.Score) { continue; } var fieldName = field.Name; if (fieldName.Value.StartsWith(Constants.Documents.Indexing.Fields.CustomSortFieldName)) { continue; } if (mapFields.ContainsKey(field.Name)) { continue; } mapFields.Add(field.Name, DynamicQueryMappingItem.Create(fieldName, field.AggregationOperation)); } } if (query.Metadata.IsGroupBy) { result.IsGroupBy = true; result.GroupByFields = CreateGroupByFields(query, mapFields); foreach (var field in mapFields) { if (field.Value.AggregationOperation == AggregationOperation.None) { throw new InvalidQueryException($"Field '{field.Key}' isn't neither an aggregation operation nor part of the group by key", query.Metadata.QueryText, query.QueryParameters); } } } result.MapFields = mapFields; return(result); }
private static DynamicQueryMatchType AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations) { var indexName = definition.Name; foreach (var mapField in query.MapFields.Values) { if (definition.ContainsField(mapField.Name) == false) { Debug.Assert(currentBestState == DynamicQueryMatchType.Partial); continue; } var field = definition.GetField(mapField.Name); if (field.Aggregation != mapField.AggregationOperation) { explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.Aggregation} operation defined, while query required {mapField.AggregationOperation}")); return(DynamicQueryMatchType.Failure); } } foreach (var groupByField in query.GroupByFields.Values) { if (definition.GroupByFields.TryGetValue(groupByField.Name, out var indexField)) { if (groupByField.GroupByArrayBehavior != indexField.GroupByArrayBehavior) { explanations?.Add(new Explanation(indexName, $"The following group by field {indexField.Name} is grouping by '{indexField.GroupByArrayBehavior}', while the query needs to perform '{groupByField.GroupByArrayBehavior}' grouping")); return(DynamicQueryMatchType.Failure); } if (groupByField.IsSpecifiedInWhere == false) { continue; } if (groupByField.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false) { explanations?.Add(new Explanation(indexName, $"The following group by field is not searchable {indexField.Name}, while the query needs to perform search() on it")); return(DynamicQueryMatchType.Partial); } if (groupByField.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false) { explanations?.Add(new Explanation(indexName, $"The following group by field is not exactable {indexField.Name}, while the query needs to perform exact() on it")); return(DynamicQueryMatchType.Partial); } } else { if (explanations != null) { var missingFields = query.GroupByFields.Where(x => definition.GroupByFields.ContainsKey(x.Value.Name) == false); explanations.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}")); } return(DynamicQueryMatchType.Failure); } } if (query.GroupByFields.Count != definition.GroupByFields.Count) { if (explanations != null) { var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Select(y => y.Value.Name.Value).Contains(x.Key) == false); explanations.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}")); } return(DynamicQueryMatchType.Failure); } return(DynamicQueryMatchType.Complete); }
internal DynamicQueryMatchResult ConsiderUsageOfIndex(DynamicQueryMapping query, AutoIndexDefinitionBase definition, List <Explanation> explanations = null) { var collection = query.ForCollection; var indexName = definition.Name; if (definition.Collections.Contains(collection) == false) { explanations?.Add(new Explanation(indexName, definition.Collections.Count == 0 ? "Query is specific for collection, but the index searches across all of them, may result in a different type being returned." : $"Index does not apply to collection '{collection}'")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } if (definition.Collections.Count > 1) // we only allow indexes with a single entity name { explanations?.Add(new Explanation(indexName, "Index contains more than a single entity name, may result in a different type being returned.")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } var index = _indexStore.GetIndex(definition.Name); if (index == null) { return(new DynamicQueryMatchResult(definition.Name, DynamicQueryMatchType.Failure)); } var state = index.State; bool isInvalidStats; try { isInvalidStats = index.IsInvalidIndex(); } catch (OperationCanceledException) { return(new DynamicQueryMatchResult(definition.Name, DynamicQueryMatchType.Failure)); } if (state == IndexState.Error || state == IndexState.Disabled || isInvalidStats) { explanations?.Add(new Explanation(indexName, $"Cannot do dynamic queries on disabled index or index with errors (index name = {indexName})")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } var currentBestState = DynamicQueryMatchType.Complete; foreach (var field in query.MapFields.Values) { if (definition.TryGetField(field.Name, out var indexField)) { if (field.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false) { explanations?.Add(new Explanation(indexName, $"The following field is not searchable {indexField.Name}, while the query needs to search() on it")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial)); } if (field.HasHighlighting && indexField.Indexing.HasFlag(AutoFieldIndexing.Highlighting) == false) { explanations?.Add(new Explanation(indexName, $"The following field does not have highlighting {indexField.Name}, while the query needs to do highlight() on it")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial)); } if (field.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false) { explanations?.Add(new Explanation(indexName, $"The following field is not exactable {indexField.Name}, while the query needs to perform exact() on it")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial)); } if (field.Spatial != null) { if (field.Spatial.Equals(indexField.Spatial) == false) { explanations?.Add(new Explanation(indexName, $"The following field is not a spatial field {indexField.Name}, while the query needs to perform spatial() on it")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } } if (field.HasSuggestions && indexField.HasSuggestions == false) { explanations?.Add(new Explanation(indexName, $"The following field does not have suggestions enabled {indexField.Name}, while the query needs to perform suggest() on it")); return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Partial)); } } else { explanations?.Add(new Explanation(indexName, $"The following field is missing: {field.Name}")); currentBestState = DynamicQueryMatchType.Partial; } } if (currentBestState == DynamicQueryMatchType.Complete && state == IndexState.Idle) { currentBestState = DynamicQueryMatchType.CompleteButIdle; explanations?.Add(new Explanation(indexName, $"The index (name = {indexName}) is idle. The preference is for active indexes - making a complete match but marking the index is idle")); } if (currentBestState != DynamicQueryMatchType.Failure && query.IsGroupBy) { var bestMapReduceMatch = AssertMapReduceFields(query, (AutoMapReduceIndexDefinition)definition, currentBestState, explanations); if (bestMapReduceMatch != DynamicQueryMatchType.Complete) { return(new DynamicQueryMatchResult(indexName, bestMapReduceMatch)); } } long lastMappedEtagFor; try { lastMappedEtagFor = index.GetLastMappedEtagFor(collection); } catch (OperationCanceledException) { // the index was disposed while we were reading it, just ignore it // probably dynamic index that was disposed by the auto cleaner return(new DynamicQueryMatchResult(indexName, DynamicQueryMatchType.Failure)); } return(new DynamicQueryMatchResult(indexName, currentBestState) { LastMappedEtag = lastMappedEtagFor, NumberOfMappedFields = definition.MapFields.Count }); }
public DynamicQueryMatchResult Match(DynamicQueryMapping query, List <Explanation> explanations = null) { var bestComplete = DynamicQueryMatchResult.Failure; foreach (var index in _indexStore.GetIndexesForCollection(query.ForCollection)) { if (query.IsGroupBy) { if (index.Type != IndexType.AutoMapReduce) { continue; } } else if (index.Type != IndexType.AutoMap) { continue; } var auto = (AutoIndexDefinitionBaseServerSide)index.Definition; var result = ConsiderUsageOfIndex(query, auto, explanations); string reason = null; bool hasBetterMatch = false; if (BetterMatchAvailable(out var bothMatch)) { hasBetterMatch = result.MatchType > bestComplete.MatchType; reason = "A better match was available"; } else if (result.LastMappedEtag != bestComplete.LastMappedEtag) { hasBetterMatch = result.LastMappedEtag > bestComplete.LastMappedEtag; reason = "Wasn't the most up to date index matching this query"; } else if (bothMatch && result.MatchType != bestComplete.MatchType) { // both indexes have reached the same etag but one of them is idle hasBetterMatch = result.MatchType > bestComplete.MatchType; reason = "The index is idle. The preference is for active indexes - making a complete match but marking the index as idle"; } else if (result.NumberOfMappedFields != bestComplete.NumberOfMappedFields) { hasBetterMatch = result.NumberOfMappedFields > bestComplete.NumberOfMappedFields; reason = "Wasn't the widest index matching this query"; } if (explanations != null && bestComplete.MatchType != DynamicQueryMatchType.Failure && reason != null) { var indexName = hasBetterMatch ? bestComplete.IndexName : result.IndexName; explanations.Add(new Explanation(indexName, reason)); } if (hasBetterMatch) { bestComplete = result; } bool BetterMatchAvailable(out bool bothComplete) { bothComplete = false; if (result.MatchType == bestComplete.MatchType) { return(false); } if (result.IsComplete() && bestComplete.IsComplete()) { // both indexes are a perfect match for the query // we'll choose the most up to date one and wake it up if needed bothComplete = true; return(false); } return(true); } } return(bestComplete); }
private async Task <(Index Index, string Collection)> MatchIndex(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound) { var collection = query.Metadata.CollectionName; if (query.Metadata.DynamicIndexName != null) { var previousIndex = _indexStore.GetIndex(query.Metadata.DynamicIndexName); if (previousIndex != null) { return(previousIndex, collection); } } if (query.Metadata.IsCollectionQuery) { return(null, collection); } var map = DynamicQueryMapping.Create(query); if (TryMatchExistingIndexToQuery(map, out Index index) == false) { if (createAutoIndexIfNoMatchIsFound == false) { throw new IndexDoesNotExistException("Could not find index for a given query."); } var definition = map.CreateAutoIndexDefinition(); var id = await _indexStore.CreateIndex(definition); index = _indexStore.GetIndex(id); var t = CleanupSupercededAutoIndexes(index, map) .ContinueWith(task => { if (task.Exception != null) { if (_token.Token.IsCancellationRequested) { return; } if (_indexStore.Logger.IsInfoEnabled) { _indexStore.Logger.Info("Failed to delete superceded indexes for index " + index.Name); } } }); if (query.WaitForNonStaleResults && _configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan == TimeSpan.Zero) { await t; // this is used in testing, mainly } if (query.WaitForNonStaleResultsTimeout.HasValue == false) { query.WaitForNonStaleResultsTimeout = TimeSpan.FromSeconds(15); // allow new auto indexes to have some results } } return(index, collection); }
public async Task <(Index Index, bool HasCreatedAutoIndex)> CreateAutoIndexIfNeeded(IndexQueryServerSide query, bool createAutoIndexIfNoMatchIsFound, TimeSpan?customStalenessWaitTimeout, CancellationToken token) { var map = DynamicQueryMapping.Create(query); bool hasCreatedAutoIndex = false; Index index; while (TryMatchExistingIndexToQuery(map, out index) == false) { if (createAutoIndexIfNoMatchIsFound == false) { throw new IndexDoesNotExistException("No Auto Index matching the given query was found."); } if (query.DisableAutoIndexCreation) { throw new InvalidOperationException("Creation of Auto Indexes was disabled and no Auto Index matching the given query was found."); } var definition = map.CreateAutoIndexDefinition(); index = await _indexStore.CreateIndex(definition, RaftIdGenerator.NewId()); if (index == null) { // the index was deleted, we'll try to find a better match (replaced by a wider index) continue; } hasCreatedAutoIndex = true; if (query.WaitForNonStaleResultsTimeout.HasValue == false) { query.WaitForNonStaleResultsTimeout = customStalenessWaitTimeout ?? TimeSpan.FromSeconds(15); } var t = CleanupSupersededAutoIndexes(index, map, RaftIdGenerator.NewId(), token) .ContinueWith(task => { if (task.Exception != null) { if (token.IsCancellationRequested) { return; } if (_indexStore.Logger.IsInfoEnabled) { _indexStore.Logger.Info("Failed to delete superseded indexes for index " + index.Name); } } }, token); if (query.WaitForNonStaleResults && Database.Configuration.Indexing.TimeToWaitBeforeDeletingAutoIndexMarkedAsIdle.AsTimeSpan == TimeSpan.Zero) { await t; // this is used in testing, mainly } break; } return(index, hasCreatedAutoIndex); }