public void Failure_when_sort_options_do_not_match() { var definition = new AutoMapReduceIndexDefinition(new [] { "LineItems" }, new[] { new IndexField { Name = "Price", Highlighted = false, Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Sum, SortOption = SortOptions.String }, }, new [] { new IndexField { Name = "Name", Storage = FieldStorage.Yes } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create("LineItems", new IndexQueryServerSide { Query = "Price:70", SortedFields = new[] { new SortedField("Price_Range") }, }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public void Complete_match_for_single_matching_index_with_sort_options() { var definition = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count, }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide( "FROM Users GROUP BY Location WHERE Location = 'Poland' ORDER BY Count AS long ASC, Location ASC SELECT Location, count() ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType); Assert.Equal(definition.Name, result.IndexName); }
public void Partial_match_for_map_reduce_index_not_having_all_map_fields_defined_in_query() { var definition = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count, }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT Location, count(), sum(Sum) ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType); Assert.Equal(definition.Name, result.IndexName); }
public void Failure_match_for_map_reduce_index_having_different_aggregation_function() { var definition = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT Location, sum(Count) ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public static object ReadIndexDefinition(BlittableJsonReaderObject reader, BuildVersionType buildVersionType, out IndexType type) { switch (buildVersionType) { case BuildVersionType.V3: // pre 4.0 support var indexDefinition = ReadLegacyIndexDefinition(reader); type = indexDefinition.Type; Debug.Assert(type.IsStatic()); return(indexDefinition); case BuildVersionType.V4: type = ReadIndexType(reader, out reader); switch (type) { case IndexType.AutoMap: return(AutoMapIndexDefinition.LoadFromJson(reader)); case IndexType.AutoMapReduce: return(AutoMapReduceIndexDefinition.LoadFromJson(reader)); case IndexType.Map: case IndexType.MapReduce: return(JsonDeserializationServer.IndexDefinition(reader)); default: throw new NotSupportedException(type.ToString()); } default: throw new ArgumentOutOfRangeException(nameof(buildVersionType), buildVersionType, null); } }
public void Complete_match_and_surpassed_map_reduce_index_is_choosen() { Initialize(); var usersByCountGroupedByLocation = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count, }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); var usersByCountAndTotalAgeGroupedByLocation = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count, }, new AutoIndexField { Name = "TotalAge", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Sum }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(usersByCountGroupedByLocation); add_index(usersByCountAndTotalAgeGroupedByLocation); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location WHERE Location = 'Poland' SELECT Location, count() ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType); Assert.Equal(usersByCountAndTotalAgeGroupedByLocation.Name, result.IndexName); }
public void Complete_match_for_single_matching_index_with_sort_options() { var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count, SortOption = SortOptions.NumericDefault }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, SortOption = SortOptions.String } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "Location:Poland", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Count }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true } }, SortedFields = new [] { new SortedField("Count_Range"), new SortedField("Location"), } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType); Assert.Equal(definition.Name, result.IndexName); }
public static void Import(BlittableJsonReaderObject indexDefinitionDoc, DocumentDatabase database, long buildVersion, bool removeAnalyzers) { if (buildVersion == 0) // pre 4.0 support { var indexDefinition = ReadLegacyIndexDefinition(indexDefinitionDoc); if (string.Equals(indexDefinition.Name, "Raven/DocumentsByEntityName", StringComparison.OrdinalIgnoreCase)) { // skipping not needed old default index return; } database.IndexStore.CreateIndex(indexDefinition); } //I think supporting only major version as a number should be here, //so we can use ServerVersion.Build to get the build and not hardcode it else if (buildVersion == 13 || (buildVersion >= 40000 && buildVersion <= 44999) || (buildVersion >= 40 && buildVersion <= 44)) { var indexType = ReadIndexType(indexDefinitionDoc, out indexDefinitionDoc); switch (indexType) { case IndexType.AutoMap: var autoMapIndexDefinition = AutoMapIndexDefinition.LoadFromJson(indexDefinitionDoc); database.IndexStore.CreateIndex(autoMapIndexDefinition); break; case IndexType.AutoMapReduce: var autoMapReduceIndexDefinition = AutoMapReduceIndexDefinition.LoadFromJson(indexDefinitionDoc); database.IndexStore.CreateIndex(autoMapReduceIndexDefinition); break; case IndexType.Map: case IndexType.MapReduce: var indexDefinition = JsonDeserializationServer.IndexDefinition(indexDefinitionDoc); if (removeAnalyzers) { foreach (var indexDefinitionField in indexDefinition.Fields) { indexDefinitionField.Value.Analyzer = null; } } database.IndexStore.CreateIndex(indexDefinition); break; default: throw new NotSupportedException(indexType.ToString()); } } else { throw new NotSupportedException($"We do not support importing indexes from '{buildVersion}' build."); } }
public void Failure_if_matching_index_has_lot_of_errors() { var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count, }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); get_index(definition.Name)._indexStorage.UpdateStats(SystemTime.UtcNow, new IndexingRunStats { MapAttempts = 1000, MapSuccesses = 1000, ReduceAttempts = 1000, ReduceErrors = 900 }); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Count }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true } } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public void Partial_match_for_map_reduce_index_not_having_all_map_fields_defined_in_query() { var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "Location:Poland", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Count }, new DynamicMapReduceField { Name = "Sum", OperationType = FieldMapReduceOperation.Sum }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true } } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType); Assert.Equal(definition.Name, result.IndexName); }
private bool AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations) { var indexName = definition.Name; foreach (var mapField in query.MapFields) { if (definition.ContainsField(mapField.Name) == false) { Debug.Assert(currentBestState == DynamicQueryMatchType.Partial); continue; } var field = definition.GetField(mapField.Name); if (field.MapReduceOperation != mapField.MapReduceOperation) { explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.MapReduceOperation} operation defined, while query required {mapField.MapReduceOperation}")); return(false); } } if (query.GroupByFields.All(definition.ContainsGroupByField) == false) { if (explanations != null) { var missingFields = query.GroupByFields.Where(x => definition.ContainsGroupByField(x) == false); explanations?.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}")); } return(false); } if (query.GroupByFields.Length != definition.GroupByFields.Count) { if (explanations != null) { var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Contains(x.Key) == false); explanations?.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}")); } return(false); } return(true); }
public void Failure_match_for_map_reduce_index_having_different_aggregation_function() { var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "Location:Poland", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Sum }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true } } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public void Failure_if_matching_index_has_lot_of_errors() { Initialize(); var definition = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count, }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(definition); get_index(definition.Name)._indexStorage.UpdateStats(SystemTime.UtcNow, new IndexingRunStats { MapAttempts = 1000, MapSuccesses = 1000, ReduceAttempts = 1000, ReduceErrors = 900 }); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT count() ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public void Failure_match_for_map_reduce_index_not_matching_exactly_group_by_fields() { // missing nick name field to match var usersByCountReducedByAgeAndLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, }, new IndexField { Name = "Age", Storage = FieldStorage.Yes, } }); // additional Age field to match var usersByCountReducedByLocationAndNickNameAndAge = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, }, new IndexField { Name = "NickName", Storage = FieldStorage.Yes, }, new IndexField { Name = "Age", Storage = FieldStorage.Yes, } }); add_index(usersByCountReducedByAgeAndLocation); add_index(usersByCountReducedByLocationAndNickNameAndAge); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "Location:Poland", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Count }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true }, new DynamicMapReduceField { Name = "NickName", IsGroupBy = true } } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
public void Complete_match_and_surpassed_map_reduce_index_is_choosen() { var usersByCountGroupedByLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, } }); var usersByCountAndTotalAgeGroupedByLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[] { new IndexField { Name = "Count", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Count }, new IndexField { Name = "TotalAge", Storage = FieldStorage.Yes, MapReduceOperation = FieldMapReduceOperation.Sum }, }, new[] { new IndexField { Name = "Location", Storage = FieldStorage.Yes, } }); add_index(usersByCountGroupedByLocation); add_index(usersByCountAndTotalAgeGroupedByLocation); var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide { Query = "Location:Poland", DynamicMapReduceFields = new[] { new DynamicMapReduceField { Name = "Count", OperationType = FieldMapReduceOperation.Count }, new DynamicMapReduceField { Name = "Location", IsGroupBy = true } } }); var result = _sut.Match(dynamicQuery); Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType); Assert.Equal(usersByCountAndTotalAgeGroupedByLocation.Name, result.IndexName); }
public void Failure_match_for_map_reduce_index_not_matching_exactly_group_by_fields() { // missing nick name field to match var usersByCountReducedByAgeAndLocation = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, }, new AutoIndexField { Name = "Age", Storage = FieldStorage.Yes, } }); // additional Age field to match var usersByCountReducedByLocationAndNickNameAndAge = new AutoMapReduceIndexDefinition("Users", new[] { new AutoIndexField { Name = "Count", Storage = FieldStorage.Yes, Aggregation = AggregationOperation.Count }, }, new[] { new AutoIndexField { Name = "Location", Storage = FieldStorage.Yes, }, new AutoIndexField { Name = "NickName", Storage = FieldStorage.Yes, }, new AutoIndexField { Name = "Age", Storage = FieldStorage.Yes, } }); add_index(usersByCountReducedByAgeAndLocation); add_index(usersByCountReducedByLocationAndNickNameAndAge); var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location, NickName WHERE Location = 'Poland' SELECT Location, count() ")); var result = _sut.Match(dynamicQuery, null); Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType); }
private static DynamicQueryMatchType AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations) { var indexName = definition.Name; foreach (var mapField in query.MapFields.Values) { if (definition.ContainsField(mapField.Name) == false) { Debug.Assert(currentBestState == DynamicQueryMatchType.Partial); continue; } var field = definition.GetField(mapField.Name); if (field.Aggregation != mapField.AggregationOperation) { explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.Aggregation} operation defined, while query required {mapField.AggregationOperation}")); return(DynamicQueryMatchType.Failure); } } foreach (var groupByField in query.GroupByFields.Values) { if (definition.GroupByFields.TryGetValue(groupByField.Name, out var indexField)) { if (groupByField.GroupByArrayBehavior != indexField.GroupByArrayBehavior) { explanations?.Add(new Explanation(indexName, $"The following group by field {indexField.Name} is grouping by '{indexField.GroupByArrayBehavior}', while the query needs to perform '{groupByField.GroupByArrayBehavior}' grouping")); return(DynamicQueryMatchType.Failure); } if (groupByField.IsSpecifiedInWhere == false) { continue; } if (groupByField.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false) { explanations?.Add(new Explanation(indexName, $"The following group by field is not searchable {indexField.Name}, while the query needs to perform search() on it")); return(DynamicQueryMatchType.Partial); } if (groupByField.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false) { explanations?.Add(new Explanation(indexName, $"The following group by field is not exactable {indexField.Name}, while the query needs to perform exact() on it")); return(DynamicQueryMatchType.Partial); } } else { if (explanations != null) { var missingFields = query.GroupByFields.Where(x => definition.GroupByFields.ContainsKey(x.Value.Name) == false); explanations.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}")); } return(DynamicQueryMatchType.Failure); } } if (query.GroupByFields.Count != definition.GroupByFields.Count) { if (explanations != null) { var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Select(y => y.Value.Name.Value).Contains(x.Key) == false); explanations.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}")); } return(DynamicQueryMatchType.Failure); } return(DynamicQueryMatchType.Complete); }
private static AutoIndexDefinitionBase CreateAutoDefinition(AutoIndexDefinition definition) { var mapFields = definition .MapFields .Select(x => { var field = AutoIndexField.Create(x.Key, x.Value); field.Aggregation = x.Value.Aggregation; Debug.Assert(x.Value.GroupByArrayBehavior == GroupByArrayBehavior.NotApplicable); return(field); }) .ToArray(); if (definition.Type == IndexType.AutoMap) { var result = new AutoMapIndexDefinition(definition.Collection, mapFields); if (definition.LockMode.HasValue) { result.LockMode = definition.LockMode.Value; } if (definition.Priority.HasValue) { result.Priority = definition.Priority.Value; } return(result); } if (definition.Type == IndexType.AutoMapReduce) { var groupByFields = definition .GroupByFields .Select(x => { var field = AutoIndexField.Create(x.Key, x.Value); field.Aggregation = x.Value.Aggregation; field.GroupByArrayBehavior = x.Value.GroupByArrayBehavior; return(field); }) .ToArray(); var result = new AutoMapReduceIndexDefinition(definition.Collection, mapFields, groupByFields); if (definition.LockMode.HasValue) { result.LockMode = definition.LockMode.Value; } if (definition.Priority.HasValue) { result.Priority = definition.Priority.Value; } return(result); } throw new NotSupportedException("Cannot create auto-index from " + definition.Type); }