public void Failure_when_sort_options_do_not_match()
{
var definition = new AutoMapReduceIndexDefinition(new [] { "LineItems" }, new[]
{
new IndexField
{
Name = "Price",
Highlighted = false,
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Sum,
SortOption = SortOptions.String
},
}, new []
{
new IndexField
{
Name = "Name",
Storage = FieldStorage.Yes
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("LineItems", new IndexQueryServerSide
{
Query = "Price:70",
SortedFields = new[] { new SortedField("Price_Range") },
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_for_single_matching_index_with_sort_options()
{
var definition = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count,
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide(
"FROM Users GROUP BY Location WHERE Location = 'Poland' ORDER BY Count AS long ASC, Location ASC SELECT Location, count() "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Partial_match_for_map_reduce_index_not_having_all_map_fields_defined_in_query()
{
var definition = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count,
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT Location, count(), sum(Sum) "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Failure_match_for_map_reduce_index_having_different_aggregation_function()
{
var definition = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT Location, sum(Count) "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public static object ReadIndexDefinition(BlittableJsonReaderObject reader, BuildVersionType buildVersionType, out IndexType type)
{
switch (buildVersionType)
{
case BuildVersionType.V3:
// pre 4.0 support
var indexDefinition = ReadLegacyIndexDefinition(reader);
type = indexDefinition.Type;
Debug.Assert(type.IsStatic());
return(indexDefinition);
case BuildVersionType.V4:
type = ReadIndexType(reader, out reader);
switch (type)
{
case IndexType.AutoMap:
return(AutoMapIndexDefinition.LoadFromJson(reader));
case IndexType.AutoMapReduce:
return(AutoMapReduceIndexDefinition.LoadFromJson(reader));
case IndexType.Map:
case IndexType.MapReduce:
return(JsonDeserializationServer.IndexDefinition(reader));
default:
throw new NotSupportedException(type.ToString());
}
default:
throw new ArgumentOutOfRangeException(nameof(buildVersionType), buildVersionType, null);
}
}
public void Complete_match_and_surpassed_map_reduce_index_is_choosen()
{
Initialize();
var usersByCountGroupedByLocation = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count,
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
var usersByCountAndTotalAgeGroupedByLocation = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count,
},
new AutoIndexField
{
Name = "TotalAge",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Sum
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(usersByCountGroupedByLocation);
add_index(usersByCountAndTotalAgeGroupedByLocation);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location WHERE Location = 'Poland' SELECT Location, count() "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(usersByCountAndTotalAgeGroupedByLocation.Name, result.IndexName);
}
public void Complete_match_for_single_matching_index_with_sort_options()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count,
SortOption = SortOptions.NumericDefault
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
SortOption = SortOptions.String
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
},
SortedFields = new []
{
new SortedField("Count_Range"),
new SortedField("Location"),
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public static void Import(BlittableJsonReaderObject indexDefinitionDoc, DocumentDatabase database, long buildVersion, bool removeAnalyzers)
{
if (buildVersion == 0) // pre 4.0 support
{
var indexDefinition = ReadLegacyIndexDefinition(indexDefinitionDoc);
if (string.Equals(indexDefinition.Name, "Raven/DocumentsByEntityName", StringComparison.OrdinalIgnoreCase))
{
// skipping not needed old default index
return;
}
database.IndexStore.CreateIndex(indexDefinition);
}
//I think supporting only major version as a number should be here,
//so we can use ServerVersion.Build to get the build and not hardcode it
else if (buildVersion == 13 || (buildVersion >= 40000 && buildVersion <= 44999) || (buildVersion >= 40 && buildVersion <= 44))
{
var indexType = ReadIndexType(indexDefinitionDoc, out indexDefinitionDoc);
switch (indexType)
{
case IndexType.AutoMap:
var autoMapIndexDefinition = AutoMapIndexDefinition.LoadFromJson(indexDefinitionDoc);
database.IndexStore.CreateIndex(autoMapIndexDefinition);
break;
case IndexType.AutoMapReduce:
var autoMapReduceIndexDefinition = AutoMapReduceIndexDefinition.LoadFromJson(indexDefinitionDoc);
database.IndexStore.CreateIndex(autoMapReduceIndexDefinition);
break;
case IndexType.Map:
case IndexType.MapReduce:
var indexDefinition = JsonDeserializationServer.IndexDefinition(indexDefinitionDoc);
if (removeAnalyzers)
{
foreach (var indexDefinitionField in indexDefinition.Fields)
{
indexDefinitionField.Value.Analyzer = null;
}
}
database.IndexStore.CreateIndex(indexDefinition);
break;
default:
throw new NotSupportedException(indexType.ToString());
}
}
else
{
throw new NotSupportedException($"We do not support importing indexes from '{buildVersion}' build.");
}
}
public void Failure_if_matching_index_has_lot_of_errors()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count,
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
get_index(definition.Name)._indexStorage.UpdateStats(SystemTime.UtcNow, new IndexingRunStats
{
MapAttempts = 1000,
MapSuccesses = 1000,
ReduceAttempts = 1000,
ReduceErrors = 900
});
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Partial_match_for_map_reduce_index_not_having_all_map_fields_defined_in_query()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Sum",
OperationType = FieldMapReduceOperation.Sum
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
private bool AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState, List <Explanation> explanations)
{
var indexName = definition.Name;
foreach (var mapField in query.MapFields)
{
if (definition.ContainsField(mapField.Name) == false)
{
Debug.Assert(currentBestState == DynamicQueryMatchType.Partial);
continue;
}
var field = definition.GetField(mapField.Name);
if (field.MapReduceOperation != mapField.MapReduceOperation)
{
explanations?.Add(new Explanation(indexName, $"The following field {field.Name} has {field.MapReduceOperation} operation defined, while query required {mapField.MapReduceOperation}"));
return(false);
}
}
if (query.GroupByFields.All(definition.ContainsGroupByField) == false)
{
if (explanations != null)
{
var missingFields = query.GroupByFields.Where(x => definition.ContainsGroupByField(x) == false);
explanations?.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}"));
}
return(false);
}
if (query.GroupByFields.Length != definition.GroupByFields.Count)
{
if (explanations != null)
{
var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Contains(x.Key) == false);
explanations?.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}"));
}
return(false);
}
return(true);
}
public void Failure_match_for_map_reduce_index_having_different_aggregation_function()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Sum
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Failure_if_matching_index_has_lot_of_errors()
{
Initialize();
var definition = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count,
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
get_index(definition.Name)._indexStorage.UpdateStats(SystemTime.UtcNow, new IndexingRunStats
{
MapAttempts = 1000,
MapSuccesses = 1000,
ReduceAttempts = 1000,
ReduceErrors = 900
});
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT count() "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Failure_match_for_map_reduce_index_not_matching_exactly_group_by_fields()
{
// missing nick name field to match
var usersByCountReducedByAgeAndLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
},
new IndexField
{
Name = "Age",
Storage = FieldStorage.Yes,
}
});
// additional Age field to match
var usersByCountReducedByLocationAndNickNameAndAge = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
},
new IndexField
{
Name = "NickName",
Storage = FieldStorage.Yes,
},
new IndexField
{
Name = "Age",
Storage = FieldStorage.Yes,
}
});
add_index(usersByCountReducedByAgeAndLocation);
add_index(usersByCountReducedByLocationAndNickNameAndAge);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
},
new DynamicMapReduceField
{
Name = "NickName",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_and_surpassed_map_reduce_index_is_choosen()
{
var usersByCountGroupedByLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
var usersByCountAndTotalAgeGroupedByLocation = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
new IndexField
{
Name = "TotalAge",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Sum
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(usersByCountGroupedByLocation);
add_index(usersByCountAndTotalAgeGroupedByLocation);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(usersByCountAndTotalAgeGroupedByLocation.Name, result.IndexName);
}
public void Failure_match_for_map_reduce_index_not_matching_exactly_group_by_fields()
{
// missing nick name field to match
var usersByCountReducedByAgeAndLocation = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
},
new AutoIndexField
{
Name = "Age",
Storage = FieldStorage.Yes,
}
});
// additional Age field to match
var usersByCountReducedByLocationAndNickNameAndAge = new AutoMapReduceIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
Aggregation = AggregationOperation.Count
},
},
new[]
{
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
},
new AutoIndexField
{
Name = "NickName",
Storage = FieldStorage.Yes,
},
new AutoIndexField
{
Name = "Age",
Storage = FieldStorage.Yes,
}
});
add_index(usersByCountReducedByAgeAndLocation);
add_index(usersByCountReducedByLocationAndNickNameAndAge);
var dynamicQuery =
DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location, NickName WHERE Location = 'Poland' SELECT Location, count() "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
private static DynamicQueryMatchType AssertMapReduceFields(DynamicQueryMapping query, AutoMapReduceIndexDefinition definition, DynamicQueryMatchType currentBestState,
List <Explanation> explanations)
{
var indexName = definition.Name;
foreach (var mapField in query.MapFields.Values)
{
if (definition.ContainsField(mapField.Name) == false)
{
Debug.Assert(currentBestState == DynamicQueryMatchType.Partial);
continue;
}
var field = definition.GetField(mapField.Name);
if (field.Aggregation != mapField.AggregationOperation)
{
explanations?.Add(new Explanation(indexName,
$"The following field {field.Name} has {field.Aggregation} operation defined, while query required {mapField.AggregationOperation}"));
return(DynamicQueryMatchType.Failure);
}
}
foreach (var groupByField in query.GroupByFields.Values)
{
if (definition.GroupByFields.TryGetValue(groupByField.Name, out var indexField))
{
if (groupByField.GroupByArrayBehavior != indexField.GroupByArrayBehavior)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field {indexField.Name} is grouping by '{indexField.GroupByArrayBehavior}', while the query needs to perform '{groupByField.GroupByArrayBehavior}' grouping"));
return(DynamicQueryMatchType.Failure);
}
if (groupByField.IsSpecifiedInWhere == false)
{
continue;
}
if (groupByField.IsFullTextSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Search) == false)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field is not searchable {indexField.Name}, while the query needs to perform search() on it"));
return(DynamicQueryMatchType.Partial);
}
if (groupByField.IsExactSearch && indexField.Indexing.HasFlag(AutoFieldIndexing.Exact) == false)
{
explanations?.Add(new Explanation(indexName,
$"The following group by field is not exactable {indexField.Name}, while the query needs to perform exact() on it"));
return(DynamicQueryMatchType.Partial);
}
}
else
{
if (explanations != null)
{
var missingFields = query.GroupByFields.Where(x => definition.GroupByFields.ContainsKey(x.Value.Name) == false);
explanations.Add(new Explanation(indexName, $"The following group by fields are missing: {string.Join(", ", missingFields)}"));
}
return(DynamicQueryMatchType.Failure);
}
}
if (query.GroupByFields.Count != definition.GroupByFields.Count)
{
if (explanations != null)
{
var extraFields = definition.GroupByFields.Where(x => query.GroupByFields.Select(y => y.Value.Name.Value).Contains(x.Key) == false);
explanations.Add(new Explanation(indexName, $"Index {indexName} has additional group by fields: {string.Join(", ", extraFields)}"));
}
return(DynamicQueryMatchType.Failure);
}
return(DynamicQueryMatchType.Complete);
}
private static AutoIndexDefinitionBase CreateAutoDefinition(AutoIndexDefinition definition)
{
var mapFields = definition
.MapFields
.Select(x =>
{
var field = AutoIndexField.Create(x.Key, x.Value);
field.Aggregation = x.Value.Aggregation;
Debug.Assert(x.Value.GroupByArrayBehavior == GroupByArrayBehavior.NotApplicable);
return(field);
})
.ToArray();
if (definition.Type == IndexType.AutoMap)
{
var result = new AutoMapIndexDefinition(definition.Collection, mapFields);
if (definition.LockMode.HasValue)
{
result.LockMode = definition.LockMode.Value;
}
if (definition.Priority.HasValue)
{
result.Priority = definition.Priority.Value;
}
return(result);
}
if (definition.Type == IndexType.AutoMapReduce)
{
var groupByFields = definition
.GroupByFields
.Select(x =>
{
var field = AutoIndexField.Create(x.Key, x.Value);
field.Aggregation = x.Value.Aggregation;
field.GroupByArrayBehavior = x.Value.GroupByArrayBehavior;
return(field);
})
.ToArray();
var result = new AutoMapReduceIndexDefinition(definition.Collection, mapFields, groupByFields);
if (definition.LockMode.HasValue)
{
result.LockMode = definition.LockMode.Value;
}
if (definition.Priority.HasValue)
{
result.Priority = definition.Priority.Value;
}
return(result);
}
throw new NotSupportedException("Cannot create auto-index from " + definition.Type);
}
