public void Map_all_fields()
{
_sut = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location SELECT Location, count() "));
var definition = _sut.CreateAutoIndexDefinition();
Assert.Equal(1, definition.Collections.Count);
Assert.Equal("Users", definition.Collections.Single());
Assert.True(definition.ContainsField("Count"));
Assert.Equal("Auto/Users/ByCountReducedByLocation", definition.Name);
}
private static IndexQuery CreateIndexQuery(IndexQuery query, DynamicQueryMapping map, string realQuery)
{
var indexQuery = new IndexQuery
{
Cutoff = query.Cutoff,
WaitForNonStaleResultsAsOfNow = query.WaitForNonStaleResultsAsOfNow,
PageSize = query.PageSize,
Query = realQuery,
Start = query.Start,
FieldsToFetch = query.FieldsToFetch,
IsDistinct = query.IsDistinct,
SortedFields = query.SortedFields,
DefaultField = query.DefaultField,
CutoffEtag = query.CutoffEtag,
DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries,
DefaultOperator = query.DefaultOperator,
SkippedResults = query.SkippedResults,
HighlighterPreTags = query.HighlighterPreTags,
HighlighterPostTags = query.HighlighterPostTags,
HighlightedFields = query.HighlightedFields,
HighlighterKeyName = query.HighlighterKeyName,
ResultsTransformer = query.ResultsTransformer,
TransformerParameters = query.TransformerParameters,
ExplainScores = query.ExplainScores,
SortHints = query.SortHints
};
if (indexQuery.SortedFields == null)
{
return(indexQuery);
}
for (int index = 0; index < indexQuery.SortedFields.Length; index++)
{
var sortedField = indexQuery.SortedFields[index];
var fieldName = sortedField.Field;
bool hasRange = false;
if (fieldName.EndsWith("_Range"))
{
fieldName = fieldName.Substring(0, fieldName.Length - "_Range".Length);
hasRange = true;
}
var item = map.Items.FirstOrDefault(x => string.Equals(x.QueryFrom, fieldName, StringComparison.InvariantCultureIgnoreCase));
if (item == null)
{
continue;
}
indexQuery.SortedFields[index] = new SortedField(hasRange ? item.To + "_Range" : item.To);
indexQuery.SortedFields[index].Descending = sortedField.Descending;
}
return(indexQuery);
}
public void Complete_match_for_single_matching_index_with_sort_options()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count,
SortOption = SortOptions.NumericDefault
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
SortOption = SortOptions.String
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
},
SortedFields = new []
{
new SortedField("Count_Range"),
new SortedField("Location"),
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Failure_if_matching_index_has_lot_of_errors()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count,
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
get_index(definition.Name)._indexStorage.UpdateStats(SystemTime.UtcNow, new IndexingRunStats
{
MapAttempts = 1000,
MapSuccesses = 1000,
ReduceAttempts = 1000,
ReduceErrors = 900
});
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Partial_match_for_map_reduce_index_not_having_all_map_fields_defined_in_query()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Sum",
OperationType = FieldMapReduceOperation.Sum
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public QueryResult ExecuteDynamicQuery(string entityName, IndexQuery query)
{
// Create the map
var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);
var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);
map.IndexName = touchTemporaryIndexResult.Item1;
// Re-write the query
string realQuery = map.Items.Aggregate(query.Query,
(current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));
return(ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery));
}
public void WillGenerateProperQueryForMetadata()
{
using (var documentDatabase = new DocumentDatabase(new RavenConfiguration {
RunInMemory = true
}))
{
var mapping = DynamicQueryMapping.Create(documentDatabase, "@metadata.Raven-Graph-Type:Edge", null);
var indexDefinition = mapping.CreateIndexDefinition();
Assert.Equal(
"from doc in docs\nselect new {\n\t_metadata_Raven_Graph_Type = doc[\"@metadata\"][\"Raven-Graph-Type\"]\n}",
indexDefinition.Map);
}
}
private static void UpdateFieldNamesForSortedFields(IndexQuery query, DynamicQueryMapping map)
{
if (query.SortedFields == null)
{
return;
}
foreach (var sortedField in query.SortedFields)
{
var item = map.Items.FirstOrDefault(x => x.From == sortedField.Field);
if (item != null)
{
sortedField.Field = item.To;
}
}
}
private static void UpdateFieldsInArray(DynamicQueryMapping map, string[] fields)
{
if (fields == null)
{
return;
}
for (var i = 0; i < fields.Length; i++)
{
var item = map.Items.FirstOrDefault(x => x.From == fields[i]);
if (item != null)
{
fields[i] = item.To;
}
}
}
public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query, CancellationToken token)
{
// Create the map
var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);
var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);
string realQuery = map.Items.Aggregate(query.Query,
(current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));
UpdateFieldNamesForSortedFields(query, map);
// We explicitly do NOT want to update the field names of FieldsToFetch - that reads directly from the document
//UpdateFieldsInArray(map, query.FieldsToFetch);
return(ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery, token));
}
public void CreateDefinitionForQueryWithRangeField()
{
_sut = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Age_Range:{Lx30 TO NULL}"
});
var definition = _sut.CreateAutoIndexDefinition();
Assert.Equal(1, definition.Collections.Length);
Assert.Equal("Users", definition.Collections[0]);
Assert.True(definition.ContainsField("Age"));
Assert.Equal("Auto/Users/ByAgeSortByAge", definition.Name);
var nameField = definition.GetField("Age");
Assert.Equal(SortOptions.NumericDefault, nameField.SortOption);
}
public QueryResultWithIncludes ExecuteDynamicQuery(string entityName, IndexQuery query)
{
// Create the map
var map = DynamicQueryMapping.Create(documentDatabase, query, entityName);
var touchTemporaryIndexResult = GetAppropriateIndexToQuery(entityName, query, map);
map.IndexName = touchTemporaryIndexResult.Item1;
// Re-write the query
string realQuery = map.Items.Aggregate(query.Query,
(current, mapItem) => current.Replace(mapItem.QueryFrom, mapItem.To));
UpdateFieldNamesForSortedFields(query, map);
UpdateFieldsInArray(map, query.FieldsToFetch);
UpdateFieldsInArray(map, query.GroupBy);
return(ExecuteActualQuery(query, map, touchTemporaryIndexResult, realQuery));
}
public void DefinitionExtensionWontDuplicateFields()
{
_sut = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "FirstName:A* LastName:a*",
SortedFields = new[]
{
new SortedField("Count_Range"),
},
});
var existingDefinition = _sut.CreateAutoIndexDefinition();
_sut = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "FirstName:A* AddressId:addresses/1",
SortedFields = new[]
{
new SortedField("Age_Range"),
new SortedField("Count_Range")
},
});
_sut.ExtendMappingBasedOn(existingDefinition);
var definition = _sut.CreateAutoIndexDefinition();
Assert.Equal(1, definition.Collections.Length);
Assert.Equal("Users", definition.Collections[0]);
Assert.True(definition.ContainsField("FirstName"));
Assert.True(definition.ContainsField("LastName"));
Assert.True(definition.ContainsField("AddressId"));
Assert.True(definition.ContainsField("Age"));
Assert.True(definition.ContainsField("Count"));
Assert.Equal("Auto/Users/ByAddressIdAndAgeAndCountAndFirstNameAndLastNameSortByAgeCount", definition.Name);
var ageField = definition.GetField("Age");
Assert.Equal(SortOptions.NumericDefault, ageField.SortOption);
var countField = definition.GetField("Count");
Assert.Equal(SortOptions.NumericDefault, countField.SortOption);
}
public void Failure_match_for_map_reduce_index_having_different_aggregation_function()
{
var definition = new AutoMapReduceIndexDefinition(new[] { "Users" }, new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
MapReduceOperation = FieldMapReduceOperation.Count
},
},
new[]
{
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Location:Poland",
DynamicMapReduceFields = new[]
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Sum
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
private string GetIndexNameFromCollectionAndField(string field)
{
var collection = GetStringQueryString("collection", false);
if (string.IsNullOrEmpty(collection))
{
return(null);
}
var query = new IndexQueryServerSide(new QueryMetadata($"from {collection} select {field}", null, 0));
var dynamicQueryToIndex = new DynamicQueryToIndexMatcher(Database.IndexStore);
var match = dynamicQueryToIndex.Match(DynamicQueryMapping.Create(query));
if (match.MatchType == DynamicQueryMatchType.Complete ||
match.MatchType == DynamicQueryMatchType.CompleteButIdle)
{
return(match.IndexName);
}
throw new IndexDoesNotExistException($"There is no index to answer the following query: from {collection} select {field}");
}
public void Partial_match_if_exact_is_required()
{
using (var db = CreateDocumentDatabase())
{
var mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"from Users
where Name = 'arek'"));
db.IndexStore.CreateIndex(mapping.CreateAutoIndexDefinition()).Wait();
mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"from Users
where exact(Name = 'arek')"));
var matcher = new DynamicQueryToIndexMatcher(db.IndexStore);
var result = matcher.Match(mapping, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
}
}
public void Failure_if_there_is_no_index_for_given_collection()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Companies WHERE Name = 'IBM'"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Failure_if_matching_index_is_disabled_errored_or_has_lot_of_errors()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide {
Query = "Name:Arek"
});
var index = get_index(definition.Name);
index.SetState(IndexState.Disabled);
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetState(IndexState.Error);
result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetPriority(IndexPriority.Normal);
index._indexStorage.UpdateStats(DateTime.UtcNow, new IndexingRunStats()
{
MapAttempts = 1000,
MapErrors = 900
});
result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_for_single_matching_index()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek'"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Partial_match_when_sort_field_is_not_mapped()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' ORDER BY Weight"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Complete_match_for_single_matching_index_with_numeric_sort_option_for_nested_field()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Address.ZipCode",
Storage = FieldStorage.No,
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users ORDER BY Address.ZipCode AS double"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void PartialMatch_for_index_containing_only_part_of_indexes_fields()
{
var usersByName = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(usersByName);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' AND Age = 29"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(usersByName.Name, result.IndexName);
}
public async Task Partial_match_if_analyzer_is_required()
{
Initialize();
using (var db = CreateDocumentDatabase())
{
var mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"from Users
where Name = 'arek'"));
await db.IndexStore.CreateIndex(mapping.CreateAutoIndexDefinition(), Guid.NewGuid().ToString());
mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"from Users
where search(Name, 'arek')"));
var matcher = new DynamicQueryToIndexMatcher(db.IndexStore);
var result = matcher.Match(mapping, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
}
}
public void ExtendsMappingBasedOnExistingDefinition()
{
_sut = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE StartsWith(FirstName, 'a') ORDER BY Count AS long"));
var existingDefinition = _sut.CreateAutoIndexDefinition();
_sut = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE StartsWith(LastName, 'A') ORDER BY Age AS double"));
_sut.ExtendMappingBasedOn(existingDefinition);
var definition = _sut.CreateAutoIndexDefinition();
Assert.Equal(1, definition.Collections.Count);
Assert.Equal("Users", definition.Collections.Single());
Assert.True(definition.ContainsField("FirstName"));
Assert.True(definition.ContainsField("LastName"));
Assert.True(definition.ContainsField("Age"));
Assert.True(definition.ContainsField("Count"));
Assert.Equal("Auto/Users/ByAgeAndCountAndFirstNameAndLastName", definition.Name);
}
private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple <string, bool> touchTemporaryIndexResult, string realQuery)
{
// Perform the query until we have some results at least
QueryResultWithIncludes result;
var sp = Stopwatch.StartNew();
while (true)
{
result = documentDatabase.Query(map.IndexName,
new IndexQuery
{
Cutoff = query.Cutoff,
PageSize = query.PageSize,
Query = realQuery,
Start = query.Start,
FieldsToFetch = query.FieldsToFetch,
GroupBy = query.GroupBy,
AggregationOperation = query.AggregationOperation,
SortedFields = query.SortedFields,
DefaultField = query.DefaultField,
CutoffEtag = query.CutoffEtag,
DebugOptionGetIndexEntries = query.DebugOptionGetIndexEntries,
DefaultOperator = query.DefaultOperator,
SkipTransformResults = query.SkipTransformResults,
SkippedResults = query.SkippedResults,
HighlighterPreTags = query.HighlighterPreTags,
HighlighterPostTags = query.HighlighterPostTags,
HighlightedFields = query.HighlightedFields
});
if (!touchTemporaryIndexResult.Item2 ||
!result.IsStale ||
(result.Results.Count >= query.PageSize && query.PageSize > 0) ||
sp.Elapsed.TotalSeconds > 15)
{
return(result);
}
Thread.Sleep(100);
}
}
public void Failure_if_matching_index_is_disabled_errored_or_has_lot_of_errors()
{
Initialize();
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
AddIndex(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek'"));
var index = GetIndex(definition.Name);
index.SetState(IndexState.Disabled);
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetState(IndexState.Error);
result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetPriority(IndexPriority.Normal);
index._indexStorage.UpdateStats(DateTime.UtcNow, new IndexingRunStats()
{
MapAttempts = 1000,
MapErrors = 900
});
result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_for_index_containing_all_fields()
{
var usersByName = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
var usersByNameAndAge = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
new IndexField
{
Name = "Age",
Highlighted = false,
Storage = FieldStorage.No
}
});
add_index(usersByName);
add_index(usersByNameAndAge);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide {
Query = "Name:Arek Age:29"
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(usersByNameAndAge.Name, result.IndexName);
}
public void Should_match_auto_map_index_if_analyzed_field_isnt_used_in_where()
{
using (var db = CreateDocumentDatabase())
{
var mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"
from LastFms
where search(Artist, ""Chri"") and Genre = ""jazz"""));
db.IndexStore.CreateIndex(mapping.CreateAutoIndexDefinition(), Guid.NewGuid().ToString()).Wait();
mapping = DynamicQueryMapping.Create(new IndexQueryServerSide(@"
from LastFms
where Genre = ""jazz""
select Artist"));
var matcher = new DynamicQueryToIndexMatcher(db.IndexStore);
var result = matcher.Match(mapping, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
}
}
public void Failure_if_there_is_no_index_for_given_collection()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Companies", new IndexQueryServerSide {
Query = "Name:IBM"
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
private QueryResultWithIncludes ExecuteActualQuery(IndexQuery query, DynamicQueryMapping map, Tuple <string, bool> touchTemporaryIndexResult, string realQuery, CancellationToken token)
{
// Perform the query until we have some results at least
QueryResultWithIncludes result;
var sp = Stopwatch.StartNew();
while (true)
{
var indexQuery = CreateIndexQuery(query, map, realQuery);
result = documentDatabase.Queries.Query(map.IndexName, indexQuery, token);
if (!touchTemporaryIndexResult.Item2 ||
!result.IsStale ||
(result.Results.Count >= query.PageSize && query.PageSize > 0) ||
sp.Elapsed.TotalSeconds > 15)
{
return(result);
}
Thread.Sleep(100);
}
}
