public void Failure_if_there_is_no_index_for_given_collection()
{
Initialize();
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Companies WHERE Name = 'IBM'"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_for_single_matching_index_with_default_string_sort_option()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' ORDER BY Name"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void PartialMatch_for_index_containing_only_part_of_indexes_fields()
{
var usersByName = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(usersByName);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' AND Age = 29"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(usersByName.Name, result.IndexName);
}
public void Complete_match_for_single_matching_index_with_numeric_sort_option_for_nested_field()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Address.ZipCode",
Storage = FieldStorage.No,
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users ORDER BY Address.ZipCode AS double"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Partial_match_when_sort_field_is_not_mapped()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' ORDER BY Weight ASC"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Partial, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Failure_if_matching_index_is_disabled_errored_or_has_lot_of_errors()
{
Initialize();
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek'"));
var index = get_index(definition.Name);
index.SetState(IndexState.Disabled);
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetState(IndexState.Error);
result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
index.SetPriority(IndexPriority.Normal);
index._indexStorage.UpdateStats(DateTime.UtcNow, new IndexingRunStats()
{
MapAttempts = 1000,
MapErrors = 900
});
result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public async Task Can_enable_disabled_index()
{
using (var database = CreateDocumentDatabase())
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
var index = await database.IndexStore.CreateIndex(definition, Guid.NewGuid().ToString());
index.Disable();
index.Enable();
Assert.Equal(IndexState.Normal, index.State);
}
}
public void Complete_match_for_index_containing_all_fields()
{
var usersByName = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
var usersByNameAndAge = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
new IndexField
{
Name = "Age",
Highlighted = false,
Storage = FieldStorage.No
}
});
add_index(usersByName);
add_index(usersByNameAndAge);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide {
Query = "Name:Arek Age:29"
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(usersByNameAndAge.Name, result.IndexName);
}
public void Failure_if_there_is_no_index_for_given_collection()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Companies", new IndexQueryServerSide {
Query = "Name:IBM"
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public static object ReadIndexDefinition(BlittableJsonReaderObject reader, BuildVersionType buildVersionType, out IndexType type)
{
switch (buildVersionType)
{
case BuildVersionType.V3:
// pre 4.0 support
var indexDefinition = ReadLegacyIndexDefinition(reader);
type = indexDefinition.Type;
Debug.Assert(type.IsStatic());
return(indexDefinition);
case BuildVersionType.V4:
case BuildVersionType.V5:
case BuildVersionType.GreaterThanCurrent:
type = ReadIndexType(reader, out reader);
switch (type)
{
case IndexType.AutoMap:
return(AutoMapIndexDefinition.LoadFromJson(reader));
case IndexType.AutoMapReduce:
return(AutoMapReduceIndexDefinition.LoadFromJson(reader));
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
return(JsonDeserializationServer.IndexDefinition(reader));
default:
throw new NotSupportedException(type.ToString());
}
default:
throw new ArgumentOutOfRangeException(nameof(buildVersionType), buildVersionType, null);
}
}
public async Task Can_enable_errored_index()
{
using (var database = CreateDocumentDatabase())
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
var index = await database.IndexStore.CreateIndex(definition);
index.SetState(IndexState.Error); // will also stop the indexing thread
index.Enable();
Assert.Equal(IndexState.Normal, index.State);
}
}
public void Failure_match_for_map_index_containing_the_same_field_names()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
},
new IndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide {
Query = "Location:Poland", DynamicMapReduceFields = new []
{
new DynamicMapReduceField
{
Name = "Count",
OperationType = FieldMapReduceOperation.Count
},
new DynamicMapReduceField
{
Name = "Location",
IsGroupBy = true
}
}
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Complete_match_for_single_matching_index()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Name",
Highlighted = false,
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide {
Query = "Name:Arek"
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Complete_match_query_sort_is_default_and_definition_doesn_not_specify_sorting_at_all()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Age",
Highlighted = false,
Storage = FieldStorage.No,
SortOption = SortOptions.NumericDefault
},
});
add_index(definition);
var dynamicQueryWithStringSorting = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Age_Range:{Lx9 TO NULL}",
SortedFields = new[] { new SortedField("Age_Range") },
});
var result = _sut.Match(dynamicQueryWithStringSorting);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
var dynamicQueryWithNoneSorting = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Age_Range:31",
SortedFields = new[] { new SortedField("Age_Range") },
});
result = _sut.Match(dynamicQueryWithNoneSorting);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(definition.Name, result.IndexName);
}
public void Complete_match_for_index_containing_all_fields()
{
Initialize();
var usersByName = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
});
var usersByNameAndAge = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No
},
new AutoIndexField
{
Name = "Age",
Storage = FieldStorage.No,
}
});
add_index(usersByName);
add_index(usersByNameAndAge);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users WHERE Name = 'Arek' AND Age = 29"));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Complete, result.MatchType);
Assert.Equal(usersByNameAndAge.Name, result.IndexName);
}
public void Failure_when_sort_options_do_not_match()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new IndexField
{
Name = "Weight",
Highlighted = false,
Storage = FieldStorage.No
},
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create("Users", new IndexQueryServerSide
{
Query = "Weight:70",
SortedFields = new[] { new SortedField("Weight_Range") },
});
var result = _sut.Match(dynamicQuery);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
public void Failure_match_for_map_index_containing_the_same_field_names()
{
var definition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Count",
Storage = FieldStorage.Yes,
},
new AutoIndexField
{
Name = "Location",
Storage = FieldStorage.Yes,
}
});
add_index(definition);
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide("FROM Users GROUP BY Location WHERE Location = 'Poland' SELECT Location, count() "));
var result = _sut.Match(dynamicQuery, null);
Assert.Equal(DynamicQueryMatchType.Failure, result.MatchType);
}
private void add_index(AutoMapIndexDefinition definition)
{
AsyncHelpers.RunSync(() => _documentDatabase.IndexStore.CreateIndex(definition));
}
public void AutoIndex_WhenCreateSecondIndexForTwoProperty_ShouldUseOnlyIfItIsMoreUpdatedThenTheFirstIndex()
{
Initialize();
var narrowIndexDefinition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
});
AddIndex(narrowIndexDefinition);
using (var context = DocumentsOperationContext.ShortTermSingleUse(_documentDatabase))
using (var tx = context.OpenWriteTransaction())
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "Grisha",
["Company"] = "Hibernating Rhinos",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
_documentDatabase.DocumentsStorage.Put(context, "users/1", null, doc);
tx.Commit();
}
var narrowIndex = GetIndex(narrowIndexDefinition.Name);
WaitForIndexMap(narrowIndex, 1);
_documentDatabase.IndexStore.StopIndexing();
var extendedIndexDefinition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
new AutoIndexField
{
Name = "Company",
Storage = FieldStorage.No,
},
});
AddIndex(extendedIndexDefinition);
_documentDatabase.IndexStore.RunIdleOperations();
Assert.Equal(2, _documentDatabase.IndexStore.GetIndexes().Count());
_documentDatabase.IndexStore.StartIndexing();
var extendedIndex = GetIndex(extendedIndexDefinition.Name);
WaitForIndexMap(extendedIndex, 1);
_documentDatabase.IndexStore.RunIdleOperations();
Assert.Equal(2, _documentDatabase.IndexStore.GetIndexes().Count());
}
public async Task AutoIndex_WhenCreateSecondIndexForTwoProperty_ShouldUseOnlyIfItIsMoreUpdatedThenTheFirstIndex()
{
Initialize();
var narrowIndexDefinition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
});
AddIndex(narrowIndexDefinition);
using (var context = DocumentsOperationContext.ShortTermSingleUse(_documentDatabase))
using (var tx = context.OpenWriteTransaction())
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "Grisha",
["Company"] = "Hibernating Rhinos",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
_documentDatabase.DocumentsStorage.Put(context, "users/1", null, doc);
tx.Commit();
}
var narrowIndex = GetIndex(narrowIndexDefinition.Name);
WaitForIndexMap(narrowIndex, 1);
_documentDatabase.IndexStore.StopIndexing();
var extendedIndexDefinition = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
new AutoIndexField
{
Name = "Company",
Storage = FieldStorage.No,
},
});
AddIndex(extendedIndexDefinition);
_documentDatabase.IndexStore.RunIdleOperations();
Assert.Equal(2, _documentDatabase.IndexStore.GetIndexes().Count());
_documentDatabase.IndexStore.StartIndexing();
var extendedIndex = GetIndex(extendedIndexDefinition.Name);
WaitForIndexMap(extendedIndex, 1);
{
//Database records needs to change between `RunIdleOperations` calls to apply auto indexes modifications
var addDatabaseCommand = new AddDatabaseCommand(Guid.NewGuid().ToString())
{
Record = _documentDatabase.ReadDatabaseRecord(),
Name = _documentDatabase.Name
};
await _documentDatabase.ServerStore.SendToLeaderAsync(addDatabaseCommand);
}
_documentDatabase.IndexStore.RunIdleOperations();
Assert.Equal(1, _documentDatabase.IndexStore.GetIndexes().Count());
}
public void Choose_the_most_up_to_date_index_including_an_idle_one()
{
Initialize();
var definition1 = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
});
AddIndex(definition1);
using (var context = DocumentsOperationContext.ShortTermSingleUse(_documentDatabase))
using (var tx = context.OpenWriteTransaction())
using (var doc = CreateDocument(context, "users/1", new DynamicJsonValue
{
["Name"] = "Grisha",
["Company"] = "Hibernating Rhinos",
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "Users"
}
}))
{
_documentDatabase.DocumentsStorage.Put(context, "users/1", null, doc);
tx.Commit();
}
var index = GetIndex(definition1.Name);
WaitForIndexMap(index, 1);
var explanations = new List <DynamicQueryToIndexMatcher.Explanation>();
VerifyIndex("FROM Users where Name = 'Grisha'", definition1.Name, DynamicQueryMatchType.Complete);
Assert.Empty(explanations);
_documentDatabase.IndexStore.StopIndexing();
var definition2 = new AutoMapIndexDefinition("Users", new[]
{
new AutoIndexField
{
Name = "Name",
Storage = FieldStorage.No,
},
new AutoIndexField
{
Name = "Company",
Storage = FieldStorage.No,
},
});
AddIndex(definition2);
index.SetState(IndexState.Idle);
VerifyIndex("FROM Users where Name = 'Grisha'", definition1.Name, DynamicQueryMatchType.CompleteButIdle);
Assert.Equal(1, explanations.Count);
Assert.Equal(definition2.Name, explanations[0].Index);
Assert.Equal("Wasn't the most up to date index matching this query", explanations[0].Reason);
_documentDatabase.IndexStore.StartIndexing();
var index2 = GetIndex(definition2.Name);
WaitForIndexMap(index2, 1);
index.SetState(IndexState.Idle);
explanations.Clear();
VerifyIndex("FROM Users where Name = 'Grisha'", definition2.Name, DynamicQueryMatchType.Complete);
Assert.Equal(1, explanations.Count);
Assert.Equal(definition1.Name, explanations[0].Index);
Assert.Equal("The index is idle. The preference is for active indexes - making a complete match but marking the index as idle", explanations[0].Reason);
index.SetState(IndexState.Normal);
index2.SetState(IndexState.Idle);
explanations.Clear();
VerifyIndex("FROM Users where Name = 'Grisha'", definition1.Name, DynamicQueryMatchType.Complete);
Assert.Equal(1, explanations.Count);
Assert.Equal(definition2.Name, explanations[0].Index);
Assert.Equal("The index is idle. The preference is for active indexes - making a complete match but marking the index as idle", explanations[0].Reason);
void VerifyIndex(string query, string expectedIndexName, DynamicQueryMatchType matchType)
{
var dynamicQuery = DynamicQueryMapping.Create(new IndexQueryServerSide(query));
var result = _sut.Match(dynamicQuery, explanations);
Assert.Equal(expectedIndexName, result.IndexName);
Assert.Equal(matchType, result.MatchType);
}
}
private void add_index(AutoMapIndexDefinition definition)
{
AsyncHelpers.RunSync(() => _documentDatabase.IndexStore.CreateIndex(definition, Guid.NewGuid().ToString()));
}
private void add_index(AutoMapIndexDefinition definition)
{
_documentDatabase.IndexStore.CreateIndex(definition);
}
private static AutoIndexDefinitionBase CreateAutoDefinition(AutoIndexDefinition definition)
{
var mapFields = definition
.MapFields
.Select(x =>
{
var field = AutoIndexField.Create(x.Key, x.Value);
field.Aggregation = x.Value.Aggregation;
Debug.Assert(x.Value.GroupByArrayBehavior == GroupByArrayBehavior.NotApplicable);
return(field);
})
.ToArray();
if (definition.Type == IndexType.AutoMap)
{
var result = new AutoMapIndexDefinition(definition.Collection, mapFields);
if (definition.LockMode.HasValue)
{
result.LockMode = definition.LockMode.Value;
}
if (definition.Priority.HasValue)
{
result.Priority = definition.Priority.Value;
}
return(result);
}
if (definition.Type == IndexType.AutoMapReduce)
{
var groupByFields = definition
.GroupByFields
.Select(x =>
{
var field = AutoIndexField.Create(x.Key, x.Value);
field.Aggregation = x.Value.Aggregation;
field.GroupByArrayBehavior = x.Value.GroupByArrayBehavior;
return(field);
})
.ToArray();
var result = new AutoMapReduceIndexDefinition(definition.Collection, mapFields, groupByFields);
if (definition.LockMode.HasValue)
{
result.LockMode = definition.LockMode.Value;
}
if (definition.Priority.HasValue)
{
result.Priority = definition.Priority.Value;
}
return(result);
}
throw new NotSupportedException("Cannot create auto-index from " + definition.Type);
}
