public async Task IndexPathsInheritance_DatabaseSpecificSettings()
{
var path1 = NewDataPath();
var path3 = NewDataPath();
using (var store = GetDocumentStore(new Options
{
Path = path1,
ModifyDatabaseRecord = document =>
{
document.Settings[RavenConfiguration.GetKey(x => x.Indexing.TempPath)] = path3;
}
}))
{
var index = new SimpleIndex();
index.Execute(store);
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
Assert.Equal(path1, database.Configuration.Core.DataDirectory.FullPath);
Assert.Equal(path3, database.Configuration.Indexing.TempPath.FullPath);
var indexInstance = database.IndexStore.GetIndex(index.IndexName);
var safeName = IndexDefinitionBaseServerSide.GetIndexNameSafeForFileSystem(indexInstance.Name);
var tempPath = Path.Combine(path3, safeName);
Assert.True(Directory.Exists(tempPath));
await store.Maintenance.SendAsync(new DeleteIndexOperation(index.IndexName));
Assert.False(Directory.Exists(tempPath));
}
}
public CurrentIndexingScope(Index index, DocumentsStorage documentsStorage, QueryOperationContext queryContext, IndexDefinitionBaseServerSide indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool)
{
_documentsStorage = documentsStorage;
QueryContext = queryContext;
Index = index;
UnmanagedBuffersPool = _unmanagedBuffersPool;
IndexDefinition = indexDefinition;
IndexContext = indexContext;
_getSpatialField = getSpatialField;
}
private static Dictionary <string, FieldToFetch> GetFieldsToFetch(
QueryMetadata metadata,
ProjectionBehavior?projectionBehavior,
IndexDefinitionBaseServerSide indexDefinition,
out bool anyExtractableFromIndex,
out bool extractAllStoredFields,
out bool singleFieldNoAlias,
out bool anyTimeSeries)
{
anyExtractableFromIndex = false;
extractAllStoredFields = false;
singleFieldNoAlias = false;
anyTimeSeries = false;
if (metadata.SelectFields == null || metadata.SelectFields.Length == 0)
{
return(null);
}
if (metadata.SelectFields.Length == 1)
{
var selectField = metadata.SelectFields[0];
singleFieldNoAlias = ((selectField.Alias == null && selectField.Function != null) || (selectField.Name == string.Empty && selectField.Function == null));
if (singleFieldNoAlias && metadata.Query.From.Alias != null && metadata.Query.From.Alias == selectField.Alias)
{
return(null); // from 'Index' as doc select doc -> we do not want to treat this as a projection
}
}
var result = new Dictionary <string, FieldToFetch>(StringComparer.Ordinal);
for (var i = 0; i < metadata.SelectFields.Length; i++)
{
var selectField = metadata.SelectFields[i];
var val = GetFieldToFetch(indexDefinition, metadata, projectionBehavior, selectField, result,
out var key, ref anyExtractableFromIndex, ref extractAllStoredFields, ref anyTimeSeries);
if (extractAllStoredFields)
{
return(result);
}
if (val == null)
{
continue;
}
result[key] = val;
}
if (indexDefinition != null)
{
anyExtractableFromIndex |= (indexDefinition.HasDynamicFields && projectionBehavior.FromIndexOrDefault());
}
return(result);
}
private static string GetTreeName(BlittableJsonReaderObject reduceEntry, IndexDefinitionBaseServerSide indexDefinition, JsonOperationContext context)
{
Dictionary <string, CompiledIndexField> groupByFields;
if (indexDefinition is MapReduceIndexDefinition mapReduceIndexDefinition)
{
groupByFields = mapReduceIndexDefinition.GroupByFields;
}
else if (indexDefinition is AutoMapReduceIndexDefinition autoMapReduceIndexDefinition)
{
groupByFields = autoMapReduceIndexDefinition.GroupByFields
.ToDictionary(x => x.Key, x => (CompiledIndexField) new SimpleField(x.Key));
}
else
{
throw new InvalidOperationException("Invalid map reduce index definition: " + indexDefinition.GetType());
}
foreach (var prop in reduceEntry.GetPropertyNames())
{
var skip = false;
foreach (var groupByField in groupByFields.Values)
{
if (groupByField.IsMatch(prop))
{
skip = true;
break;
}
}
if (skip)
{
continue;
}
if (reduceEntry.Modifications == null)
{
reduceEntry.Modifications = new DynamicJsonValue(reduceEntry);
}
reduceEntry.Modifications.Remove(prop);
}
var reduceKey = context.ReadObject(reduceEntry, "debug: creating reduce tree name");
return(reduceKey.ToString());
}
public FieldsToFetch(IndexQueryServerSide query, IndexDefinitionBaseServerSide indexDefinition)
{
Projection = new ProjectionOptions(query);
Fields = GetFieldsToFetch(query.Metadata, query.ProjectionBehavior, indexDefinition, out AnyExtractableFromIndex, out bool extractAllStoredFields, out SingleBodyOrMethodWithNoAlias, out AnyTimeSeries);
IsProjection = Fields != null && Fields.Count > 0;
IndexFields = indexDefinition?.IndexFields;
AnyDynamicIndexFields = indexDefinition != null && indexDefinition.HasDynamicFields;
if (extractAllStoredFields)
{
AnyExtractableFromIndex = true;
ExtractAllFromIndex = true; // we want to add dynamic fields also to the result (stored only)
IsProjection = true;
}
IsDistinct = query.Metadata.IsDistinct && IsProjection;
}
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide other)
{
var otherDefinition = other as AutoMapReduceIndexDefinition;
if (otherDefinition == null)
{
return(IndexDefinitionCompareDifferences.All);
}
if (ReferenceEquals(this, other))
{
return(IndexDefinitionCompareDifferences.None);
}
var result = IndexDefinitionCompareDifferences.None;
if (Collections.SetEquals(otherDefinition.Collections) == false || DictionaryExtensions.ContentEquals(MapFields, otherDefinition.MapFields) == false)
{
result |= IndexDefinitionCompareDifferences.Maps;
}
if (DictionaryExtensions.ContentEquals(GroupByFields, otherDefinition.GroupByFields) == false)
{
result |= IndexDefinitionCompareDifferences.Reduce;
}
if (LockMode != other.LockMode)
{
result |= IndexDefinitionCompareDifferences.LockMode;
}
if (Priority != other.Priority)
{
result |= IndexDefinitionCompareDifferences.Priority;
}
if (State != otherDefinition.State)
{
result |= IndexDefinitionCompareDifferences.State;
}
return(result);
}
public IndexFacetedReadOperation(Index index,
IndexDefinitionBaseServerSide indexDefinition,
LuceneVoronDirectory directory,
IndexSearcherHolder searcherHolder,
QueryBuilderFactories queryBuilderFactories,
Transaction readTransaction,
DocumentDatabase documentDatabase)
: base(index, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name))
{
try
{
_analyzer = CreateAnalyzer(index, indexDefinition, forQuerying: true);
}
catch (Exception e)
{
throw new IndexAnalyzerException(e);
}
_queryBuilderFactories = queryBuilderFactories;
_releaseReadTransaction = directory.SetTransaction(readTransaction, out _state);
_releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher);
}
public async Task IndexPathsInheritance_ServerWideSettings()
{
var path1 = NewDataPath();
var path3 = NewDataPath();
DoNotReuseServer(new Dictionary <string, string>
{
{ RavenConfiguration.GetKey(x => x.Indexing.TempPath), path3 },
});
using (GetNewServer())
{
using (var store = GetDocumentStore(new Options
{
Path = path1
}))
{
var index = new SimpleIndex();
index.Execute(store);
var database = await Databases.GetDocumentDatabaseInstanceFor(store);
Assert.Equal(path1, database.Configuration.Core.DataDirectory.FullPath);
Assert.Equal(Path.Combine(path3, "Databases", store.Database), database.Configuration.Indexing.TempPath.FullPath);
var indexInstance = database.IndexStore.GetIndex(index.IndexName);
var safeName = IndexDefinitionBaseServerSide.GetIndexNameSafeForFileSystem(indexInstance.Name);
var tempPath = Path.Combine(path3, "Databases", store.Database, safeName);
Assert.True(Directory.Exists(tempPath));
await store.Maintenance.SendAsync(new DeleteIndexOperation(index.IndexName));
Assert.False(Directory.Exists(tempPath));
}
}
}
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition)
{
throw new NotImplementedException();
}
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition)
{
return(Definition.Compare(indexDefinition));
}
public override void Update(IndexDefinitionBaseServerSide definition, IndexingConfiguration configuration)
{
throw new NotSupportedException($"{Type} index does not support updating it's definition and configuration.");
}
private FaultyInMemoryIndex(Exception e, IndexingConfiguration configuration, IndexDefinitionBaseServerSide definition)
: base(IndexType.Faulty, IndexSourceType.None, definition)
{
_e = e;
_createdAt = DateTime.UtcNow;
State = IndexState.Error;
Configuration = configuration;
}
private static FieldToFetch GetFieldToFetch(
IndexDefinitionBaseServerSide indexDefinition,
QueryMetadata metadata,
ProjectionBehavior?projectionBehavior,
SelectField selectField,
Dictionary <string, FieldToFetch> results,
out string selectFieldKey,
ref bool anyExtractableFromIndex,
ref bool extractAllStoredFields,
ref bool anyTimeSeries)
{
var mustExtractFromIndex = projectionBehavior.FromIndexOnly();
var maybeExtractFromIndex = projectionBehavior.FromIndexOrDefault();
selectFieldKey = selectField.Alias ?? selectField.Name;
var selectFieldName = selectField.Name;
if (selectField.ValueTokenType != null)
{
return(new FieldToFetch(string.Empty, selectField, selectField.Alias,
canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false));
}
if (selectField.Function != null)
{
var isTimeSeries = metadata.DeclaredFunctions != null && metadata.DeclaredFunctions.TryGetValue(selectField.Function, out var func) && func.Type == DeclaredFunction.FunctionType.TimeSeries;
if (isTimeSeries)
{
anyTimeSeries = true;
}
var fieldToFetch = new FieldToFetch(
selectField.Name,
selectField,
selectField.Alias,
canExtractFromIndex: false,
isDocumentId: false,
isTimeSeries: isTimeSeries)
{
FunctionArgs = new FieldToFetch[selectField.FunctionArgs.Length]
};
for (int j = 0; j < selectField.FunctionArgs.Length; j++)
{
var ignored = false;
fieldToFetch.FunctionArgs[j] = GetFieldToFetch(
indexDefinition,
metadata,
projectionBehavior,
selectField.FunctionArgs[j],
null,
out _,
ref ignored,
ref ignored,
ref ignored
);
}
return(fieldToFetch);
}
if (selectField.IsCounter)
{
var fieldToFetch = new FieldToFetch(selectField.Name, selectField, selectField.Alias ?? selectField.Name,
canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false);
if (selectField.FunctionArgs != null)
{
fieldToFetch.FunctionArgs = new FieldToFetch[0];
}
return(fieldToFetch);
}
if (selectFieldName == null)
{
if (selectField.IsGroupByKey == false)
{
return(null);
}
if (selectField.GroupByKeys.Length == 1)
{
selectFieldName = selectField.GroupByKeys[0].Name;
if (selectFieldKey == null)
{
selectFieldKey = selectFieldName;
}
}
else
{
selectFieldKey = selectFieldKey ?? "Key";
return(new FieldToFetch(selectFieldKey, selectField.GroupByKeyNames));
}
}
if (indexDefinition == null)
{
return(new FieldToFetch(selectFieldName, selectField, selectField.Alias, canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false));
}
if (selectFieldName.Value.Length > 0)
{
if (selectFieldName == Constants.Documents.Indexing.Fields.DocumentIdFieldName)
{
anyExtractableFromIndex = maybeExtractFromIndex;
return(new FieldToFetch(selectFieldName, selectField, selectField.Alias,
canExtractFromIndex: indexDefinition is MapReduceIndexDefinition or AutoMapReduceIndexDefinition,
isDocumentId: true, isTimeSeries: false));
}
if (selectFieldName.Value[0] == '_')
{
if (selectFieldName == Constants.Documents.Indexing.Fields.AllStoredFields)
{
if (results == null)
{
ThrowInvalidFetchAllStoredDocuments();
}
Debug.Assert(results != null);
results.Clear(); // __all_stored_fields should only return stored fields so we are ensuring that no other fields will be returned
extractAllStoredFields = maybeExtractFromIndex;
foreach (var kvp in indexDefinition.MapFields)
{
var stored = kvp.Value.Storage == FieldStorage.Yes;
if (stored == false)
{
continue;
}
anyExtractableFromIndex = maybeExtractFromIndex;
results[kvp.Key] = new FieldToFetch(kvp.Key, null, null, canExtractFromIndex: maybeExtractFromIndex, isDocumentId: false, isTimeSeries: false);
}
return(null);
}
}
}
var bySourceAlias = ShouldTryToExtractBySourceAliasName(selectFieldName.Value, selectField);
var key = bySourceAlias
? selectField.SourceAlias
: selectFieldName;
var extract = mustExtractFromIndex || (maybeExtractFromIndex && indexDefinition.MapFields.TryGetValue(key, out var value) && value.Storage == FieldStorage.Yes);
if (extract)
{
anyExtractableFromIndex = true;
}
if (bySourceAlias == false && maybeExtractFromIndex)
{
extract |= indexDefinition.HasDynamicFields;
}
return(new FieldToFetch(selectFieldName, selectField, selectField.Alias, extract, isDocumentId: false, isTimeSeries: false));
}
public abstract override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition);
protected static RavenPerFieldAnalyzerWrapper CreateAnalyzer(Index index, IndexDefinitionBaseServerSide indexDefinition, bool forQuerying = false)
{
if (indexDefinition.IndexFields.ContainsKey(Constants.Documents.Indexing.Fields.AllFields))
{
throw new InvalidOperationException($"Detected '{Constants.Documents.Indexing.Fields.AllFields}'. This field should not be present here, because inheritance is done elsewhere.");
}
var analyzers = new Dictionary <Type, Analyzer>();
var hasDefaultFieldOptions = false;
Analyzer defaultAnalyzerToUse = null;
Analyzer defaultAnalyzer = null;
if (indexDefinition is MapIndexDefinition mid)
{
if (mid.IndexDefinition.Fields.TryGetValue(Constants.Documents.Indexing.Fields.AllFields, out var value))
{
hasDefaultFieldOptions = true;
switch (value.Indexing)
{
case FieldIndexing.Exact:
defaultAnalyzerToUse = GetOrCreateAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer);
break;
case FieldIndexing.Search:
if (value.Analyzer != null)
{
defaultAnalyzerToUse = GetAnalyzer(Constants.Documents.Indexing.Fields.AllFields, value.Analyzer, analyzers, forQuerying, index.DocumentDatabase.Name);
}
if (defaultAnalyzerToUse == null)
{
defaultAnalyzerToUse = GetOrCreateAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer);
}
break;
default:
// explicitly ignore all other values
break;
}
}
}
if (defaultAnalyzerToUse == null)
{
defaultAnalyzerToUse = defaultAnalyzer = CreateDefaultAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultAnalyzerType.Value.Type);
analyzers.Add(defaultAnalyzerToUse.GetType(), defaultAnalyzerToUse);
}
var perFieldAnalyzerWrapper = forQuerying == false && indexDefinition.HasDynamicFields
? new RavenPerFieldAnalyzerWrapper(
defaultAnalyzerToUse,
fieldName => GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer),
fieldName => GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer))
: new RavenPerFieldAnalyzerWrapper(defaultAnalyzerToUse);
foreach (var field in indexDefinition.IndexFields)
{
var fieldName = field.Value.Name;
switch (field.Value.Indexing)
{
case FieldIndexing.Exact:
var keywordAnalyzer = GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer);
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, keywordAnalyzer);
break;
case FieldIndexing.Search:
var analyzer = GetAnalyzer(fieldName, field.Value.Analyzer, analyzers, forQuerying, index.DocumentDatabase.Name);
if (analyzer != null)
{
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, analyzer);
continue;
}
AddStandardAnalyzer(fieldName);
break;
case FieldIndexing.Default:
if (hasDefaultFieldOptions)
{
// if we have default field options then we need to take into account overrides for regular fields
if (defaultAnalyzer == null)
{
defaultAnalyzer = CreateDefaultAnalyzer(fieldName, index.Configuration.DefaultAnalyzerType.Value.Type);
}
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, defaultAnalyzer);
continue;
}
break;
}
}
return(perFieldAnalyzerWrapper);
void AddStandardAnalyzer(string fieldName)
{
var standardAnalyzer = GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer);
perFieldAnalyzerWrapper.AddAnalyzer(fieldName, standardAnalyzer);
}
Analyzer GetOrCreateAnalyzer(string fieldName, Type analyzerType, Func <string, Type, Analyzer> createAnalyzer)
{
if (analyzers.TryGetValue(analyzerType, out var analyzer) == false)
{
analyzers[analyzerType] = analyzer = createAnalyzer(fieldName, analyzerType);
}
return(analyzer);
}
Analyzer CreateDefaultAnalyzer(string fieldName, Type analyzerType)
{
if (analyzerType == typeof(LowerCaseKeywordAnalyzer))
{
return(new LowerCaseKeywordAnalyzer());
}
return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType));
}
Analyzer CreateKeywordAnalyzer(string fieldName, Type analyzerType)
{
if (analyzerType == typeof(KeywordAnalyzer))
{
return(new KeywordAnalyzer());
}
return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType));
}
Analyzer CreateStandardAnalyzer(string fieldName, Type analyzerType)
{
if (analyzerType == typeof(RavenStandardAnalyzer))
{
return(new RavenStandardAnalyzer(Version.LUCENE_29));
}
return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType));
}
}
public override void Update(IndexDefinitionBaseServerSide definition, IndexingConfiguration configuration)
{
SetLock(definition.LockMode);
SetPriority(definition.Priority);
}
protected void add_index(IndexDefinitionBaseServerSide definition)
{
AsyncHelpers.RunSync(() => _documentDatabase.IndexStore.CreateIndex(definition, Guid.NewGuid().ToString()));
}
