public async Task IndexPathsInheritance_DatabaseSpecificSettings() { var path1 = NewDataPath(); var path3 = NewDataPath(); using (var store = GetDocumentStore(new Options { Path = path1, ModifyDatabaseRecord = document => { document.Settings[RavenConfiguration.GetKey(x => x.Indexing.TempPath)] = path3; } })) { var index = new SimpleIndex(); index.Execute(store); var database = await Databases.GetDocumentDatabaseInstanceFor(store); Assert.Equal(path1, database.Configuration.Core.DataDirectory.FullPath); Assert.Equal(path3, database.Configuration.Indexing.TempPath.FullPath); var indexInstance = database.IndexStore.GetIndex(index.IndexName); var safeName = IndexDefinitionBaseServerSide.GetIndexNameSafeForFileSystem(indexInstance.Name); var tempPath = Path.Combine(path3, safeName); Assert.True(Directory.Exists(tempPath)); await store.Maintenance.SendAsync(new DeleteIndexOperation(index.IndexName)); Assert.False(Directory.Exists(tempPath)); } }
public CurrentIndexingScope(Index index, DocumentsStorage documentsStorage, QueryOperationContext queryContext, IndexDefinitionBaseServerSide indexDefinition, TransactionOperationContext indexContext, Func <string, SpatialField> getSpatialField, UnmanagedBuffersPoolWithLowMemoryHandling _unmanagedBuffersPool) { _documentsStorage = documentsStorage; QueryContext = queryContext; Index = index; UnmanagedBuffersPool = _unmanagedBuffersPool; IndexDefinition = indexDefinition; IndexContext = indexContext; _getSpatialField = getSpatialField; }
private static Dictionary <string, FieldToFetch> GetFieldsToFetch( QueryMetadata metadata, ProjectionBehavior?projectionBehavior, IndexDefinitionBaseServerSide indexDefinition, out bool anyExtractableFromIndex, out bool extractAllStoredFields, out bool singleFieldNoAlias, out bool anyTimeSeries) { anyExtractableFromIndex = false; extractAllStoredFields = false; singleFieldNoAlias = false; anyTimeSeries = false; if (metadata.SelectFields == null || metadata.SelectFields.Length == 0) { return(null); } if (metadata.SelectFields.Length == 1) { var selectField = metadata.SelectFields[0]; singleFieldNoAlias = ((selectField.Alias == null && selectField.Function != null) || (selectField.Name == string.Empty && selectField.Function == null)); if (singleFieldNoAlias && metadata.Query.From.Alias != null && metadata.Query.From.Alias == selectField.Alias) { return(null); // from 'Index' as doc select doc -> we do not want to treat this as a projection } } var result = new Dictionary <string, FieldToFetch>(StringComparer.Ordinal); for (var i = 0; i < metadata.SelectFields.Length; i++) { var selectField = metadata.SelectFields[i]; var val = GetFieldToFetch(indexDefinition, metadata, projectionBehavior, selectField, result, out var key, ref anyExtractableFromIndex, ref extractAllStoredFields, ref anyTimeSeries); if (extractAllStoredFields) { return(result); } if (val == null) { continue; } result[key] = val; } if (indexDefinition != null) { anyExtractableFromIndex |= (indexDefinition.HasDynamicFields && projectionBehavior.FromIndexOrDefault()); } return(result); }
private static string GetTreeName(BlittableJsonReaderObject reduceEntry, IndexDefinitionBaseServerSide indexDefinition, JsonOperationContext context) { Dictionary <string, CompiledIndexField> groupByFields; if (indexDefinition is MapReduceIndexDefinition mapReduceIndexDefinition) { groupByFields = mapReduceIndexDefinition.GroupByFields; } else if (indexDefinition is AutoMapReduceIndexDefinition autoMapReduceIndexDefinition) { groupByFields = autoMapReduceIndexDefinition.GroupByFields .ToDictionary(x => x.Key, x => (CompiledIndexField) new SimpleField(x.Key)); } else { throw new InvalidOperationException("Invalid map reduce index definition: " + indexDefinition.GetType()); } foreach (var prop in reduceEntry.GetPropertyNames()) { var skip = false; foreach (var groupByField in groupByFields.Values) { if (groupByField.IsMatch(prop)) { skip = true; break; } } if (skip) { continue; } if (reduceEntry.Modifications == null) { reduceEntry.Modifications = new DynamicJsonValue(reduceEntry); } reduceEntry.Modifications.Remove(prop); } var reduceKey = context.ReadObject(reduceEntry, "debug: creating reduce tree name"); return(reduceKey.ToString()); }
public FieldsToFetch(IndexQueryServerSide query, IndexDefinitionBaseServerSide indexDefinition) { Projection = new ProjectionOptions(query); Fields = GetFieldsToFetch(query.Metadata, query.ProjectionBehavior, indexDefinition, out AnyExtractableFromIndex, out bool extractAllStoredFields, out SingleBodyOrMethodWithNoAlias, out AnyTimeSeries); IsProjection = Fields != null && Fields.Count > 0; IndexFields = indexDefinition?.IndexFields; AnyDynamicIndexFields = indexDefinition != null && indexDefinition.HasDynamicFields; if (extractAllStoredFields) { AnyExtractableFromIndex = true; ExtractAllFromIndex = true; // we want to add dynamic fields also to the result (stored only) IsProjection = true; } IsDistinct = query.Metadata.IsDistinct && IsProjection; }
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide other) { var otherDefinition = other as AutoMapReduceIndexDefinition; if (otherDefinition == null) { return(IndexDefinitionCompareDifferences.All); } if (ReferenceEquals(this, other)) { return(IndexDefinitionCompareDifferences.None); } var result = IndexDefinitionCompareDifferences.None; if (Collections.SetEquals(otherDefinition.Collections) == false || DictionaryExtensions.ContentEquals(MapFields, otherDefinition.MapFields) == false) { result |= IndexDefinitionCompareDifferences.Maps; } if (DictionaryExtensions.ContentEquals(GroupByFields, otherDefinition.GroupByFields) == false) { result |= IndexDefinitionCompareDifferences.Reduce; } if (LockMode != other.LockMode) { result |= IndexDefinitionCompareDifferences.LockMode; } if (Priority != other.Priority) { result |= IndexDefinitionCompareDifferences.Priority; } if (State != otherDefinition.State) { result |= IndexDefinitionCompareDifferences.State; } return(result); }
public IndexFacetedReadOperation(Index index, IndexDefinitionBaseServerSide indexDefinition, LuceneVoronDirectory directory, IndexSearcherHolder searcherHolder, QueryBuilderFactories queryBuilderFactories, Transaction readTransaction, DocumentDatabase documentDatabase) : base(index, LoggingSource.Instance.GetLogger <IndexFacetedReadOperation>(documentDatabase.Name)) { try { _analyzer = CreateAnalyzer(index, indexDefinition, forQuerying: true); } catch (Exception e) { throw new IndexAnalyzerException(e); } _queryBuilderFactories = queryBuilderFactories; _releaseReadTransaction = directory.SetTransaction(readTransaction, out _state); _releaseSearcher = searcherHolder.GetSearcher(readTransaction, _state, out _searcher); }
public async Task IndexPathsInheritance_ServerWideSettings() { var path1 = NewDataPath(); var path3 = NewDataPath(); DoNotReuseServer(new Dictionary <string, string> { { RavenConfiguration.GetKey(x => x.Indexing.TempPath), path3 }, }); using (GetNewServer()) { using (var store = GetDocumentStore(new Options { Path = path1 })) { var index = new SimpleIndex(); index.Execute(store); var database = await Databases.GetDocumentDatabaseInstanceFor(store); Assert.Equal(path1, database.Configuration.Core.DataDirectory.FullPath); Assert.Equal(Path.Combine(path3, "Databases", store.Database), database.Configuration.Indexing.TempPath.FullPath); var indexInstance = database.IndexStore.GetIndex(index.IndexName); var safeName = IndexDefinitionBaseServerSide.GetIndexNameSafeForFileSystem(indexInstance.Name); var tempPath = Path.Combine(path3, "Databases", store.Database, safeName); Assert.True(Directory.Exists(tempPath)); await store.Maintenance.SendAsync(new DeleteIndexOperation(index.IndexName)); Assert.False(Directory.Exists(tempPath)); } } }
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition) { throw new NotImplementedException(); }
public override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition) { return(Definition.Compare(indexDefinition)); }
public override void Update(IndexDefinitionBaseServerSide definition, IndexingConfiguration configuration) { throw new NotSupportedException($"{Type} index does not support updating it's definition and configuration."); }
private FaultyInMemoryIndex(Exception e, IndexingConfiguration configuration, IndexDefinitionBaseServerSide definition) : base(IndexType.Faulty, IndexSourceType.None, definition) { _e = e; _createdAt = DateTime.UtcNow; State = IndexState.Error; Configuration = configuration; }
private static FieldToFetch GetFieldToFetch( IndexDefinitionBaseServerSide indexDefinition, QueryMetadata metadata, ProjectionBehavior?projectionBehavior, SelectField selectField, Dictionary <string, FieldToFetch> results, out string selectFieldKey, ref bool anyExtractableFromIndex, ref bool extractAllStoredFields, ref bool anyTimeSeries) { var mustExtractFromIndex = projectionBehavior.FromIndexOnly(); var maybeExtractFromIndex = projectionBehavior.FromIndexOrDefault(); selectFieldKey = selectField.Alias ?? selectField.Name; var selectFieldName = selectField.Name; if (selectField.ValueTokenType != null) { return(new FieldToFetch(string.Empty, selectField, selectField.Alias, canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false)); } if (selectField.Function != null) { var isTimeSeries = metadata.DeclaredFunctions != null && metadata.DeclaredFunctions.TryGetValue(selectField.Function, out var func) && func.Type == DeclaredFunction.FunctionType.TimeSeries; if (isTimeSeries) { anyTimeSeries = true; } var fieldToFetch = new FieldToFetch( selectField.Name, selectField, selectField.Alias, canExtractFromIndex: false, isDocumentId: false, isTimeSeries: isTimeSeries) { FunctionArgs = new FieldToFetch[selectField.FunctionArgs.Length] }; for (int j = 0; j < selectField.FunctionArgs.Length; j++) { var ignored = false; fieldToFetch.FunctionArgs[j] = GetFieldToFetch( indexDefinition, metadata, projectionBehavior, selectField.FunctionArgs[j], null, out _, ref ignored, ref ignored, ref ignored ); } return(fieldToFetch); } if (selectField.IsCounter) { var fieldToFetch = new FieldToFetch(selectField.Name, selectField, selectField.Alias ?? selectField.Name, canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false); if (selectField.FunctionArgs != null) { fieldToFetch.FunctionArgs = new FieldToFetch[0]; } return(fieldToFetch); } if (selectFieldName == null) { if (selectField.IsGroupByKey == false) { return(null); } if (selectField.GroupByKeys.Length == 1) { selectFieldName = selectField.GroupByKeys[0].Name; if (selectFieldKey == null) { selectFieldKey = selectFieldName; } } else { selectFieldKey = selectFieldKey ?? "Key"; return(new FieldToFetch(selectFieldKey, selectField.GroupByKeyNames)); } } if (indexDefinition == null) { return(new FieldToFetch(selectFieldName, selectField, selectField.Alias, canExtractFromIndex: false, isDocumentId: false, isTimeSeries: false)); } if (selectFieldName.Value.Length > 0) { if (selectFieldName == Constants.Documents.Indexing.Fields.DocumentIdFieldName) { anyExtractableFromIndex = maybeExtractFromIndex; return(new FieldToFetch(selectFieldName, selectField, selectField.Alias, canExtractFromIndex: indexDefinition is MapReduceIndexDefinition or AutoMapReduceIndexDefinition, isDocumentId: true, isTimeSeries: false)); } if (selectFieldName.Value[0] == '_') { if (selectFieldName == Constants.Documents.Indexing.Fields.AllStoredFields) { if (results == null) { ThrowInvalidFetchAllStoredDocuments(); } Debug.Assert(results != null); results.Clear(); // __all_stored_fields should only return stored fields so we are ensuring that no other fields will be returned extractAllStoredFields = maybeExtractFromIndex; foreach (var kvp in indexDefinition.MapFields) { var stored = kvp.Value.Storage == FieldStorage.Yes; if (stored == false) { continue; } anyExtractableFromIndex = maybeExtractFromIndex; results[kvp.Key] = new FieldToFetch(kvp.Key, null, null, canExtractFromIndex: maybeExtractFromIndex, isDocumentId: false, isTimeSeries: false); } return(null); } } } var bySourceAlias = ShouldTryToExtractBySourceAliasName(selectFieldName.Value, selectField); var key = bySourceAlias ? selectField.SourceAlias : selectFieldName; var extract = mustExtractFromIndex || (maybeExtractFromIndex && indexDefinition.MapFields.TryGetValue(key, out var value) && value.Storage == FieldStorage.Yes); if (extract) { anyExtractableFromIndex = true; } if (bySourceAlias == false && maybeExtractFromIndex) { extract |= indexDefinition.HasDynamicFields; } return(new FieldToFetch(selectFieldName, selectField, selectField.Alias, extract, isDocumentId: false, isTimeSeries: false)); }
public abstract override IndexDefinitionCompareDifferences Compare(IndexDefinitionBaseServerSide indexDefinition);
protected static RavenPerFieldAnalyzerWrapper CreateAnalyzer(Index index, IndexDefinitionBaseServerSide indexDefinition, bool forQuerying = false) { if (indexDefinition.IndexFields.ContainsKey(Constants.Documents.Indexing.Fields.AllFields)) { throw new InvalidOperationException($"Detected '{Constants.Documents.Indexing.Fields.AllFields}'. This field should not be present here, because inheritance is done elsewhere."); } var analyzers = new Dictionary <Type, Analyzer>(); var hasDefaultFieldOptions = false; Analyzer defaultAnalyzerToUse = null; Analyzer defaultAnalyzer = null; if (indexDefinition is MapIndexDefinition mid) { if (mid.IndexDefinition.Fields.TryGetValue(Constants.Documents.Indexing.Fields.AllFields, out var value)) { hasDefaultFieldOptions = true; switch (value.Indexing) { case FieldIndexing.Exact: defaultAnalyzerToUse = GetOrCreateAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer); break; case FieldIndexing.Search: if (value.Analyzer != null) { defaultAnalyzerToUse = GetAnalyzer(Constants.Documents.Indexing.Fields.AllFields, value.Analyzer, analyzers, forQuerying, index.DocumentDatabase.Name); } if (defaultAnalyzerToUse == null) { defaultAnalyzerToUse = GetOrCreateAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer); } break; default: // explicitly ignore all other values break; } } } if (defaultAnalyzerToUse == null) { defaultAnalyzerToUse = defaultAnalyzer = CreateDefaultAnalyzer(Constants.Documents.Indexing.Fields.AllFields, index.Configuration.DefaultAnalyzerType.Value.Type); analyzers.Add(defaultAnalyzerToUse.GetType(), defaultAnalyzerToUse); } var perFieldAnalyzerWrapper = forQuerying == false && indexDefinition.HasDynamicFields ? new RavenPerFieldAnalyzerWrapper( defaultAnalyzerToUse, fieldName => GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer), fieldName => GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer)) : new RavenPerFieldAnalyzerWrapper(defaultAnalyzerToUse); foreach (var field in indexDefinition.IndexFields) { var fieldName = field.Value.Name; switch (field.Value.Indexing) { case FieldIndexing.Exact: var keywordAnalyzer = GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultExactAnalyzerType.Value.Type, CreateKeywordAnalyzer); perFieldAnalyzerWrapper.AddAnalyzer(fieldName, keywordAnalyzer); break; case FieldIndexing.Search: var analyzer = GetAnalyzer(fieldName, field.Value.Analyzer, analyzers, forQuerying, index.DocumentDatabase.Name); if (analyzer != null) { perFieldAnalyzerWrapper.AddAnalyzer(fieldName, analyzer); continue; } AddStandardAnalyzer(fieldName); break; case FieldIndexing.Default: if (hasDefaultFieldOptions) { // if we have default field options then we need to take into account overrides for regular fields if (defaultAnalyzer == null) { defaultAnalyzer = CreateDefaultAnalyzer(fieldName, index.Configuration.DefaultAnalyzerType.Value.Type); } perFieldAnalyzerWrapper.AddAnalyzer(fieldName, defaultAnalyzer); continue; } break; } } return(perFieldAnalyzerWrapper); void AddStandardAnalyzer(string fieldName) { var standardAnalyzer = GetOrCreateAnalyzer(fieldName, index.Configuration.DefaultSearchAnalyzerType.Value.Type, CreateStandardAnalyzer); perFieldAnalyzerWrapper.AddAnalyzer(fieldName, standardAnalyzer); } Analyzer GetOrCreateAnalyzer(string fieldName, Type analyzerType, Func <string, Type, Analyzer> createAnalyzer) { if (analyzers.TryGetValue(analyzerType, out var analyzer) == false) { analyzers[analyzerType] = analyzer = createAnalyzer(fieldName, analyzerType); } return(analyzer); } Analyzer CreateDefaultAnalyzer(string fieldName, Type analyzerType) { if (analyzerType == typeof(LowerCaseKeywordAnalyzer)) { return(new LowerCaseKeywordAnalyzer()); } return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType)); } Analyzer CreateKeywordAnalyzer(string fieldName, Type analyzerType) { if (analyzerType == typeof(KeywordAnalyzer)) { return(new KeywordAnalyzer()); } return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType)); } Analyzer CreateStandardAnalyzer(string fieldName, Type analyzerType) { if (analyzerType == typeof(RavenStandardAnalyzer)) { return(new RavenStandardAnalyzer(Version.LUCENE_29)); } return(IndexingExtensions.CreateAnalyzerInstance(fieldName, analyzerType)); } }
public override void Update(IndexDefinitionBaseServerSide definition, IndexingConfiguration configuration) { SetLock(definition.LockMode); SetPriority(definition.Priority); }
protected void add_index(IndexDefinitionBaseServerSide definition) { AsyncHelpers.RunSync(() => _documentDatabase.IndexStore.CreateIndex(definition, Guid.NewGuid().ToString())); }