public IEnumerable <IndexQueryResult> Query() { using (IndexStorage.EnsureInvariantCulture()) { AssertQueryDoesNotContainFieldsThatAreNotIndexes(); IndexSearcher indexSearcher; using (parent.GetSearcher(out indexSearcher)) { var luceneQuery = GetLuceneQuery(); foreach (var indexQueryTrigger in indexQueryTriggers) { luceneQuery = indexQueryTrigger.Value.ProcessQuery(parent.name, luceneQuery, indexQuery); } int start = indexQuery.Start; int pageSize = indexQuery.PageSize; int returnedResults = 0; int skippedResultsInCurrentLoop = 0; do { if (skippedResultsInCurrentLoop > 0) { start = start + pageSize; // trying to guesstimate how many results we will need to read from the index // to get enough unique documents to match the page size pageSize = skippedResultsInCurrentLoop * indexQuery.PageSize; skippedResultsInCurrentLoop = 0; } TopDocs search = ExecuteQuery(indexSearcher, luceneQuery, start, pageSize, indexQuery); indexQuery.TotalSize.Value = search.TotalHits; RecordResultsAlreadySeenForDistinctQuery(indexSearcher, search, start); for (int i = start; i < search.TotalHits && (i - start) < pageSize; i++) { Document document = indexSearcher.Doc(search.ScoreDocs[i].doc); IndexQueryResult indexQueryResult = parent.RetrieveDocument(document, fieldsToFetch, search.ScoreDocs[i].score); if (ShouldIncludeInResults(indexQueryResult) == false) { indexQuery.SkippedResults.Value++; skippedResultsInCurrentLoop++; continue; } returnedResults++; yield return(indexQueryResult); if (returnedResults == indexQuery.PageSize) { yield break; } } } while (skippedResultsInCurrentLoop > 0 && returnedResults < indexQuery.PageSize); } } }
public DocumentDatabase(RavenConfiguration configuration) { Configuration = configuration; configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers }; TransactionalStorage = configuration.CreateTransactionalStorage(workContext.NotifyAboutWork); configuration.Container.SatisfyImportsOnce(TransactionalStorage); bool newDb; try { newDb = TransactionalStorage.Initialize(); } catch (Exception) { TransactionalStorage.Dispose(); throw; } IndexDefinitionStorage = new IndexDefinitionStorage( TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration); workContext.PerformanceCounters = new PerformanceCounters("Instance @ " + configuration.Port); workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; InitializeTriggers(); ExecuteStartupTasks(); if (!newDb) return; OnNewlyCreatedDatabase(); }
protected override void HandleCommitPoints(IndexedItemsInfo itemsInfo, IndexSegmentsInfo segmentsInfo) { if (ShouldStoreCommitPoint(itemsInfo) && itemsInfo.HighestETag != null) { context.IndexStorage.StoreCommitPoint(indexId.ToString(), new IndexCommitPoint { HighestCommitedETag = itemsInfo.HighestETag, TimeStamp = LastIndexTime, SegmentsInfo = segmentsInfo ?? IndexStorage.GetCurrentSegmentsInfo(indexDefinition.Name, directory) }); LastCommitPointStoreTime = SystemTime.UtcNow; } else if (itemsInfo.DeletedKeys != null && directory is RAMDirectory == false) { context.IndexStorage.AddDeletedKeysToCommitPoints(indexDefinition, itemsInfo.DeletedKeys); } }
public DocumentDatabase(RavenConfiguration configuration) { this.configuration = configuration; workContext = new WorkContext(); TransactionalStorage = new TransactionalStorage(configuration.DataDirectory, workContext.NotifyAboutWork); bool newDb; try { newDb = TransactionalStorage.Initialize(); } catch (Exception) { TransactionalStorage.Dispose(); throw; } IndexDefinitionStorage = new IndexDefinitionStorage(configuration.DataDirectory); IndexStorage = new IndexStorage(configuration.DataDirectory, IndexDefinitionStorage); workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; if (!newDb) return; if(configuration.ShouldCreateDefaultsWhenBuildingNewDatabaseFromScratch) { PutIndex("Raven/DocumentsByEntityName", new IndexDefinition { Map = @" from doc in docs where doc[""@metadata""][""Raven-Entity-Name""] != null select new { Tag = doc[""@metadata""][""Raven-Entity-Name""] }; " }); } configuration.RaiseDatabaseCreatedFromScratch(this); }
public DocumentDatabase(InMemoryRavenConfiguration configuration) { this.configuration = configuration; using (LogManager.OpenMappedContext("database", configuration.DatabaseName ?? Constants.SystemDatabase)) { if (configuration.IsTenantDatabase == false) { validateLicense = new ValidateLicense(); validateLicense.Execute(configuration); } AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit; AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit; Name = configuration.DatabaseName; backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Current; ExtensionsState = new AtomicDictionary<object>(); Configuration = configuration; ExecuteAlterConfiguration(); configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { Database = this, DatabaseName = Name, IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers, RaiseIndexChangeNotification = RaiseNotifications, TaskScheduler = backgroundTaskScheduler, Configuration = configuration }; TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); try { sequentialUuidGenerator = new SequentialUuidGenerator(); TransactionalStorage.Initialize(sequentialUuidGenerator, DocumentCodecs); } catch (Exception) { TransactionalStorage.Dispose(); throw; } try { TransactionalStorage.Batch(actions => sequentialUuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); TransportState = new TransportState(); // Index codecs must be initialized before we try to read an index InitializeIndexCodecTriggers(); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this); CompleteWorkContextSetup(); indexingExecuter = new IndexingExecuter(workContext); InitializeTriggersExceptIndexCodecs(); SecondStageInitialization(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } } }
public DocumentDatabase(InMemoryRavenConfiguration configuration) { ExternalState = new ConcurrentDictionary<string, object>(); Name = configuration.DatabaseName; if (configuration.BackgroundTasksPriority != ThreadPriority.Normal) { backgroundTaskScheduler = new TaskSchedulerWithCustomPriority( // we need a minimum of four task threads - one for indexing dispatch, one for reducing dispatch, one for tasks, one for indexing/reducing ops Math.Max(4, configuration.MaxNumberOfParallelIndexTasks + 2), configuration.BackgroundTasksPriority); } else { backgroundTaskScheduler = TaskScheduler.Current; } ExtensionsState = new ConcurrentDictionary<object, object>(); Configuration = configuration; ExecuteAlterConfiguration(); configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers }; TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); configuration.Container.SatisfyImportsOnce(TransactionalStorage); try { TransactionalStorage.Initialize(this); } catch (Exception) { TransactionalStorage.Dispose(); throw; } TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration); workContext.Configuration = configuration; workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; try { InitializeTriggers(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } }
public IEnumerable <IndexQueryResult> IntersectionQuery() { using (IndexStorage.EnsureInvariantCulture()) { AssertQueryDoesNotContainFieldsThatAreNotIndexes(); IndexSearcher indexSearcher; using (parent.GetSearcher(out indexSearcher)) { var subQueries = indexQuery.Query.Split(new[] { Constants.IntersectSeperator }, StringSplitOptions.RemoveEmptyEntries); if (subQueries.Length <= 1) { throw new InvalidOperationException("Invalid INTRESECT query, must have multiple intersect clauses."); } //Not sure how to select the page size here??? The problem is that only docs in this search can be part //of the final result because we're doing an intersection query (but we might exclude some of them) int pageSizeBestGuess = (indexQuery.Start + indexQuery.PageSize) * 2; int intersectMatches = 0, skippedResultsInCurrentLoop = 0; int previousBaseQueryMatches = 0, currentBaseQueryMatches = 0; var firstSubLuceneQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[0], indexQuery.DefaultField)); //Do the first sub-query in the normal way, so that sorting, filtering etc is accounted for var search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery); currentBaseQueryMatches = search.ScoreDocs.Length; var intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs); do { if (skippedResultsInCurrentLoop > 0) { // We get here because out first attempt didn't get enough docs (after INTERSECTION was calculated) pageSizeBestGuess = pageSizeBestGuess * 2; search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery); previousBaseQueryMatches = currentBaseQueryMatches; currentBaseQueryMatches = search.ScoreDocs.Length; intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs); } for (int i = 1; i < subQueries.Length; i++) { var luceneSubQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[i], indexQuery.DefaultField)); indexSearcher.Search(luceneSubQuery, null, intersectionCollector); } var currentIntersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList(); intersectMatches = currentIntersectResults.Count; skippedResultsInCurrentLoop = pageSizeBestGuess - intersectMatches; } while (intersectMatches < indexQuery.PageSize && //stop if we've got enough results to satisfy the pageSize currentBaseQueryMatches < search.TotalHits && //stop if increasing the page size wouldn't make any difference previousBaseQueryMatches < currentBaseQueryMatches); //stop if increasing the page size didn't result in any more "base query" results var intersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList(); //It's hard to know what to do here, the TotalHits from the base search isn't really the TotalSize, //because it's before the INTERSECTION has been applied, so only some of those results make it out. //Trying to give an accurate answer is going to be too costly, so we aren't going to try. indexQuery.TotalSize.Value = search.TotalHits; indexQuery.SkippedResults.Value = skippedResultsInCurrentLoop; //Using the final set of results in the intersectionCollector int returnedResults = 0; for (int i = indexQuery.Start; i < intersectResults.Count && (i - indexQuery.Start) < pageSizeBestGuess; i++) { Document document = indexSearcher.Doc(intersectResults[i].LuceneId); IndexQueryResult indexQueryResult = parent.RetrieveDocument(document, fieldsToFetch, search.ScoreDocs[i].score); if (ShouldIncludeInResults(indexQueryResult) == false) { indexQuery.SkippedResults.Value++; skippedResultsInCurrentLoop++; continue; } returnedResults++; yield return(indexQueryResult); if (returnedResults == indexQuery.PageSize) { yield break; } } } } }
public DocumentDatabase(InMemoryRavenConfiguration configuration, TransportState transportState = null) { DocumentLock = new PutSerialLock(); this.configuration = configuration; this.transportState = transportState ?? new TransportState(); using (LogManager.OpenMappedContext("database", configuration.DatabaseName ?? Constants.SystemDatabase)) { log.Debug("Start loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase); if (configuration.IsTenantDatabase == false) { validateLicense = new ValidateLicense(); validateLicense.Execute(configuration); } AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit; AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit; Name = configuration.DatabaseName; backgroundTaskScheduler = configuration.CustomTaskScheduler ?? TaskScheduler.Default; ExtensionsState = new AtomicDictionary<object>(); Configuration = configuration; ExecuteAlterConfiguration(); recentTouches = new SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo>(configuration.MaxRecentTouchesToRemember, StringComparer.OrdinalIgnoreCase); configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { Database = this, DatabaseName = Name, IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers, RaiseIndexChangeNotification = RaiseNotifications, TaskScheduler = backgroundTaskScheduler, Configuration = configuration, IndexReaderWarmers = IndexReaderWarmers }; TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); try { sequentialUuidGenerator = new SequentialUuidGenerator(); TransactionalStorage.Initialize(sequentialUuidGenerator, DocumentCodecs); } catch (Exception) { TransactionalStorage.Dispose(); throw; } try { inFlightTransactionalState = TransactionalStorage.GetInFlightTransactionalState(Put, Delete); TransactionalStorage.Batch(actions => sequentialUuidGenerator.EtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); // Index codecs must be initialized before we try to read an index InitializeIndexCodecTriggers(); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this); CompleteWorkContextSetup(); prefetcher = new Prefetcher(workContext); indexingExecuter = new IndexingExecuter(workContext, prefetcher); InitializeTriggersExceptIndexCodecs(); SecondStageInitialization(); ExecuteStartupTasks(); log.Debug("Finish loading the following database: {0}", configuration.DatabaseName ?? Constants.SystemDatabase); } catch (Exception) { Dispose(); throw; } } }
public DocumentDatabase(InMemoryRavenConfiguration configuration) { if (configuration.IsTenantDatabase == false) { validateLicense = new ValidateLicense(); validateLicense.Execute(configuration); } AppDomain.CurrentDomain.DomainUnload += DomainUnloadOrProcessExit; AppDomain.CurrentDomain.ProcessExit += DomainUnloadOrProcessExit; Name = configuration.DatabaseName; if (configuration.CustomTaskScheduler != null) { backgroundTaskScheduler = configuration.CustomTaskScheduler; } else if (configuration.BackgroundTasksPriority != ThreadPriority.Normal) { backgroundTaskScheduler = new TaskSchedulerWithCustomPriority( // we need a minimum of four task threads - one for indexing dispatch, one for reducing dispatch, one for tasks, one for indexing/reducing ops Math.Max(4, configuration.MaxNumberOfParallelIndexTasks + 2), configuration.BackgroundTasksPriority); } else { backgroundTaskScheduler = TaskScheduler.Current; } ExtensionsState = new AtomicDictionary<object>(); Configuration = configuration; ExecuteAlterConfiguration(); configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { DatabaseName = Name, IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers, RaiseIndexChangeNotification = RaiseNotifications, TaskScheduler = backgroundTaskScheduler, Configuration = configuration }; TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); try { TransactionalStorage.Initialize(this, DocumentCodecs); } catch (Exception) { TransactionalStorage.Dispose(); throw; } try { TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); TransportState = new TransportState(); // Index codecs must be initialized before we try to read an index InitializeIndexCodecTriggers(); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration, this); CompleteWorkContextSetup(); indexingExecuter = new IndexingExecuter(workContext); InitializeTriggersExceptIndexCodecs(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } }
public DocumentDatabase(InMemoryRavenConfiguration configuration) { Configuration = configuration; configuration.Container.SatisfyImportsOnce(this); workContext = new WorkContext { IndexUpdateTriggers = IndexUpdateTriggers, ReadTriggers = ReadTriggers }; dynamicQueryRunner = new DynamicQueryRunner(this); suggestionQueryRunner = new SuggestionQueryRunner(this); TransactionalStorage = configuration.CreateTransactionalStorage(workContext.HandleWorkNotifications); configuration.Container.SatisfyImportsOnce(TransactionalStorage); bool newDb; try { newDb = TransactionalStorage.Initialize(this); } catch (Exception) { TransactionalStorage.Dispose(); throw; } TransactionalStorage.Batch(actions => currentEtagBase = actions.General.GetNextIdentityValue("Raven/Etag")); IndexDefinitionStorage = new IndexDefinitionStorage( configuration, TransactionalStorage, configuration.DataDirectory, configuration.Container.GetExportedValues<AbstractViewGenerator>(), Extensions); IndexStorage = new IndexStorage(IndexDefinitionStorage, configuration); workContext.IndexStorage = IndexStorage; workContext.TransactionaStorage = TransactionalStorage; workContext.IndexDefinitionStorage = IndexDefinitionStorage; try { InitializeTriggers(); ExecuteStartupTasks(); } catch (Exception) { Dispose(); throw; } if (!newDb) return; OnNewlyCreatedDatabase(); }