public void get_debug_info_ForSpecifiedDatabase_IndependentBatchSizeAutoTuner() { using (var documentStore = NewDocumentStore()) { documentStore.DatabaseCommands.GlobalAdmin.CreateDatabase(new DatabaseDocument { Id = "Dba1", Settings = { { "Raven/DataDir", "Dba1" } } }); var dbWorkContext = documentStore.ServerIfEmbedded.Options.DatabaseLandlord.GetResourceInternal("Dba1").Result.WorkContext; var dbName = dbWorkContext.DatabaseName; var independentBatchSizeAutoTuner = new IndependentBatchSizeAutoTuner(dbWorkContext, PrefetchingUser.Indexer); var prefetchingBehavior = new PrefetchingBehavior(PrefetchingUser.Indexer, dbWorkContext, independentBatchSizeAutoTuner, string.Empty); independentBatchSizeAutoTuner.NumberOfItemsToProcessInSingleBatch = 200; independentBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue); independentBatchSizeAutoTuner.HandleLowMemory(); MemoryStatistics.RunLowMemoryHandlers("System detected low memory"); prefetchingBehavior.OutOfMemoryExceptionHappened(); prefetchingBehavior.HandleLowMemory(); var url = $"http://localhost:8079/databases/{dbName}/debug/auto-tuning-info"; var requestWithDbName = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get, documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions)); var results = requestWithDbName.ReadResponseJson().JsonDeserialization <AutoTunerInfo>(); var reason = results.Reason; var reasonForLowMemoryCall = results.LowMemoryCallsRecords.First().Reason; var lowMemoryRecords = results.LowMemoryCallsRecords.First().Operations; Assert.Equal("System detected low memory", reasonForLowMemoryCall); var urlAdmin = $"http://localhost:8079/databases/{dbName}/admin/debug/auto-tuning-info"; var requestWithDbNameAdmin = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get, documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions)); var resultsAdmin = requestWithDbNameAdmin.ReadResponseJson().JsonDeserialization <AutoTunerInfo>(); var reasonAdmin = results.Reason; var reasonForLowMemoryCallAdmin = resultsAdmin.LowMemoryCallsRecords.First().Reason; var lowMemoryRecordsAdmin = resultsAdmin.LowMemoryCallsRecords.First().Operations; var cpuUsageRecordsAdmin = resultsAdmin.CpuUsageCallsRecords; Assert.Equal("System detected low memory", reasonForLowMemoryCallAdmin); } }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.GetConfigurationValue<int>("Raven/Replication/ReplicationRequestTimeout") ?? 60 * 1000; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); task.Start(); }
public void get_debug_info_ForAdmin() { using (var documentStore = NewDocumentStore()) { var workContext = documentStore.SystemDatabase.WorkContext; var prefetchingBehavior = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new IndexBatchSizeAutoTuner(workContext), string.Empty); MemoryStatistics.RunLowMemoryHandlers("System detected low memory"); var prefetchingBehavior2 = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new IndependentBatchSizeAutoTuner(workContext, PrefetchingUser.Indexer), string.Empty); var prefetchingBehavior3 = new PrefetchingBehavior(PrefetchingUser.Indexer, workContext, new ReduceBatchSizeAutoTuner(workContext), string.Empty); var indexBatchAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior.PrefetchingUser); var independentBatchSizeAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior2.PrefetchingUser); var reduceBatchSizeAutoTuner = new IndependentBatchSizeAutoTuner(workContext, prefetchingBehavior3.PrefetchingUser); independentBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue); reduceBatchSizeAutoTuner.AutoThrottleBatchSize(500, 1024, TimeSpan.MinValue); indexBatchAutoTuner.HandleLowMemory(); reduceBatchSizeAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue); prefetchingBehavior3.HandleLowMemory(); indexBatchAutoTuner.AutoThrottleBatchSize(100, 1024, TimeSpan.MinValue); prefetchingBehavior2.HandleLowMemory(); prefetchingBehavior.OutOfMemoryExceptionHappened(); prefetchingBehavior.HandleLowMemory(); MemoryStatistics.RunLowMemoryHandlers("System detected low memory"); var url = "http://localhost:8079/admin/debug/auto-tuning-info"; var requestWithDbName = documentStore.JsonRequestFactory.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Get, documentStore.DatabaseCommands.PrimaryCredentials, documentStore.Conventions)); var results = requestWithDbName.ReadResponseJson().JsonDeserialization <AutoTunerInfo>(); var reason = results.Reason; var reasonForLowMemoryCall = results.LowMemoryCallsRecords.First().Reason; var lowMemoryRecords = results.LowMemoryCallsRecords.First().Operations; Assert.Equal("System detected low memory", reasonForLowMemoryCall); } }
public void Execute(DocumentDatabase database) { docDb = database; var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; nonBufferedHttpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs, AllowWriteStreamBuffering = false }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); task.Start(); }
public void Execute(DocumentDatabase database) { docDb = database; docDb.Notifications.OnIndexChange += OnIndexChange; docDb.Notifications.OnTransformerChange += OnTransformerChange; var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); _replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min _lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2); _indexReplicationTaskTimer = database.TimerManager.NewTimer(x => ReplicateIndexesAndTransformersTask(x), TimeSpan.Zero, _replicationFrequency); _lastQueriedTaskTimer = database.TimerManager.NewTimer(SendLastQueriedTask, TimeSpan.Zero, _lastQueriedFrequency); task.Start(); }
public void Execute(DocumentDatabase database) { docDb = database; _transactionalStorageId = docDb.TransactionalStorage.Id.ToString(); var replicationRequestTimeoutInMs = docDb.Configuration.Replication.ReplicationRequestTimeoutInMilliseconds; autoTuner = new IndependentBatchSizeAutoTuner(docDb.WorkContext, PrefetchingUser.Replicator); httpRavenRequestFactory = new HttpRavenRequestFactory { RequestTimeoutInMs = replicationRequestTimeoutInMs }; var task = new Task(Execute, TaskCreationOptions.LongRunning); var disposableAction = new DisposableAction(task.Wait); // make sure that the doc db waits for the replication task shutdown docDb.ExtensionsState.GetOrAdd(Guid.NewGuid().ToString(), s => disposableAction); IndexReplication = new IndexReplicationTask(database, httpRavenRequestFactory, this); TransformerReplication = new TransformerReplicationTask(database, httpRavenRequestFactory, this); task.Start(); IndexReplication.Start(); TransformerReplication.Start(); }