public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress) { var progress = new DatabaseCompactionProgress { Message = $"Started database compaction for {_database}" }; onProgress?.Invoke(progress); using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database)) { var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database); using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(), new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception)))) { var basePath = configuration.Core.DataDirectory.FullPath; IOExtensions.DeleteDirectory(basePath + "-Compacting"); IOExtensions.DeleteDirectory(basePath + "-old"); try { configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting"); using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(), new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception)))) { _token.ThrowIfCancellationRequested(); StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport => { progress.ObjectType = progressReport.ObjectType.ToString(); progress.GlobalProgress = progressReport.GlobalProgress; progress.GlobalTotal = progressReport.GlobalTotal; progress.ObjectName = progressReport.ObjectName; progress.ObjectProgress = progressReport.ObjectProgress; progress.ObjectTotal = progressReport.ObjectTotal; progress.Message = progressReport.Message; onProgress?.Invoke(progress); }, _token); } _token.ThrowIfCancellationRequested(); IOExtensions.MoveDirectory(basePath, basePath + "-old"); IOExtensions.MoveDirectory(basePath + "-Compacting", basePath); } catch (Exception e) { throw new InvalidOperationException($"Failed to execute compaction for {_database}", e); } finally { IOExtensions.DeleteDirectory(basePath + "-Compacting"); IOExtensions.DeleteDirectory(basePath + "-old"); } } } return(DatabaseCompactionResult.Instance); }
public async Task WillNotLoadTwoIndexesWithTheSameId() { var index = new Users_ByCity(); string destPath; var name = Guid.NewGuid().ToString("N"); var path = NewDataPath(); var otherPath = NewDataPath(); using (var store = GetDocumentStore( path: path, modifyDatabaseDocument: document => document.Settings[RavenConfiguration.GetKey(x => x.Indexing.AdditionalIndexStoragePaths)] = otherPath, modifyName: n => name)) { var indexDefinition1 = index.CreateIndexDefinition(); indexDefinition1.Configuration[RavenConfiguration.GetKey(x => x.Indexing.IndexStoragePath)] = otherPath; store.DatabaseCommands.PutIndex(index.IndexName + "_1", indexDefinition1); var indexDefinition2 = index.CreateIndexDefinition(); indexDefinition2.Configuration[RavenConfiguration.GetKey(x => x.Indexing.IndexStoragePath)] = otherPath; store.DatabaseCommands.PutIndex(index.IndexName + "_2", indexDefinition2); var database = await GetDocumentDatabaseInstanceFor(store); destPath = database.Configuration.Indexing.IndexStoragePath; } var srcDirectories = Directory.GetDirectories(Path.Combine(otherPath, name)); var srcDirectory1 = new DirectoryInfo(srcDirectories[0]); var srcDirectory2 = new DirectoryInfo(srcDirectories[1]); IOExtensions.MoveDirectory(srcDirectory2.FullName, Path.Combine(destPath, srcDirectory1.Name)); using (var store = GetDocumentStore( path: path, modifyDatabaseDocument: document => { document.Settings[RavenConfiguration.GetKey(x => x.Indexing.AdditionalIndexStoragePaths)] = otherPath; document.Settings[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexOrTransformerCouldNotBeOpened)] = "false"; }, modifyName: n => name)) { var indexNames = store.DatabaseCommands.GetIndexNames(0, 128); Assert.Equal(1, indexNames.Length); } }
public async Task Should_Be_Able_To_Open_Index_Copied_From_Different_Database_When_IndexStartupBehavior_Is_Set_To_ResetAndStart() { var options = new Options { RunInMemory = false, ModifyDatabaseRecord = r => { r.Settings[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexCannotBeOpened)] = "false"; r.Settings[RavenConfiguration.GetKey(x => x.Indexing.ErrorIndexStartupBehavior)] = IndexingConfiguration.ErrorIndexStartupBehaviorType.ResetAndStart.ToString(); } }; using (var store1 = GetDocumentStore(options)) using (var store2 = GetDocumentStore(options)) { await new Products_ByName().ExecuteAsync(store1); await new Products_ByName().ExecuteAsync(store2); var database1 = await Databases.GetDocumentDatabaseInstanceFor(store1); var database2 = await Databases.GetDocumentDatabaseInstanceFor(store2); var indexPath1 = database1.IndexStore.GetIndex(new Products_ByName().IndexName).Configuration.StoragePath.FullPath; var indexPath2 = database2.IndexStore.GetIndex(new Products_ByName().IndexName).Configuration.StoragePath.FullPath; Server.ServerStore.DatabasesLandlord.UnloadDirectly(store1.Database); Server.ServerStore.DatabasesLandlord.UnloadDirectly(store2.Database); database1 = await Databases.GetDocumentDatabaseInstanceFor(store1); var index1 = database1.IndexStore.GetIndex(new Products_ByName().IndexName); Assert.IsType(typeof(MapIndex), index1); Server.ServerStore.DatabasesLandlord.UnloadDirectly(store1.Database); IOExtensions.DeleteDirectory(indexPath2); IOExtensions.MoveDirectory(indexPath1, indexPath2); database2 = await Databases.GetDocumentDatabaseInstanceFor(store2); var index2 = database2.IndexStore.GetIndex(new Products_ByName().IndexName); Assert.IsType(typeof(MapIndex), index2); } }
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result) { if (_isCompactionInProgress) { throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress."); } result.AddMessage($"Started database compaction for {_database}"); onProgress?.Invoke(result); _isCompactionInProgress = true; var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database); var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database); using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database)) using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(), new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception)))) { src.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager; src.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError; src.OnRecoveryError += documentDatabase.HandleOnRecoveryError; src.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes); src.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds; src.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive; Sodium.CloneKey(out src.MasterKey, documentDatabase.MasterKey); var basePath = configuration.Core.DataDirectory.FullPath; IOExtensions.DeleteDirectory(basePath + "-Compacting"); IOExtensions.DeleteDirectory(basePath + "-old"); try { configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting"); using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(), new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception)))) { dst.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError; dst.OnRecoveryError += documentDatabase.HandleOnRecoveryError; dst.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes); dst.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager; dst.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds; dst.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive; Sodium.CloneKey(out dst.MasterKey, documentDatabase.MasterKey); _token.ThrowIfCancellationRequested(); StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport => { result.Progress.TreeProgress = progressReport.TreeProgress; result.Progress.TreeTotal = progressReport.TreeTotal; result.Progress.TreeName = progressReport.TreeName; result.AddMessage(progressReport.Message); onProgress?.Invoke(result); }, _token); } _token.ThrowIfCancellationRequested(); IOExtensions.MoveDirectory(basePath, basePath + "-old"); IOExtensions.MoveDirectory(basePath + "-Compacting", basePath); var oldIndexesPath = new PathSetting(basePath + "-old").Combine("Indexes"); var newIndexesPath = new PathSetting(basePath).Combine("Indexes"); IOExtensions.MoveDirectory(oldIndexesPath.FullPath, newIndexesPath.FullPath); var oldConfigPath = new PathSetting(basePath + "-old").Combine("Configuration"); var newConfigPath = new PathSetting(basePath).Combine("Configuration"); IOExtensions.MoveDirectory(oldConfigPath.FullPath, newConfigPath.FullPath); } catch (Exception e) { throw new InvalidOperationException($"Failed to execute compaction for {_database}", e); } finally { IOExtensions.DeleteDirectory(basePath + "-Compacting"); IOExtensions.DeleteDirectory(basePath + "-old"); _isCompactionInProgress = false; } } }
public bool TryReplaceIndexes(string oldIndexName, string replacementIndexName) { bool lockTaken = false; try { Monitor.TryEnter(_locker, 16, ref lockTaken); if (lockTaken == false) { return(false); } if (_indexes.TryGetByName(replacementIndexName, out Index newIndex) == false) { return(true); } if (_indexes.TryGetByName(oldIndexName, out Index oldIndex)) { oldIndexName = oldIndex.Name; if (oldIndex.Type.IsStatic() && newIndex.Type.IsStatic()) { var oldIndexDefinition = oldIndex.GetIndexDefinition(); var newIndexDefinition = newIndex.Definition.GetOrCreateIndexDefinitionInternal(); if (newIndex.Definition.LockMode == IndexLockMode.Unlock && newIndexDefinition.LockMode.HasValue == false && oldIndexDefinition.LockMode.HasValue) { newIndex.SetLock(oldIndexDefinition.LockMode.Value); } if (newIndex.Definition.Priority == IndexPriority.Normal && newIndexDefinition.Priority.HasValue == false && oldIndexDefinition.Priority.HasValue) { newIndex.SetPriority(oldIndexDefinition.Priority.Value); } } } _indexes.ReplaceIndex(oldIndexName, oldIndex, newIndex); newIndex.Rename(oldIndexName); newIndex.ResetIsSideBySideAfterReplacement(); if (oldIndex != null) { while (_documentDatabase.DatabaseShutdown.IsCancellationRequested == false) { try { using (oldIndex.DrainRunningQueries()) DeleteIndexInternal(oldIndex); break; } catch (TimeoutException) { } } } if (newIndex.Configuration.RunInMemory == false) { while (_documentDatabase.DatabaseShutdown.IsCancellationRequested == false) { try { using (newIndex.DrainRunningQueries()) using (newIndex.StorageOperation()) { var oldIndexDirectoryName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(oldIndexName); var replacementIndexDirectoryName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(replacementIndexName); IOExtensions.MoveDirectory(newIndex.Configuration.StoragePath.Combine(replacementIndexDirectoryName).FullPath, newIndex.Configuration.StoragePath.Combine(oldIndexDirectoryName).FullPath); if (newIndex.Configuration.TempPath != null) { IOExtensions.MoveDirectory(newIndex.Configuration.TempPath.Combine(replacementIndexDirectoryName).FullPath, newIndex.Configuration.TempPath.Combine(oldIndexDirectoryName).FullPath); } } break; } catch (TimeoutException) { } } } _documentDatabase.Changes.RaiseNotifications( new IndexChange { Name = oldIndexName, Type = IndexChangeTypes.SideBySideReplace }); return(true); } finally { if (lockTaken) { Monitor.Exit(_locker); } } }