public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var progress = new DatabaseCompactionProgress
{
Message = $"Started database compaction for {_database}"
};
onProgress?.Invoke(progress);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
{
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
progress.ObjectType = progressReport.ObjectType.ToString();
progress.GlobalProgress = progressReport.GlobalProgress;
progress.GlobalTotal = progressReport.GlobalTotal;
progress.ObjectName = progressReport.ObjectName;
progress.ObjectProgress = progressReport.ObjectProgress;
progress.ObjectTotal = progressReport.ObjectTotal;
progress.Message = progressReport.Message;
onProgress?.Invoke(progress);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
}
}
}
return(DatabaseCompactionResult.Instance);
}
public async Task WillNotLoadTwoIndexesWithTheSameId()
{
var index = new Users_ByCity();
string destPath;
var name = Guid.NewGuid().ToString("N");
var path = NewDataPath();
var otherPath = NewDataPath();
using (var store = GetDocumentStore(
path: path,
modifyDatabaseDocument: document => document.Settings[RavenConfiguration.GetKey(x => x.Indexing.AdditionalIndexStoragePaths)] = otherPath,
modifyName: n => name))
{
var indexDefinition1 = index.CreateIndexDefinition();
indexDefinition1.Configuration[RavenConfiguration.GetKey(x => x.Indexing.IndexStoragePath)] = otherPath;
store.DatabaseCommands.PutIndex(index.IndexName + "_1", indexDefinition1);
var indexDefinition2 = index.CreateIndexDefinition();
indexDefinition2.Configuration[RavenConfiguration.GetKey(x => x.Indexing.IndexStoragePath)] = otherPath;
store.DatabaseCommands.PutIndex(index.IndexName + "_2", indexDefinition2);
var database = await GetDocumentDatabaseInstanceFor(store);
destPath = database.Configuration.Indexing.IndexStoragePath;
}
var srcDirectories = Directory.GetDirectories(Path.Combine(otherPath, name));
var srcDirectory1 = new DirectoryInfo(srcDirectories[0]);
var srcDirectory2 = new DirectoryInfo(srcDirectories[1]);
IOExtensions.MoveDirectory(srcDirectory2.FullName, Path.Combine(destPath, srcDirectory1.Name));
using (var store = GetDocumentStore(
path: path,
modifyDatabaseDocument: document =>
{
document.Settings[RavenConfiguration.GetKey(x => x.Indexing.AdditionalIndexStoragePaths)] = otherPath;
document.Settings[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexOrTransformerCouldNotBeOpened)] = "false";
},
modifyName: n => name))
{
var indexNames = store.DatabaseCommands.GetIndexNames(0, 128);
Assert.Equal(1, indexNames.Length);
}
}
public async Task Should_Be_Able_To_Open_Index_Copied_From_Different_Database_When_IndexStartupBehavior_Is_Set_To_ResetAndStart()
{
var options = new Options
{
RunInMemory = false,
ModifyDatabaseRecord = r =>
{
r.Settings[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexCannotBeOpened)] = "false";
r.Settings[RavenConfiguration.GetKey(x => x.Indexing.ErrorIndexStartupBehavior)] = IndexingConfiguration.ErrorIndexStartupBehaviorType.ResetAndStart.ToString();
}
};
using (var store1 = GetDocumentStore(options))
using (var store2 = GetDocumentStore(options))
{
await new Products_ByName().ExecuteAsync(store1);
await new Products_ByName().ExecuteAsync(store2);
var database1 = await Databases.GetDocumentDatabaseInstanceFor(store1);
var database2 = await Databases.GetDocumentDatabaseInstanceFor(store2);
var indexPath1 = database1.IndexStore.GetIndex(new Products_ByName().IndexName).Configuration.StoragePath.FullPath;
var indexPath2 = database2.IndexStore.GetIndex(new Products_ByName().IndexName).Configuration.StoragePath.FullPath;
Server.ServerStore.DatabasesLandlord.UnloadDirectly(store1.Database);
Server.ServerStore.DatabasesLandlord.UnloadDirectly(store2.Database);
database1 = await Databases.GetDocumentDatabaseInstanceFor(store1);
var index1 = database1.IndexStore.GetIndex(new Products_ByName().IndexName);
Assert.IsType(typeof(MapIndex), index1);
Server.ServerStore.DatabasesLandlord.UnloadDirectly(store1.Database);
IOExtensions.DeleteDirectory(indexPath2);
IOExtensions.MoveDirectory(indexPath1, indexPath2);
database2 = await Databases.GetDocumentDatabaseInstanceFor(store2);
var index2 = database2.IndexStore.GetIndex(new Products_ByName().IndexName);
Assert.IsType(typeof(MapIndex), index2);
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
src.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
src.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
src.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
src.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
src.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
src.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out src.MasterKey, documentDatabase.MasterKey);
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
dst.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
dst.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
dst.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
dst.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
dst.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
dst.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out dst.MasterKey, documentDatabase.MasterKey);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
var oldIndexesPath = new PathSetting(basePath + "-old").Combine("Indexes");
var newIndexesPath = new PathSetting(basePath).Combine("Indexes");
IOExtensions.MoveDirectory(oldIndexesPath.FullPath, newIndexesPath.FullPath);
var oldConfigPath = new PathSetting(basePath + "-old").Combine("Configuration");
var newConfigPath = new PathSetting(basePath).Combine("Configuration");
IOExtensions.MoveDirectory(oldConfigPath.FullPath, newConfigPath.FullPath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
_isCompactionInProgress = false;
}
}
}
public bool TryReplaceIndexes(string oldIndexName, string replacementIndexName)
{
bool lockTaken = false;
try
{
Monitor.TryEnter(_locker, 16, ref lockTaken);
if (lockTaken == false)
{
return(false);
}
if (_indexes.TryGetByName(replacementIndexName, out Index newIndex) == false)
{
return(true);
}
if (_indexes.TryGetByName(oldIndexName, out Index oldIndex))
{
oldIndexName = oldIndex.Name;
if (oldIndex.Type.IsStatic() && newIndex.Type.IsStatic())
{
var oldIndexDefinition = oldIndex.GetIndexDefinition();
var newIndexDefinition = newIndex.Definition.GetOrCreateIndexDefinitionInternal();
if (newIndex.Definition.LockMode == IndexLockMode.Unlock && newIndexDefinition.LockMode.HasValue == false && oldIndexDefinition.LockMode.HasValue)
{
newIndex.SetLock(oldIndexDefinition.LockMode.Value);
}
if (newIndex.Definition.Priority == IndexPriority.Normal && newIndexDefinition.Priority.HasValue == false && oldIndexDefinition.Priority.HasValue)
{
newIndex.SetPriority(oldIndexDefinition.Priority.Value);
}
}
}
_indexes.ReplaceIndex(oldIndexName, oldIndex, newIndex);
newIndex.Rename(oldIndexName);
newIndex.ResetIsSideBySideAfterReplacement();
if (oldIndex != null)
{
while (_documentDatabase.DatabaseShutdown.IsCancellationRequested == false)
{
try
{
using (oldIndex.DrainRunningQueries())
DeleteIndexInternal(oldIndex);
break;
}
catch (TimeoutException)
{
}
}
}
if (newIndex.Configuration.RunInMemory == false)
{
while (_documentDatabase.DatabaseShutdown.IsCancellationRequested == false)
{
try
{
using (newIndex.DrainRunningQueries())
using (newIndex.StorageOperation())
{
var oldIndexDirectoryName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(oldIndexName);
var replacementIndexDirectoryName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(replacementIndexName);
IOExtensions.MoveDirectory(newIndex.Configuration.StoragePath.Combine(replacementIndexDirectoryName).FullPath,
newIndex.Configuration.StoragePath.Combine(oldIndexDirectoryName).FullPath);
if (newIndex.Configuration.TempPath != null)
{
IOExtensions.MoveDirectory(newIndex.Configuration.TempPath.Combine(replacementIndexDirectoryName).FullPath,
newIndex.Configuration.TempPath.Combine(oldIndexDirectoryName).FullPath);
}
}
break;
}
catch (TimeoutException)
{
}
}
}
_documentDatabase.Changes.RaiseNotifications(
new IndexChange
{
Name = oldIndexName,
Type = IndexChangeTypes.SideBySideReplace
});
return(true);
}
finally
{
if (lockTaken)
{
Monitor.Exit(_locker);
}
}
}
