public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var progress = new DatabaseCompactionProgress
{
Message = $"Started database compaction for {_database}"
};
onProgress?.Invoke(progress);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
{
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
progress.ObjectType = progressReport.ObjectType.ToString();
progress.GlobalProgress = progressReport.GlobalProgress;
progress.GlobalTotal = progressReport.GlobalTotal;
progress.ObjectName = progressReport.ObjectName;
progress.ObjectProgress = progressReport.ObjectProgress;
progress.ObjectTotal = progressReport.ObjectTotal;
progress.Message = progressReport.Message;
onProgress?.Invoke(progress);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
}
}
}
return(DatabaseCompactionResult.Instance);
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification((endId, path, exception) => throw new InvalidOperationException($"Failed to compact database {_database} ({path})", exception))))
{
InitializeOptions(src, configuration, documentDatabase);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification((envId, path, exception) => throw new InvalidOperationException($"Failed to compact database {_database} ({path})", exception))))
{
InitializeOptions(dst, configuration, documentDatabase);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
}
_isCompactionInProgress = false;
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result.Progress);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
string compactTempDirectory = null;
byte[] encryptionKey = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
DatabaseRecord databaseRecord = documentDatabase.ReadDatabaseRecord();
// save the key before unloading the database (it is zeroed when disposing DocumentDatabase).
if (documentDatabase.MasterKey != null)
{
encryptionKey = documentDatabase.MasterKey.ToArray();
}
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((endId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}), StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(src, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
if (configuration.Storage.TempPath != null)
{
compactTempDirectory = configuration.Storage.TempPath.FullPath + "-temp-compacting";
EnsureDirectoriesPermission(compactTempDirectory);
IOExtensions.DeleteDirectory(compactTempDirectory);
configuration.Storage.TempPath = new PathSetting(compactTempDirectory);
}
var revisionsPrefix = CollectionName.GetTablePrefix(CollectionTableType.Revisions);
var compressedCollectionsTableNames = databaseRecord.DocumentsCompression?.Collections
.Select(name => new CollectionName(name).GetTableName(CollectionTableType.Documents))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((envId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}). StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(dst, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result.Progress);
}, (name, schema) =>
{
bool isRevision = name.StartsWith(revisionsPrefix, StringComparison.OrdinalIgnoreCase);
schema.Compressed =
(isRevision && databaseRecord.DocumentsCompression?.CompressRevisions == true) ||
compressedCollectionsTableNames?.Contains(name) == true;
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
if (compactTempDirectory != null)
{
IOExtensions.DeleteDirectory(compactTempDirectory);
}
}
_isCompactionInProgress = false;
if (encryptionKey != null)
{
Sodium.ZeroBuffer(encryptionKey);
}
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
src.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
src.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
src.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
src.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
src.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
src.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out src.MasterKey, documentDatabase.MasterKey);
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
dst.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
dst.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
dst.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
dst.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
dst.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
dst.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out dst.MasterKey, documentDatabase.MasterKey);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
var oldIndexesPath = new PathSetting(basePath + "-old").Combine("Indexes");
var newIndexesPath = new PathSetting(basePath).Combine("Indexes");
IOExtensions.MoveDirectory(oldIndexesPath.FullPath, newIndexesPath.FullPath);
var oldConfigPath = new PathSetting(basePath + "-old").Combine("Configuration");
var newConfigPath = new PathSetting(basePath).Combine("Configuration");
IOExtensions.MoveDirectory(oldConfigPath.FullPath, newConfigPath.FullPath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
_isCompactionInProgress = false;
}
}
}