public void ShouldDeleteCurrentJournalEvenThoughItHasAvailableSpace()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
tx.Commit();
}
}
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
var compactedDir = new DirectoryInfo(CompactedData);
var journalsAfterCompaction = compactedDir.GetFiles("*.journal").Select(x => x.Name).ToList();
Assert.Equal(0, journalsAfterCompaction.Count);
// ensure it can write more data
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactedData)))
{
using (var tx = compacted.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = compacted.CreateTree(tx, "fruits");
tree.Add("peach", new byte[144]);
}
}
}
public static string Encrypt(string srcDir)
{
var masterKey = Sodium.GenerateRandomBuffer((int)Sodium.crypto_aead_xchacha20poly1305_ietf_keybytes());
var dstDir = Path.Combine(Path.GetDirectoryName(srcDir), "Temp.Encryption");
var srcOptions = StorageEnvironmentOptions.ForPath(srcDir);
var dstOptions = StorageEnvironmentOptions.ForPath(dstDir);
dstOptions.Encryption.MasterKey = masterKey;
var protect = new SecretProtection(new SecurityConfiguration()).Protect(masterKey);
StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dstOptions);
using (var f = File.OpenWrite(Path.Combine(dstDir, SecretKeyEncrypted)))
{
f.Write(protect, 0, protect.Length);
f.Flush();
}
IOExtensions.DeleteDirectory(srcDir);
Directory.Move(dstDir, srcDir);
return($"Encrypt: {Path.Combine(dstDir, SecretKeyEncrypted)} Created Successfully");
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var progress = new DatabaseCompactionProgress
{
Message = $"Started database compaction for {_database}"
};
onProgress?.Invoke(progress);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
{
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
progress.ObjectType = progressReport.ObjectType.ToString();
progress.GlobalProgress = progressReport.GlobalProgress;
progress.GlobalTotal = progressReport.GlobalTotal;
progress.ObjectName = progressReport.ObjectName;
progress.ObjectProgress = progressReport.ObjectProgress;
progress.ObjectTotal = progressReport.ObjectTotal;
progress.Message = progressReport.Message;
onProgress?.Invoke(progress);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
}
}
}
return(DatabaseCompactionResult.Instance);
}
public void ShouldReportProgress()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
var tree2 = tx.CreateTree("vegetables");
tree2.Add("carrot", new byte[123]);
tree2.Add("potato", new byte[99]);
var tree3 = tx.CreateTree("multi");
tree3.MultiAdd("fruits", "apple");
tree3.MultiAdd("fruits", "orange");
tree3.MultiAdd("vegetables", "carrot");
tree3.MultiAdd("vegetables", "carrot");
tx.Commit();
}
}
var progressReport = new List <string>();
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "Compacted")),
x => progressReport.Add($"{x.Message} ({x.TreeName} - {x.TreeProgress}/{x.TreeTotal}). Copied {x.GlobalProgress} of {x.GlobalTotal} trees."));
Assert.NotEmpty(progressReport);
var lines = new[]
{
"Copying variable size tree ($Database-Metadata - 0/2). Copied 0 of 4 trees.",
"Copied variable size tree ($Database-Metadata - 2/2). Copied 1 of 4 trees.",
"Copying variable size tree (fruits - 0/2). Copied 1 of 4 trees.",
"Copied variable size tree (fruits - 2/2). Copied 2 of 4 trees.",
"Copying variable size tree (multi - 0/2). Copied 2 of 4 trees.",
"Copied variable size tree (multi - 2/2). Copied 3 of 4 trees.",
"Copying variable size tree (vegetables - 0/2). Copied 3 of 4 trees.",
"Copied variable size tree (vegetables - 2/2). Copied 4 of 4 trees."
};
foreach (var line in lines)
{
Assert.Contains(line, progressReport);
}
}
public void ShouldReportProgress()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
var tree2 = tx.CreateTree("vegetables");
tree2.Add("carrot", new byte[123]);
tree2.Add("potato", new byte[99]);
var tree3 = tx.CreateTree("multi");
tree3.MultiAdd("fruits", "apple");
tree3.MultiAdd("fruits", "orange");
tree3.MultiAdd("vegetables", "carrot");
tree3.MultiAdd("vegetables", "carrot");
tx.Commit();
}
}
var progressReport = new List <string>();
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "Compacted")),
x => progressReport.Add($"Copied {x.ObjectProgress} of {x.ObjectTotal} records in '{x.ObjectName}' tree. Copied {x.GlobalProgress} of {x.GlobalTotal} trees."));
Assert.NotEmpty(progressReport);
var lines = new[]
{
"Copied 0 of 2 records in '$Database-Metadata' tree. Copied 0 of 4 trees.",
"Copied 2 of 2 records in '$Database-Metadata' tree. Copied 1 of 4 trees.",
"Copied 0 of 2 records in 'fruits' tree. Copied 1 of 4 trees.",
"Copied 2 of 2 records in 'fruits' tree. Copied 2 of 4 trees.",
"Copied 0 of 2 records in 'multi' tree. Copied 2 of 4 trees.",
"Copied 2 of 2 records in 'multi' tree. Copied 3 of 4 trees.",
"Copied 0 of 2 records in 'vegetables' tree. Copied 3 of 4 trees.",
"Copied 2 of 2 records in 'vegetables' tree. Copied 4 of 4 trees."
};
foreach (var line in lines)
{
Assert.Contains(line, lines);
}
}
public void ShouldOccupyLessSpace(int seed)
{
var r = new Random(seed);
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(DataDir);
storageEnvironmentOptions.ManualFlushing = true;
using (var env = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("records");
for (int i = 0; i < 100; i++)
{
var bytes = new byte[r.Next(10, 2 * 1024 * 1024)];
r.NextBytes(bytes);
tree.Add("record/" + i, bytes);
}
tx.Commit();
}
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("records");
for (int i = 0; i < 50; i++)
{
tree.Delete("record/" + r.Next(0, 100));
}
tx.Commit();
}
env.FlushLogToDataFile();
}
var oldSize = GetDirSize(new DirectoryInfo(DataDir));
storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(DataDir);
storageEnvironmentOptions.ManualFlushing = true;
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(storageEnvironmentOptions,
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactedData));
var newSize = GetDirSize(new DirectoryInfo(compactedData));
Assert.True(newSize < oldSize, string.Format("Old size: {0:#,#;;0} MB, new size {1:#,#;;0} MB", oldSize / 1024 / 1024, newSize / 1024 / 1024));
}
public void Streams_RavenDB_6510(int fooSize, int barSize, int seed)
{
var r = new Random(seed);
var fooBytes = new byte[fooSize];
var barBytes = new byte[barSize];
r.NextBytes(fooBytes);
r.NextBytes(barBytes);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("streams");
tree.AddStream("foo", new MemoryStream(fooBytes), "t4g");
tree.AddStream("bar", new MemoryStream(barBytes));
tx.Commit();
}
}
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(compactedData)))
{
using (var tx = compacted.WriteTransaction())
{
var tree = tx.ReadTree("streams");
var fooStream = tree.ReadStream("foo");
Assert.Equal(fooSize, fooStream.Length);
Assert.Equal(fooBytes, fooStream.ReadData());
Assert.Equal("t4g", tree.GetStreamTag("foo"));
var barStream = tree.ReadStream("bar");
Assert.Equal(barSize, barStream.Length);
Assert.Equal(barBytes, barStream.ReadData());
Assert.Null(tree.GetStreamTag("bar"));
}
}
}
public static string Decrypt(string srcDir)
{
var dstDir = Path.Combine(Path.GetDirectoryName(srcDir), "Temp.Decryption");
var bytes = File.ReadAllBytes(Path.Combine(srcDir, SecretKeyEncrypted));
var srcOptions = StorageEnvironmentOptions.ForPath(srcDir);
var dstOptions = StorageEnvironmentOptions.ForPath(dstDir);
srcOptions.Encryption.MasterKey = new SecretProtection(new SecurityConfiguration()).Unprotect(bytes);
StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dstOptions);
IOExtensions.DeleteDirectory(srcDir);
Directory.Move(dstDir, srcDir);
return("Decrypt: Completed Successfully");
}
public void ShouldOccupyLessSpace()
{
var r = new Random();
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 100; i++)
{
var bytes = new byte[r.Next(10, 2 * 1024 * 1024)];
r.NextBytes(bytes);
tree.Add("record/" + i, bytes);
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 50; i++)
{
tree.Delete("record/" + r.Next(0, 100));
}
tx.Commit();
}
}
var oldSize = GetDirSize(new DirectoryInfo(CompactionTestsData));
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
var newSize = GetDirSize(new DirectoryInfo(CompactedData));
Assert.True(newSize < oldSize, string.Format("Old size: {0:#,#;;0} MB, new size {1:#,#;;0} MB", oldSize / 1024 / 1024, newSize / 1024 / 1024));
}
public void ShouldReportProgress()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
var tree2 = env.CreateTree(tx, "vegetables");
tree2.Add("carrot", new byte[123]);
tree2.Add("potato", new byte[99]);
var tree3 = env.CreateTree(tx, "multi");
tree3.MultiAdd("fruits", "apple");
tree3.MultiAdd("fruits", "orange");
tree3.MultiAdd("vegetables", "carrot");
tree3.MultiAdd("vegetables", "carrot");
tx.Commit();
}
}
var progressReport = new List <string>();
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData), x => progressReport.Add(string.Format("Copied {0} of {1} records in '{2}' tree. Copied {3} of {4} trees.", x.CopiedTreeRecords, x.TotalTreeRecordsCount, x.TreeName, x.CopiedTrees, x.TotalTreeCount)));
Assert.NotEmpty(progressReport);
Assert.Contains("Copied 0 of 2 records in 'fruits' tree. Copied 0 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'fruits' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'multi' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'multi' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'vegetables' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'vegetables' tree. Copied 3 of 3 trees.", progressReport);
}
public static string Decrypt(string srcDir)
{
var dstDir = Path.Combine(Path.GetDirectoryName(srcDir), "Temp.Decryption");
var bytes = File.ReadAllBytes(Path.Combine(srcDir, SecretKeyEncrypted));
var secret = new byte[bytes.Length - 32];
var entropy = new byte[32];
Array.Copy(bytes, 0, secret, 0, bytes.Length - 32);
Array.Copy(bytes, bytes.Length - 32, entropy, 0, 32);
var srcOptions = StorageEnvironmentOptions.ForPath(srcDir);
var dstOptions = StorageEnvironmentOptions.ForPath(dstDir);
srcOptions.MasterKey = new SecretProtection(new SecurityConfiguration()).Unprotect(secret, entropy);
StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dstOptions);
IOExtensions.DeleteDirectory(srcDir);
Directory.Move(dstDir, srcDir);
return($"Decrypt: {Path.Combine(dstDir, SecretKeyEncrypted)} Created Successfully");
}
public static string GetKey(string srcDir)
{
var masterKey = Sodium.GenerateMasterKey();
var dstDir = Path.Combine(Path.GetDirectoryName(srcDir), "Temp.Encryption");
var srcOptions = StorageEnvironmentOptions.ForPath(srcDir);
var dstOptions = StorageEnvironmentOptions.ForPath(dstDir);
dstOptions.MasterKey = masterKey;
var entropy = Sodium.GenerateRandomBuffer(256);
var protect = new SecretProtection(new SecurityConfiguration()).Protect(masterKey, entropy);
StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dstOptions);
using (var f = File.OpenWrite(Path.Combine(dstDir, SecretKeyEncrypted)))
{
f.Write(protect, 0, protect.Length);
f.Write(entropy, 0, entropy.Length);
f.Flush();
}
return($"GetKey: {Path.Combine(dstDir, SecretKeyEncrypted)} Created Successfully");
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
src.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
src.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
src.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
src.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
src.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
src.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out src.MasterKey, documentDatabase.MasterKey);
var basePath = configuration.Core.DataDirectory.FullPath;
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
try
{
configuration.Core.DataDirectory = new PathSetting(basePath + "-Compacting");
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification(exception => throw new InvalidOperationException($"Failed to compact database {_database}", exception))))
{
dst.OnNonDurableFileSystemError += documentDatabase.HandleNonDurableFileSystemError;
dst.OnRecoveryError += documentDatabase.HandleOnRecoveryError;
dst.CompressTxAboveSizeInBytes = configuration.Storage.CompressTxAboveSize.GetValue(SizeUnit.Bytes);
dst.ForceUsing32BitsPager = configuration.Storage.ForceUsing32BitsPager;
dst.TimeToSyncAfterFlashInSec = (int)configuration.Storage.TimeToSyncAfterFlash.AsTimeSpan.TotalSeconds;
dst.NumOfConcurrentSyncsPerPhysDrive = configuration.Storage.NumberOfConcurrentSyncsPerPhysicalDrive;
Sodium.CloneKey(out dst.MasterKey, documentDatabase.MasterKey);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
_token.ThrowIfCancellationRequested();
IOExtensions.MoveDirectory(basePath, basePath + "-old");
IOExtensions.MoveDirectory(basePath + "-Compacting", basePath);
var oldIndexesPath = new PathSetting(basePath + "-old").Combine("Indexes");
var newIndexesPath = new PathSetting(basePath).Combine("Indexes");
IOExtensions.MoveDirectory(oldIndexesPath.FullPath, newIndexesPath.FullPath);
var oldConfigPath = new PathSetting(basePath + "-old").Combine("Configuration");
var newConfigPath = new PathSetting(basePath).Combine("Configuration");
IOExtensions.MoveDirectory(oldConfigPath.FullPath, newConfigPath.FullPath);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(basePath + "-Compacting");
IOExtensions.DeleteDirectory(basePath + "-old");
_isCompactionInProgress = false;
}
}
}
public void CompactionMustNotLooseAnyData()
{
var treeNames = new List <string>();
var multiValueTreeNames = new List <string>();
var random = new Random();
var value1 = new byte[random.Next(1024 * 1024 * 2)];
var value2 = new byte[random.Next(1024 * 1024 * 2)];
random.NextBytes(value1);
random.NextBytes(value2);
const int treeCount = 5;
const int recordCount = 6;
const int multiValueTreeCount = 7;
const int multiValueRecordsCount = 4;
const int multiValuesCount = 3;
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
for (int i = 0; i < treeCount; i++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
string name = "tree/" + i;
treeNames.Add(name);
var tree = env.State.GetTree(tx, name);
for (int j = 0; j < recordCount; j++)
{
tree.Add(string.Format("{0}/items/{1}", name, j), j % 2 == 0 ? value1 : value2);
}
tx.Commit();
}
}
for (int i = 0; i < multiValueTreeCount; i++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var name = "multiValueTree/" + i;
multiValueTreeNames.Add(name);
var tree = env.CreateTree(tx, name);
for (int j = 0; j < multiValueRecordsCount; j++)
{
for (int k = 0; k < multiValuesCount; k++)
{
tree.MultiAdd("record/" + j, "value/" + k);
}
}
tx.Commit();
}
}
}
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactedData)))
{
using (var tx = compacted.NewTransaction(TransactionFlags.Read))
{
foreach (var treeName in treeNames)
{
var tree = compacted.State.GetTree(tx, treeName);
for (int i = 0; i < recordCount; i++)
{
var readResult = tree.Read(string.Format("{0}/items/{1}", treeName, i));
Assert.NotNull(readResult);
if (i % 2 == 0)
{
var readBytes = new byte[value1.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
Assert.Equal(value1, readBytes);
}
else
{
var readBytes = new byte[value2.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
Assert.Equal(value2, readBytes);
}
}
}
foreach (var treeName in multiValueTreeNames)
{
var tree = compacted.State.GetTree(tx, treeName);
for (int i = 0; i < multiValueRecordsCount; i++)
{
var multiRead = tree.MultiRead("record/" + i);
Assert.True(multiRead.Seek(Slice.BeforeAllKeys));
int count = 0;
do
{
Assert.Equal("value/" + count, multiRead.CurrentKey.ToString());
count++;
} while (multiRead.MoveNext());
Assert.Equal(multiValuesCount, count);
}
}
}
}
}
public unsafe void ShouldPreserveTables(int entries, int seed)
{
// Create random docs to check everything is preserved
using (var allocator = new ByteStringContext(SharedMultipleUseFlag.None))
{
var create = new Dictionary <Slice, long>();
var delete = new List <Slice>();
var r = new Random(seed);
for (var i = 0; i < entries; i++)
{
Slice key;
Slice.From(allocator, "test" + i, out key);
create.Add(key, r.Next());
if (r.NextDouble() < 0.5)
{
delete.Add(key);
}
}
// Create the schema
var schema = new TableSchema()
.DefineKey(new TableSchema.SchemaIndexDef
{
StartIndex = 0,
Count = 1,
IsGlobal = false
});
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
// Create table in the environment
using (var tx = env.WriteTransaction())
{
schema.Create(tx, "test", 16);
var table = tx.OpenTable(schema, "test");
foreach (var entry in create)
{
var value = entry.Value;
table.Set(new TableValueBuilder
{
entry.Key,
value
});
}
tx.Commit();
}
using (var tx = env.ReadTransaction())
{
var table = tx.OpenTable(schema, "test");
Assert.Equal(table.NumberOfEntries, entries);
}
// Delete some of the entries (this is so that compaction makes sense)
using (var tx = env.WriteTransaction())
{
var table = tx.OpenTable(schema, "test");
foreach (var entry in delete)
{
table.DeleteByKey(entry);
}
tx.Commit();
}
}
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)
StorageEnvironmentOptions.ForPath(compactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(compactedData)))
{
using (var tx = compacted.ReadTransaction())
{
var table = tx.OpenTable(schema, "test");
foreach (var entry in create)
{
TableValueReader reader;
var hasValue = table.ReadByKey(entry.Key, out reader);
if (delete.Contains(entry.Key))
{
// This key should not be here
Assert.False(hasValue);
}
else
{
// This key should be there
Assert.True(hasValue);
// Data should be the same
int size;
byte *ptr = reader.Read(0, out size);
Slice current;
using (Slice.External(allocator, ptr, size, out current))
Assert.True(SliceComparer.Equals(current, entry.Key));
ptr = reader.Read(1, out size);
Assert.Equal(entry.Value, *(long *)ptr);
}
}
tx.Commit();
}
}
}
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification((endId, path, exception) => throw new InvalidOperationException($"Failed to compact database {_database} ({path})", exception))))
{
InitializeOptions(src, configuration, documentDatabase);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications(),
new CatastrophicFailureNotification((envId, path, exception) => throw new InvalidOperationException($"Failed to compact database {_database} ({path})", exception))))
{
InitializeOptions(dst, configuration, documentDatabase);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result);
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
}
_isCompactionInProgress = false;
}
}
public void Compressed_tree_RavenDB_6510(int iterationCount, int size, int seed)
{
var r = new Random(seed);
var bytes = new byte[size];
r.NextBytes(bytes);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
using (var tx = env.WriteTransaction())
{
tx.CreateTree("tree", flags: TreeFlags.LeafsCompressed);
tx.Commit();
}
using (var tx = env.WriteTransaction())
{
var tree = tx.ReadTree("tree");
for (var i = 0; i < iterationCount; i++)
{
tree.Add(i.ToString(), new MemoryStream(bytes));
}
tx.Commit();
}
}
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(compactedData)))
{
using (var tx = compacted.WriteTransaction())
{
var tree = tx.ReadTree("tree");
Assert.True(tree.IsLeafCompressionSupported);
for (var i = 0; i < iterationCount; i++)
{
Slice key;
using (Slice.From(tx.Allocator, i.ToString(), ByteStringType.Immutable, out key))
{
using (var readResult = tree.ReadDecompressed(key))
{
if (readResult == null)
{
}
Assert.NotNull(readResult);
var result = readResult.Reader.ReadBytes(readResult.Reader.Length).ToArray();
Assert.Equal(bytes, result);
}
}
}
}
}
}
public void CannotCompactStorageIfIncrementalBackupEnabled()
{
var envOptions = StorageEnvironmentOptions.ForPath(DataDir);
envOptions.IncrementalBackupEnabled = true;
using (var env = new StorageEnvironment(envOptions))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("records");
tree.Add("record/1", new byte[9]);
tree.Add("record/2", new byte[9]);
tx.Commit();
}
}
var srcOptions = StorageEnvironmentOptions.ForPath(DataDir);
srcOptions.IncrementalBackupEnabled = true;
var invalidOperationException = Assert.Throws <InvalidOperationException>(() => StorageCompaction.Execute(srcOptions,
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(Path.Combine(DataDir, "Compacted"))));
Assert.Equal(StorageCompaction.CannotCompactBecauseOfIncrementalBackup, invalidOperationException.Message);
}
public void Can_compact_fixed_size_tree_stored_inside_variable_size_tree(int count)
{
RequireFileBasedPager();
var bytes = new byte[48];
Slice.From(Allocator, "main-tree", out Slice mainTreeId);
Slice.From(Allocator, "fst-tree", out Slice fstTreeIdreeId);
var smallValue = new byte[] { 1, 2, 3 };
var bigValue = new byte[128];
for (int i = 0; i < 128; i++)
{
bigValue[i] = (byte)i;
}
using (var tx = Env.WriteTransaction())
{
var mainTree = tx.CreateTree(mainTreeId);
var fst = mainTree.FixedTreeFor(fstTreeIdreeId, valSize: 48);
for (int i = 0; i < count; i++)
{
EndianBitConverter.Little.CopyBytes(i, bytes, 0);
fst.Add(i, bytes);
Slice read;
using (fst.Read(i, out read))
{
Assert.True(read.HasValue);
}
}
mainTree.Add("small", smallValue);
mainTree.Add("big", bigValue);
tx.Commit();
}
Env.Dispose();
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(DataDir),
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(compactedData)))
{
using (var tx = compacted.ReadTransaction())
{
var mainTree = tx.CreateTree(mainTreeId);
var fst = mainTree.FixedTreeFor(fstTreeIdreeId, valSize: 48);
for (int i = 0; i < count; i++)
{
Assert.True(fst.Contains(i), $"at {i}");
Slice read;
using (fst.Read(i, out read))
{
read.CopyTo(bytes);
Assert.Equal(i, EndianBitConverter.Little.ToInt32(bytes, 0));
}
}
var readResult = mainTree.Read("small");
Assert.Equal(smallValue, readResult.Reader.AsStream().ReadData());
readResult = mainTree.Read("big");
Assert.Equal(bigValue, readResult.Reader.AsStream().ReadData());
}
}
}
public void Compact(InMemoryRavenConfiguration ravenConfiguration, Action <string> output)
{
if (ravenConfiguration.RunInMemory)
{
throw new InvalidOperationException("Cannot compact in-memory running Voron storage");
}
tableStorage.Dispose();
var sourcePath = ravenConfiguration.DataDirectory;
var compactPath = Path.Combine(ravenConfiguration.DataDirectory, "Voron.Compaction");
if (Directory.Exists(compactPath))
{
Directory.Delete(compactPath, true);
}
RecoverFromFailedCompact(sourcePath);
var sourceOptions = CreateStorageOptionsFromConfiguration(ravenConfiguration);
var compactOptions = (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactPath);
output("Executing storage compaction");
StorageCompaction.Execute(sourceOptions, compactOptions,
x => output(string.Format("Copied {0} of {1} records in '{2}' tree. Copied {3} of {4} trees.", x.CopiedTreeRecords, x.TotalTreeRecordsCount, x.TreeName, x.CopiedTrees, x.TotalTreeCount)));
var sourceDir = new DirectoryInfo(sourcePath);
var sourceFiles = new List <FileInfo>();
foreach (var pattern in new [] { "*.journal", "headers.one", "headers.two", VoronConstants.DatabaseFilename })
{
sourceFiles.AddRange(sourceDir.GetFiles(pattern));
}
var compactionBackup = Path.Combine(sourcePath, "Voron.Compaction.Backup");
if (Directory.Exists(compactionBackup))
{
Directory.Delete(compactionBackup, true);
output("Removing existing compaction backup directory");
}
Directory.CreateDirectory(compactionBackup);
output("Backing up original data files");
foreach (var file in sourceFiles)
{
File.Move(file.FullName, Path.Combine(compactionBackup, file.Name));
}
var compactedFiles = new DirectoryInfo(compactPath).GetFiles();
output("Moving compacted files into target location");
foreach (var file in compactedFiles)
{
File.Move(file.FullName, Path.Combine(sourcePath, file.Name));
}
output("Deleting original data backup");
Directory.Delete(compactionBackup, true);
Directory.Delete(compactPath, true);
}
public async Task Execute(Action <IOperationProgress> onProgress, CompactionResult result)
{
if (_isCompactionInProgress)
{
throw new InvalidOperationException($"Database '{_database}' cannot be compacted because compaction is already in progress.");
}
result.AddMessage($"Started database compaction for {_database}");
onProgress?.Invoke(result.Progress);
_isCompactionInProgress = true;
bool done = false;
string compactDirectory = null;
string tmpDirectory = null;
string compactTempDirectory = null;
byte[] encryptionKey = null;
try
{
var documentDatabase = await _serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(_database);
var configuration = _serverStore.DatabasesLandlord.CreateDatabaseConfiguration(_database);
DatabaseRecord databaseRecord = documentDatabase.ReadDatabaseRecord();
// save the key before unloading the database (it is zeroed when disposing DocumentDatabase).
if (documentDatabase.MasterKey != null)
{
encryptionKey = documentDatabase.MasterKey.ToArray();
}
using (await _serverStore.DatabasesLandlord.UnloadAndLockDatabase(_database, "it is being compacted"))
using (var src = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((endId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}), StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(src, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(src, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
var basePath = configuration.Core.DataDirectory.FullPath;
compactDirectory = basePath + "-compacting";
tmpDirectory = basePath + "-old";
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(compactDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
configuration.Core.DataDirectory = new PathSetting(compactDirectory);
if (configuration.Storage.TempPath != null)
{
compactTempDirectory = configuration.Storage.TempPath.FullPath + "-temp-compacting";
EnsureDirectoriesPermission(compactTempDirectory);
IOExtensions.DeleteDirectory(compactTempDirectory);
configuration.Storage.TempPath = new PathSetting(compactTempDirectory);
}
var revisionsPrefix = CollectionName.GetTablePrefix(CollectionTableType.Revisions);
var compressedCollectionsTableNames = databaseRecord.DocumentsCompression?.Collections
.Select(name => new CollectionName(name).GetTableName(CollectionTableType.Documents))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
using (var dst = DocumentsStorage.GetStorageEnvironmentOptionsFromConfiguration(configuration, new IoChangesNotifications
{
DisableIoMetrics = true
},
new CatastrophicFailureNotification((envId, path, exception, stacktrace) => throw new InvalidOperationException($"Failed to compact database {_database} ({path}). StackTrace='{stacktrace}'", exception))))
{
InitializeOptions(dst, configuration, documentDatabase, encryptionKey);
DirectoryExecUtils.SubscribeToOnDirectoryInitializeExec(dst, configuration.Storage, documentDatabase.Name, DirectoryExecUtils.EnvironmentType.Compaction, Logger);
_token.ThrowIfCancellationRequested();
StorageCompaction.Execute(src, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)dst, progressReport =>
{
result.Progress.TreeProgress = progressReport.TreeProgress;
result.Progress.TreeTotal = progressReport.TreeTotal;
result.Progress.TreeName = progressReport.TreeName;
result.Progress.GlobalProgress = progressReport.GlobalProgress;
result.Progress.GlobalTotal = progressReport.GlobalTotal;
result.AddMessage(progressReport.Message);
onProgress?.Invoke(result.Progress);
}, (name, schema) =>
{
bool isRevision = name.StartsWith(revisionsPrefix, StringComparison.OrdinalIgnoreCase);
schema.Compressed =
(isRevision && databaseRecord.DocumentsCompression?.CompressRevisions == true) ||
compressedCollectionsTableNames?.Contains(name) == true;
}, _token);
}
result.TreeName = null;
_token.ThrowIfCancellationRequested();
EnsureDirectoriesPermission(basePath, compactDirectory, tmpDirectory);
IOExtensions.DeleteDirectory(tmpDirectory);
SwitchDatabaseDirectories(basePath, tmpDirectory, compactDirectory);
done = true;
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to execute compaction for {_database}", e);
}
finally
{
IOExtensions.DeleteDirectory(compactDirectory);
if (done)
{
IOExtensions.DeleteDirectory(tmpDirectory);
if (compactTempDirectory != null)
{
IOExtensions.DeleteDirectory(compactTempDirectory);
}
}
_isCompactionInProgress = false;
if (encryptionKey != null)
{
Sodium.ZeroBuffer(encryptionKey);
}
}
}
