public static void Execute(StorageEnvironmentOptions srcOptions, StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions compactOptions, Action<CompactionProgress> progressReport = null)
{
if (srcOptions.IncrementalBackupEnabled)
throw new InvalidOperationException(CannotCompactBecauseOfIncrementalBackup);
long minimalCompactedDataFileSize;
srcOptions.ManualFlushing = true; // prevent from flushing during compaction - we shouldn't touch any source files
compactOptions.ManualFlushing = true; // let us flush manually during data copy
using(var existingEnv = new StorageEnvironment(srcOptions))
using (var compactedEnv = new StorageEnvironment(compactOptions))
{
CopyTrees(existingEnv, compactedEnv, progressReport);
compactedEnv.FlushLogToDataFile(allowToFlushOverwrittenPages: true);
compactedEnv.Journal.Applicator.SyncDataFile(compactedEnv.OldestTransaction);
compactedEnv.Journal.Applicator.DeleteCurrentAlreadyFlushedJournal();
minimalCompactedDataFileSize = compactedEnv.NextPageNumber*AbstractPager.PageSize;
}
using (var compactedDataFile = new FileStream(Path.Combine(compactOptions.BasePath, Constants.DatabaseFilename), FileMode.Open, FileAccess.ReadWrite))
{
compactedDataFile.SetLength(minimalCompactedDataFileSize);
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.CreateTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
Assert.Equal(0, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
private void Restore(StorageEnvironment env, string singleBackupFile)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
env.FlushLogToDataFile();
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
if (package.Entries.Count == 0)
{
return;
}
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
Restore(env, package.Entries, tempDir, toDispose, txw);
}
}
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.WriteTransaction())
{
txw.CreateTree("test");
txw.Commit();
}
using (var txw = env.WriteTransaction())
{
var tree = txw.CreateTree("test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
// we keep track of the pages in scratch for one additional transaction, to avoid race
// condition with FlushLogToDataFile concurrently with new read transactions
Assert.Equal(2, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.CreateTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
// we keep track of the pages in scratch for one additional transaction, to avoid race
// condition with FlushLogToDataFile concurrently with new read transactions
Assert.Equal(2, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.State.GetTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
Assert.Equal(0, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public static void Execute(StorageEnvironmentOptions srcOptions,
StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions compactOptions,
Action <CompactionProgress> progressReport = null)
{
if (srcOptions.IncrementalBackupEnabled)
{
throw new InvalidOperationException(CannotCompactBecauseOfIncrementalBackup);
}
long minimalCompactedDataFileSize;
srcOptions.ManualFlushing = true; // prevent from flushing during compaction - we shouldn't touch any source files
compactOptions.ManualFlushing = true; // let us flush manually during data copy
using (var existingEnv = new StorageEnvironment(srcOptions))
using (var compactedEnv = new StorageEnvironment(compactOptions))
{
CopyTrees(existingEnv, compactedEnv, progressReport);
compactedEnv.FlushLogToDataFile();
compactedEnv.Journal.Applicator.SyncDataFile();
compactedEnv.Journal.Applicator.DeleteCurrentAlreadyFlushedJournal();
minimalCompactedDataFileSize = compactedEnv.NextPageNumber * existingEnv.Options.PageSize;
}
using (var compactedDataFile = new FileStream(Path.Combine(compactOptions.BasePath, Constants.DatabaseFilename), FileMode.Open, FileAccess.ReadWrite))
{
compactedDataFile.SetLength(minimalCompactedDataFileSize);
}
}
public static void Execute(StorageEnvironmentOptions srcOptions,
StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions compactOptions,
Action<StorageCompactionProgress> progressReport = null,
CancellationToken token = default(CancellationToken))
{
if (srcOptions.IncrementalBackupEnabled)
throw new InvalidOperationException(CannotCompactBecauseOfIncrementalBackup);
long minimalCompactedDataFileSize;
srcOptions.ManualFlushing = true; // prevent from flushing during compaction - we shouldn't touch any source files
compactOptions.ManualFlushing = true; // let us flush manually during data copy
using (var existingEnv = new StorageEnvironment(srcOptions))
using (var compactedEnv = new StorageEnvironment(compactOptions))
{
CopyTrees(existingEnv, compactedEnv, progressReport, token);
compactedEnv.FlushLogToDataFile();
bool synced;
const int maxNumberOfRetries = 100;
var syncRetries = 0;
while (true)
{
token.ThrowIfCancellationRequested();
using (var op = new WriteAheadJournal.JournalApplicator.SyncOperation(compactedEnv.Journal.Applicator))
{
try
{
synced = op.SyncDataFile();
if (synced || ++syncRetries >= maxNumberOfRetries)
break;
Thread.Sleep(100);
}
catch (Exception e)
{
existingEnv.Options.SetCatastrophicFailure(ExceptionDispatchInfo.Capture(e));
throw;
}
}
}
if (synced)
compactedEnv.Journal.Applicator.DeleteCurrentAlreadyFlushedJournal();
minimalCompactedDataFileSize = compactedEnv.NextPageNumber * Constants.Storage.PageSize;
}
using (var compactedDataFile = new FileStream(compactOptions.BasePath.Combine(Constants.DatabaseFilename).FullPath, FileMode.Open, FileAccess.ReadWrite))
{
compactedDataFile.SetLength(minimalCompactedDataFileSize);
}
}
private static long CopyFixedSizeTrees(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr,
TreeIterator rootIterator, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var treeNameSlice = rootIterator.CurrentKey.Clone(txr.Allocator);
var header = (FixedSizeTreeHeader.Embedded *)txr.LowLevelTransaction.RootObjects.DirectRead(treeNameSlice);
var fst = txr.FixedTreeFor(treeNameSlice, header->ValueSize);
Report(copiedTrees, totalTreesCount, 0, fst.NumberOfEntries, progressReport, $"Copying fixed size tree '{treeName}'. Progress: 0/{fst.NumberOfEntries} entries.", treeName);
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return(copiedTrees);
}
do
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var snd = txw.FixedTreeFor(treeNameSlice, header->ValueSize);
var transactionSize = 0L;
do
{
token.ThrowIfCancellationRequested();
Slice val;
using (it.Value(out val))
snd.Add(it.CurrentKey, val);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
var reportRate = fst.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport, $"Copying fixed size tree '{treeName}'. Progress: {copiedEntries}/{fst.NumberOfEntries} entries.", treeName);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && it.MoveNext());
txw.Commit();
}
if (fst.NumberOfEntries == copiedEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, fst.NumberOfEntries, progressReport, $"Finished copying fixed size tree '{treeName}'. Progress: {copiedEntries}/{fst.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (it.MoveNext());
}
return(copiedTrees);
}
public void OldestActiveTransactionShouldBeCalculatedProperly()
{
using (var options = StorageEnvironmentOptions.CreateMemoryOnly())
{
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
var trees = CreateTrees(env, 1, "tree");
var transactions = new List <Transaction>();
for (int a = 0; a < 100; a++)
{
var random = new Random(1337);
var buffer = new byte[random.Next(100, 1000)];
random.NextBytes(buffer);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (int i = 0; i < 100; i++)
{
foreach (var tree in trees)
{
tx.Environment.State.GetTree(tx, tree).Add(string.Format("key/{0}/{1}", a, i), new MemoryStream(buffer));
}
}
tx.Commit();
env.FlushLogToDataFile(tx);
var txr = env.NewTransaction(TransactionFlags.Read);
transactions.Add(txr);
}
}
Assert.Equal(transactions.OrderBy(x => x.Id).First().Id, env.OldestTransaction);
foreach (var tx in transactions)
{
foreach (var tree in trees)
{
using (var iterator = tx.Environment.State.GetTree(tx, tree).Iterate())
{
if (!iterator.Seek(Slice.BeforeAllKeys))
{
continue;
}
do
{
Assert.Contains("key/", iterator.CurrentKey.ToString());
} while (iterator.MoveNext());
}
}
}
}
}
}
private void ReplayFlushAction(FlushActivityEntry flushEntry, Transaction currentWriteTransaction)
{
if (flushEntry.ActionType == DebugActionType.FlushStart)
{
using (_env.Options.AllowManualFlushing())
{
_env.FlushLogToDataFile(currentWriteTransaction);
}
}
}
public void Record_debug_journal_and_replay_it_with_manual_flushing()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
env.Writer.Write(writeBatch);
}
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(tx);
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
}
}
}
public void ShouldProperlyRecover()
{
var sequentialLargeIds = ReadData("non-leaf-page-seq-id-large-values-2.txt");
var enumerator = sequentialLargeIds.GetEnumerator();
if (Directory.Exists("tests"))
{
Directory.Delete("tests", true);
}
var options = StorageEnvironmentOptions.ForPath("tests");
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
for (var transactions = 0; transactions < 100; transactions++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < 100; i++)
{
enumerator.MoveNext();
tx.Root.Add(enumerator.Current.Key.ToString("0000000000000000"), new MemoryStream(enumerator.Current.Value));
}
tx.Commit();
}
if (transactions == 50)
{
env.FlushLogToDataFile();
}
}
ValidateRecords(env, new List <string> {
"Root"
}, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
options = StorageEnvironmentOptions.ForPath("tests");
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
ValidateRecords(env, new List <string> {
"Root"
}, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
}
private PerformanceRecord ReadParallel(string operation, IEnumerable <uint> ids, PerfTracker perfTracker, int numberOfThreads)
{
var options = StorageEnvironmentOptions.ForPath(dataPath);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
env.FlushLogToDataFile();
return(ExecuteReadWithParallel(operation, ids, numberOfThreads, () => ReadInternal(ids, perfTracker, env)));
}
}
public void ShouldOccupyLessSpace(int seed)
{
var r = new Random(seed);
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(DataDir);
storageEnvironmentOptions.ManualFlushing = true;
using (var env = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("records");
for (int i = 0; i < 100; i++)
{
var bytes = new byte[r.Next(10, 2 * 1024 * 1024)];
r.NextBytes(bytes);
tree.Add("record/" + i, bytes);
}
tx.Commit();
}
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("records");
for (int i = 0; i < 50; i++)
{
tree.Delete("record/" + r.Next(0, 100));
}
tx.Commit();
}
env.FlushLogToDataFile();
}
var oldSize = GetDirSize(new DirectoryInfo(DataDir));
storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(DataDir);
storageEnvironmentOptions.ManualFlushing = true;
var compactedData = Path.Combine(DataDir, "Compacted");
StorageCompaction.Execute(storageEnvironmentOptions,
(StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactedData));
var newSize = GetDirSize(new DirectoryInfo(compactedData));
Assert.True(newSize < oldSize, string.Format("Old size: {0:#,#;;0} MB, new size {1:#,#;;0} MB", oldSize / 1024 / 1024, newSize / 1024 / 1024));
}
public void ShouldProperlyRecover()
{
var sequentialLargeIds = TestDataUtil.ReadData("non-leaf-page-seq-id-large-values-2.txt");
var enumerator = sequentialLargeIds.GetEnumerator();
var options = StorageEnvironmentOptions.ForPath(DataDir);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
for (var transactions = 0; transactions < 100; transactions++)
{
using (var tx = env.WriteTransaction())
{
var tree = tx.CreateTree("foo");
for (var i = 0; i < 100; i++)
{
enumerator.MoveNext();
tree.Add(enumerator.Current.Key.ToString("0000000000000000"), new MemoryStream(enumerator.Current.Value));
}
tx.Commit();
}
if (transactions == 50)
{
env.FlushLogToDataFile();
}
}
ValidateRecords(env, new List <string> {
"foo"
}, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
options = StorageEnvironmentOptions.ForPath(DataDir);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
ValidateRecords(env, new List <string> {
"foo"
}, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
}
public void OnDatabaseRecoverShouldMarkLastJournalAsRecyclableIfItExceedMaxLogFileSize()
{
CreateAndPopulateTree(startWithBigTx: false);
// restart
using (var env = new StorageEnvironment(ModifyOptions(StorageEnvironmentOptions.ForPath(DataDir), manualFlushing: true)))
{
var journalPath = env.Options.JournalPath.FullPath;
var journalsCount = new DirectoryInfo(journalPath).GetFiles().Length;
env.FlushLogToDataFile();
env.ForceSyncDataFile();
Assert.True(SpinWait.SpinUntil(() => new DirectoryInfo(journalPath).GetFiles($"{StorageEnvironmentOptions.RecyclableJournalFileNamePrefix}*").Length == journalsCount,
TimeSpan.FromSeconds(30)));
}
}
private void Restore(StorageEnvironment env, IEnumerable <ZipArchiveEntry> entries)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
env.FlushLogToDataFile();
var transactionPersistentContext = new TransactionPersistentContext(true);
using (var txw = env.NewLowLevelTransaction(transactionPersistentContext, TransactionFlags.ReadWrite))
{
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
Restore(env, entries, tempDir, toDispose, txw);
}
}
}
private static void CopyFixedSizeTree(FixedSizeTree fst, Func <Transaction, FixedSizeTree> createDestinationTree, StorageEnvironment compactedEnv, TransactionPersistentContext context, Action <long> onEntriesCopiedProgress, Action onAllEntriesCopied, CancellationToken token)
{
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return;
}
do
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var snd = createDestinationTree(txw);
var transactionSize = 0L;
do
{
token.ThrowIfCancellationRequested();
using (it.Value(out var val))
snd.Add(it.CurrentKey, val);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
var reportRate = fst.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
onEntriesCopiedProgress(copiedEntries);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && it.MoveNext());
txw.Commit();
}
compactedEnv.FlushLogToDataFile();
if (fst.NumberOfEntries == copiedEntries)
{
onAllEntriesCopied();
}
} while (it.MoveNext());
}
}
public void ShouldProperlyRecover()
{
var sequentialLargeIds = ReadData("non-leaf-page-seq-id-large-values-2.txt");
var enumerator = sequentialLargeIds.GetEnumerator();
if (Directory.Exists("tests"))
Directory.Delete("tests", true);
var options = StorageEnvironmentOptions.ForPath("tests");
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
for (var transactions = 0; transactions < 100; transactions++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < 100; i++)
{
enumerator.MoveNext();
tx.Root.Add (enumerator.Current.Key.ToString("0000000000000000"), new MemoryStream(enumerator.Current.Value));
}
tx.Commit();
}
if (transactions == 50)
env.FlushLogToDataFile();
}
ValidateRecords(env, new List<string> { "Root" }, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
options = StorageEnvironmentOptions.ForPath("tests");
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
ValidateRecords(env, new List<string> { "Root" }, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
}
}
private static long CopyFixedSizeTrees(StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport, Transaction txr,
TreeIterator rootIterator, string treeName, long copiedTrees, long totalTreesCount)
{
var fst = txr.FixedTreeFor(rootIterator.CurrentKey.Clone(txr.Allocator), 0);
Report(treeName, copiedTrees, totalTreesCount, 0,
fst.NumberOfEntries,
progressReport);
using (var it = fst.Iterate())
{
var copiedEntries = 0L;
if (it.Seek(Int64.MinValue) == false)
{
return(copiedTrees);
}
do
{
using (var txw = compactedEnv.WriteTransaction())
{
var snd = txw.FixedTreeFor(rootIterator.CurrentKey.Clone(txr.Allocator));
var transactionSize = 0L;
do
{
snd.Add(it.CurrentKey, it.Value);
transactionSize += fst.ValueSize + sizeof(long);
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxLogFileSize / 2 && it.MoveNext());
txw.Commit();
}
if (fst.NumberOfEntries == copiedEntries)
{
copiedTrees++;
}
Report(treeName, copiedTrees, totalTreesCount, copiedEntries,
fst.NumberOfEntries,
progressReport);
compactedEnv.FlushLogToDataFile();
} while (it.MoveNext());
}
return(copiedTrees);
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_LOTS_of_transactions()
{
var transactionsToShip = new List <TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
shippingSourceEnv.CreateTree(tx, "TestTree2");
tx.Commit();
}
WriteLotsOfTestDataForTree("TestTree", shippingSourceEnv);
WriteLotsOfTestDataForTree("TestTree2", shippingSourceEnv);
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
{
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
}
shippingDestinationEnv.FlushLogToDataFile();
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
ValidateLotsOfTestDataForTree(snapshot, "TestTree");
ValidateLotsOfTestDataForTree(snapshot, "TestTree2");
}
}
}
public void MultipleTxPagesCanPointToOnePageNumberWhichShouldNotBeCausingIssuesDuringFlushing()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
var trees = CreateTrees(env, 2, "tree");
var tree1 = trees[0];
var tree2 = trees[1];
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var t1 = tx.ReadTree(tree1);
t1.MultiAdd("key", "value/1");
t1.MultiAdd("key", "value/2");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var t1 = tx.ReadTree(tree1);
var t2 = tx.ReadTree(tree2);
var buffer = new byte[1000];
t1.MultiDelete("key", "value/1");
t1.MultiDelete("key", "value/2");
t2.Add("key/1", new MemoryStream(buffer));
t2.Add("key/2", new MemoryStream(buffer));
t2.Add("key/3", new MemoryStream(buffer));
t2.Add("key/4", new MemoryStream(buffer));
t2.Add("key/5", new MemoryStream(buffer));
tx.Commit();
}
env.FlushLogToDataFile();
}
}
public void MultipleTxPagesCanPointToOnePageNumberWhichShouldNotBeCausingIssuesDuringFlushing()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
var trees = CreateTrees(env, 2, "tree");
var tree1 = trees[0];
var tree2 = trees[1];
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var t1 = tx.State.GetTree(tx, tree1);
t1.MultiAdd("key", "value/1");
t1.MultiAdd("key", "value/2");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var t1 = tx.State.GetTree(tx, tree1);
var t2 = tx.State.GetTree(tx, tree2);
var buffer = new byte[1000];
t1.MultiDelete("key", "value/1");
t1.MultiDelete("key", "value/2");
t2.Add("key/1", new MemoryStream(buffer));
t2.Add("key/2", new MemoryStream(buffer));
t2.Add("key/3", new MemoryStream(buffer));
t2.Add("key/4", new MemoryStream(buffer));
t2.Add("key/5", new MemoryStream(buffer));
tx.Commit();
}
env.FlushLogToDataFile();
}
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_new_trees()
{
var transactionsToShip = new ConcurrentBag <TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
tx.Commit();
}
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
{
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
}
shippingDestinationEnv.FlushLogToDataFile();
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
snapshot.Read("TestTree", "Foo");
}
}
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_new_trees()
{
var transactionsToShip = new ConcurrentBag<TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
tx.Commit();
}
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
shippingDestinationEnv.FlushLogToDataFile();
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
snapshot.Read("TestTree", "Foo");
}
}
}
private void Restore(StorageEnvironment env, string singleBackupFile)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
using (var txw = env.NewLowLevelTransaction(TransactionFlags.ReadWrite))
{
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(txw);
}
using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
if (package.Entries.Count == 0)
{
return;
}
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
try
{
TransactionHeader *lastTxHeader = null;
var pagesToWrite = new Dictionary <long, TreePage>();
long journalNumber = -1;
foreach (var entry in package.Entries)
{
switch (Path.GetExtension(entry.Name))
{
case ".merged-journal":
case ".journal":
var jounalFileName = Path.Combine(tempDir, entry.Name);
using (var output = new FileStream(jounalFileName, FileMode.Create))
using (var input = entry.Open())
{
output.Position = output.Length;
input.CopyTo(output);
}
var pager = env.Options.OpenPager(jounalFileName);
toDispose.Add(pager);
if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
{
throw new InvalidOperationException("Cannot parse journal file number");
}
var recoveryPager = env.Options.CreateScratchPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
toDispose.Add(recoveryPager);
var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);
while (reader.ReadOneTransaction(env.Options))
{
lastTxHeader = reader.LastTransactionHeader;
}
foreach (var translation in reader.TransactionPageTranslation)
{
var pageInJournal = translation.Value.JournalPos;
var page = recoveryPager.Read(null, pageInJournal);
pagesToWrite[translation.Key] = page;
if (page.IsOverflow)
{
var numberOfOverflowPages = recoveryPager.GetNumberOfOverflowPages(page.OverflowSize);
for (int i = 1; i < numberOfOverflowPages; i++)
{
pagesToWrite.Remove(translation.Key + i);
}
}
}
break;
default:
throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
}
}
var sortedPages = pagesToWrite.OrderBy(x => x.Key)
.Select(x => x.Value)
.ToList();
if (sortedPages.Count == 0)
{
return;
}
var last = sortedPages.Last();
var numberOfPages = last.IsOverflow
? env.Options.DataPager.GetNumberOfOverflowPages(
last.OverflowSize)
: 1;
var pagerState = env.Options.DataPager.EnsureContinuous(last.PageNumber, numberOfPages);
txw.EnsurePagerStateReference(pagerState);
foreach (var page in sortedPages)
{
env.Options.DataPager.Write(page);
}
env.Options.DataPager.Sync();
var root = Tree.Open(txw, null, &lastTxHeader->Root);
root.Name = Constants.RootTreeName;
txw.UpdateRootsIfNeeded(root);
txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;
env.Journal.Clear(txw);
txw.Commit();
env.HeaderAccessor.Modify(header =>
{
header->TransactionId = lastTxHeader->TransactionId;
header->LastPageNumber = lastTxHeader->LastPageNumber;
header->Journal.LastSyncedJournal = journalNumber;
header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;
header->Root = lastTxHeader->Root;
header->Journal.CurrentJournal = journalNumber + 1;
header->Journal.JournalFilesCount = 0;
});
}
finally
{
toDispose.ForEach(x => x.Dispose());
try
{
Directory.Delete(tempDir, true);
}
catch
{
// this is just a temporary directory, the worst case scenario is that we dont reclaim the space from the OS temp directory
// if for some reason we cannot delete it we are safe to ignore it.
}
}
}
}
}
}
public void Record_debug_journal_and_replay_it_with_manual_flushing()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
env.Writer.Write(writeBatch);
}
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(tx);
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
}
}
}
public void OldestActiveTransactionShouldBeCalculatedProperly()
{
var directory = "Test";
if (Directory.Exists(directory))
Directory.Delete(directory, true);
var options = StorageEnvironmentOptions.ForPath(directory);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
var trees = CreateTrees(env, 1, "tree");
var transactions = new List<Transaction>();
for (int a = 0; a < 100; a++)
{
var random = new Random(1337);
var buffer = new byte[random.Next(100, 1000)];
random.NextBytes(buffer);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (int i = 0; i < 100; i++)
{
foreach (var tree in trees)
{
tx.Environment.State.GetTree(tx,tree).Add(tx, string.Format("key/{0}/{1}", a, i), new MemoryStream(buffer));
}
}
tx.Commit();
env.FlushLogToDataFile(tx);
var txr = env.NewTransaction(TransactionFlags.Read);
transactions.Add(txr);
}
}
Assert.Equal(transactions.OrderBy(x => x.Id).First().Id, env.OldestTransaction);
foreach (var tx in transactions)
{
foreach (var tree in trees)
{
using (var iterator = tx.Environment.State.GetTree(tx,tree).Iterate(tx))
{
if (!iterator.Seek(Slice.BeforeAllKeys))
continue;
do
{
Assert.Contains("key/", iterator.CurrentKey.ToString());
}
while (iterator.MoveNext());
}
}
}
}
}
private void Restore(StorageEnvironment env, string singleBackupFile)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(txw);
}
using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read))
{
if (package.Entries.Count == 0)
{
return;
}
var toDispose = new List <IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
try
{
TransactionHeader *lastTxHeader = null;
var pagesToWrite = new Dictionary <long, Func <Page> >();
long journalNumber = -1;
foreach (var entry in package.Entries)
{
switch (Path.GetExtension(entry.Name))
{
case ".journal":
var jounalFileName = Path.Combine(tempDir, entry.Name);
using (var output = new FileStream(jounalFileName, FileMode.Create))
using (var input = entry.Open())
{
output.Position = output.Length;
input.CopyTo(output);
}
var pager = new Win32MemoryMapPager(jounalFileName);
toDispose.Add(pager);
if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
{
throw new InvalidOperationException("Cannot parse journal file number");
}
var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
toDispose.Add(recoveryPager);
var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);
while (reader.ReadOneTransaction(env.Options))
{
lastTxHeader = reader.LastTransactionHeader;
}
foreach (var translation in reader.TransactionPageTranslation)
{
var pageInJournal = translation.Value.JournalPos;
pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
}
break;
default:
throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
}
}
var sortedPages = pagesToWrite.OrderBy(x => x.Key)
.Select(x => x.Value())
.ToList();
if (sortedPages.Count == 0)
{
return;
}
var last = sortedPages.Last();
env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
last.IsOverflow
? env.Options.DataPager.GetNumberOfOverflowPages(
last.OverflowSize)
: 1);
foreach (var page in sortedPages)
{
env.Options.DataPager.Write(page);
}
env.Options.DataPager.Sync();
txw.State.Root = Tree.Open(txw, &lastTxHeader->Root);
txw.State.FreeSpaceRoot = Tree.Open(txw, &lastTxHeader->FreeSpace);
txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
txw.State.Root.Name = Constants.RootTreeName;
txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;
env.Journal.Clear(txw);
txw.Commit();
env.HeaderAccessor.Modify(header =>
{
header->TransactionId = lastTxHeader->TransactionId;
header->LastPageNumber = lastTxHeader->LastPageNumber;
header->Journal.LastSyncedJournal = journalNumber;
header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;
header->Root = lastTxHeader->Root;
header->FreeSpace = lastTxHeader->FreeSpace;
header->Journal.CurrentJournal = journalNumber + 1;
header->Journal.JournalFilesCount = 0;
});
}
finally
{
toDispose.ForEach(x => x.Dispose());
try
{
Directory.Delete(tempDir, true);
}
catch (Exception)
{
// just temp dir - ignore it
}
}
}
}
}
}
private static void CopyTrees(StorageEnvironment existingEnv, StorageEnvironment compactedEnv, Action<CompactionProgress> progressReport = null)
{
using (var rootIterator = existingEnv.State.Root.Iterate())
{
if (rootIterator.Seek(Slice.BeforeAllKeys) == false)
return;
var totalTreesCount = existingEnv.State.Root.State.EntriesCount;
var copiedTrees = 0L;
do
{
var treeName = rootIterator.CurrentKey.ToString();
using (var txr = existingEnv.NewTransaction(TransactionFlags.Read))
{
var existingTree = existingEnv.State.GetTree(txr, treeName);
Report(treeName, copiedTrees, totalTreesCount, 0, existingTree.State.EntriesCount, progressReport);
using (var existingTreeIterator = existingTree.Iterate())
{
if (existingTreeIterator.Seek(Slice.BeforeAllKeys) == false)
continue;
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
compactedEnv.CreateTree(txw, treeName);
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
var newTree = txw.ReadTree(treeName);
do
{
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == NodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slice.BeforeAllKeys) == false)
continue;
do
{
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxLogFileSize/2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.EntriesCount)
copiedTrees++;
Report(treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.EntriesCount, progressReport);
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
}
} while (rootIterator.MoveNext());
}
}
private static unsafe long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport, Transaction txr,
string treeName, long copiedTrees, long totalTreesCount)
{
var existingTree = txr.ReadTree(treeName);
Report(treeName, copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
using (var txw = compactedEnv.WriteTransaction())
{
txw.CreateTree(treeName);
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
using (var txw = compactedEnv.WriteTransaction())
{
var newTree = txw.ReadTree(treeName);
do
{
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxLogFileSize / 2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
}
Report(treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries,
progressReport);
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}
private static long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var existingTree = txr.ReadTree(treeName);
Report(copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: 0/{existingTree.State.NumberOfEntries} entries.", treeName);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
if (existingTree.IsLeafCompressionSupported)
{
txw.CreateTree(treeName, flags: TreeFlags.LeafsCompressed);
}
else
{
txw.CreateTree(treeName);
}
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
token.ThrowIfCancellationRequested();
var txw = compactedEnv.WriteTransaction(context);
try
{
var newTree = txw.ReadTree(treeName);
do
{
token.ThrowIfCancellationRequested();
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
token.ThrowIfCancellationRequested();
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else if (existingTree.IsLeafCompressionSupported)
{
using (var read = existingTree.ReadDecompressed(key))
{
var value = read.Reader.AsStream();
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else if (existingTree.State.Flags == (TreeFlags.FixedSizeTrees | TreeFlags.Streams))
{
var tag = existingTree.GetStreamTag(key);
using (var stream = existingTree.ReadStream(key))
{
if (tag != null)
{
Slice tagStr;
using (Slice.From(txw.Allocator, tag, out tagStr))
newTree.AddStream(key, stream, tagStr);
}
else
{
newTree.AddStream(key, stream);
}
transactionSize += stream.Length;
}
}
else if (existingTree.State.Flags == TreeFlags.FixedSizeTrees)
{
var reader = existingTree.GetValueReaderFromHeader(existingTreeIterator.Current);
if (reader.Length >= sizeof(FixedSizeTreeHeader.Embedded))
{
var header = (FixedSizeTreeHeader.Embedded *)reader.Base;
if (header->RootObjectType == RootObjectType.FixedSizeTree || header->RootObjectType == RootObjectType.EmbeddedFixedSizeTree)
{
// CopyFixedSizeTree will open dedicated write transaction to copy fixed size tree
txw.Commit();
txw.Dispose();
txw = null;
var fixedSizeTreeName = key;
var fst = existingTree.FixedTreeFor(fixedSizeTreeName, (byte)header->ValueSize);
var currentCopiedTrees = copiedTrees;
var currentCopiedEntries = copiedEntries;
CopyFixedSizeTree(fst, tx =>
{
var treeInCompactedEnv = tx.ReadTree(treeName);
return(treeInCompactedEnv.FixedTreeFor(fixedSizeTreeName, (byte)header->ValueSize));
}, compactedEnv, context, copiedFstEntries =>
{
Report(currentCopiedTrees, totalTreesCount, currentCopiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Copying fixed size tree '{fixedSizeTreeName}' inside '{treeName}' tree. Progress: {copiedFstEntries}/{fst.NumberOfEntries} entries.",
treeName);
}, () =>
{
Report(currentCopiedTrees, totalTreesCount, currentCopiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Finished copying fixed size tree '{fixedSizeTreeName}' inside '{treeName}' tree. {fst.NumberOfEntries} entries copied.",
treeName);
}, token);
IncrementNumberOfCopiedEntries();
break; // let's open new transaction after copying fixed size tree
}
}
// if the entry wasn't recognized as fixed size tree then let's store it as regular value
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
IncrementNumberOfCopiedEntries();
void IncrementNumberOfCopiedEntries()
{
copiedEntries++;
var reportRate = existingTree.State.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport,
$"Copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && existingTreeIterator.MoveNext());
txw?.Commit();
}
finally
{
txw?.Dispose();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Finished copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}
private static void CopyTrees(StorageEnvironment existingEnv, StorageEnvironment compactedEnv, Action <CompactionProgress> progressReport = null)
{
using (var txr = existingEnv.NewTransaction(TransactionFlags.Read))
using (var rootIterator = txr.Root.Iterate())
{
if (rootIterator.Seek(Slice.BeforeAllKeys) == false)
{
return;
}
var totalTreesCount = txr.Root.State.EntriesCount;
var copiedTrees = 0L;
do
{
var treeName = rootIterator.CurrentKey.ToString();
var existingTree = txr.ReadTree(treeName);
Report(treeName, copiedTrees, totalTreesCount, 0, existingTree.State.EntriesCount, progressReport);
using (var existingTreeIterator = existingTree.Iterate())
{
if (existingTreeIterator.Seek(Slice.BeforeAllKeys) == false)
{
continue;
}
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
compactedEnv.CreateTree(txw, treeName);
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
var newTree = txw.ReadTree(treeName);
do
{
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == NodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slice.BeforeAllKeys) == false)
{
continue;
}
do
{
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxLogFileSize / 2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.EntriesCount)
{
copiedTrees++;
}
Report(treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.EntriesCount,
progressReport);
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
} while (rootIterator.MoveNext());
}
}
private void Restore(StorageEnvironment env, string singleBackupFile)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(txw);
}
using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read))
{
if (package.Entries.Count == 0)
return;
var toDispose = new List<IDisposable>();
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
try
{
TransactionHeader* lastTxHeader = null;
var pagesToWrite = new Dictionary<long, Func<Page>>();
long journalNumber = -1;
foreach (var entry in package.Entries)
{
switch (Path.GetExtension(entry.Name))
{
case ".journal":
var jounalFileName = Path.Combine(tempDir, entry.Name);
using (var output = new FileStream(jounalFileName, FileMode.Create))
using (var input = entry.Open())
{
output.Position = output.Length;
input.CopyTo(output);
}
var pager = new Win32MemoryMapPager(jounalFileName);
toDispose.Add(pager);
if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
{
throw new InvalidOperationException("Cannot parse journal file number");
}
var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
toDispose.Add(recoveryPager);
var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);
while (reader.ReadOneTransaction(env.Options))
{
lastTxHeader = reader.LastTransactionHeader;
}
foreach (var translation in reader.TransactionPageTranslation)
{
var pageInJournal = translation.Value.JournalPos;
pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
}
break;
default:
throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
}
}
var sortedPages = pagesToWrite.OrderBy(x => x.Key)
.Select(x => x.Value())
.ToList();
var last = sortedPages.Last();
env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
last.IsOverflow
? env.Options.DataPager.GetNumberOfOverflowPages(
last.OverflowSize)
: 1);
foreach (var page in sortedPages)
{
env.Options.DataPager.Write(page);
}
env.Options.DataPager.Sync();
txw.State.Root = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root);
txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace);
txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
txw.State.Root.Name = Constants.RootTreeName;
txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;
env.Journal.Clear(txw);
txw.Commit();
env.HeaderAccessor.Modify(header =>
{
header->TransactionId = lastTxHeader->TransactionId;
header->LastPageNumber = lastTxHeader->LastPageNumber;
header->Journal.LastSyncedJournal = journalNumber;
header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;
header->Root = lastTxHeader->Root;
header->FreeSpace = lastTxHeader->FreeSpace;
header->Journal.CurrentJournal = journalNumber + 1;
header->Journal.JournalFilesCount = 0;
});
}
finally
{
toDispose.ForEach(x => x.Dispose());
try
{
Directory.Delete(tempDir, true);
}
catch (Exception)
{
// just temp dir - ignore it
}
}
}
}
}
}
private static long CopyTableTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
// Load table
var tableTree = txr.ReadTree(treeName, RootObjectType.Table);
// Get the table schema
var schemaSize = tableTree.GetDataSize(TableSchema.SchemasSlice);
var schemaPtr = tableTree.DirectRead(TableSchema.SchemasSlice);
var schema = TableSchema.ReadFrom(txr.Allocator, schemaPtr, schemaSize);
// Load table into structure
var inputTable = txr.OpenTable(schema, treeName);
// The next three variables are used to know what our current
// progress is
var copiedEntries = 0;
// It is very important that these slices be allocated in the
// txr.Allocator, as the intermediate write transactions on
// the compacted environment will be destroyed between each
// loop.
var lastSlice = Slices.BeforeAllKeys;
long lastFixedIndex = 0L;
Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
using (var txw = compactedEnv.WriteTransaction(context))
{
schema.Create(txw, treeName, Math.Max((ushort)inputTable.ActiveDataSmallSection.NumberOfPages, (ushort)((ushort.MaxValue + 1) / Constants.Storage.PageSize)));
txw.Commit(); // always create a table, even if it is empty
}
while (copiedEntries < inputTable.NumberOfEntries)
{
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
long transactionSize = 0L;
var outputTable = txw.OpenTable(schema, treeName);
if (schema.Key == null || schema.Key.IsGlobal)
{
// There is no primary key, or there is one that is global to multiple tables
// we require a table to have at least a single local index that we'll use
var variableSizeIndex = schema.Indexes.Values.FirstOrDefault(x => x.IsGlobal == false);
if (variableSizeIndex != null)
{
// We have a variable size index, use it
// In case we continue an existing compaction, skip to the next slice
var skip = 0;
// can't use SliceComparer.Compare here
if (lastSlice.Options != Slices.BeforeAllKeys.Options)
{
skip = 1;
}
var sp = Stopwatch.StartNew();
foreach (var tvr in inputTable.SeekForwardFrom(variableSizeIndex, lastSlice, skip))
{
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref tvr.Result.Reader);
copiedEntries++;
transactionSize += tvr.Result.Reader.Size;
ReportIfNeeded(sp, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
// The transaction has surpassed the allowed
// size before a flush
if (lastSlice.Equals(tvr.Key) == false && transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastSlice = tvr.Key.Clone(txr.Allocator);
break;
}
}
}
else
{
// Use a fixed size index
var fixedSizeIndex = schema.FixedSizeIndexes.Values.FirstOrDefault(x => x.IsGlobal == false);
if (fixedSizeIndex == null)
{
throw new InvalidOperationException("Cannot compact table " + inputTable.Name + " because is has no local indexes, only global ones");
}
var sp = Stopwatch.StartNew();
foreach (var entry in inputTable.SeekForwardFrom(fixedSizeIndex, lastFixedIndex, lastFixedIndex > 0 ? 1 : 0))
{
token.ThrowIfCancellationRequested();
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref entry.Reader);
copiedEntries++;
transactionSize += entry.Reader.Size;
ReportIfNeeded(sp, copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
lastFixedIndex = fixedSizeIndex.GetValue(ref entry.Reader);
break;
}
}
}
}
else
{
// The table has a primary key, inserts in that order are expected to be faster
foreach (var entry in inputTable.SeekByPrimaryKey(lastSlice, 0))
{
token.ThrowIfCancellationRequested();
// The table will take care of reconstructing indexes automatically
outputTable.Insert(ref entry.Reader);
copiedEntries++;
transactionSize += entry.Reader.Size;
// The transaction has surpassed the allowed
// size before a flush
if (transactionSize >= compactedEnv.Options.MaxScratchBufferSize / 2)
{
schema.Key.GetSlice(txr.Allocator, ref entry.Reader, out lastSlice);
break;
}
}
}
txw.Commit();
}
if (copiedEntries == inputTable.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, inputTable.NumberOfEntries, progressReport, $"Finished copying table tree '{treeName}'. Progress: {copiedEntries:#,#;;0}/{inputTable.NumberOfEntries:#,#;;0} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
}
return(copiedTrees);
}
public unsafe void ValidatePageChecksumShouldDetectDataCorruption()
{
// Create some random data
var treeNames = new List <string>();
var random = new Random();
var value1 = new byte[random.Next(1024 * 1024 * 2)];
var value2 = new byte[random.Next(1024 * 1024 * 2)];
random.NextBytes(value1);
random.NextBytes(value2);
const int treeCount = 5;
const int recordCount = 6;
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
env.Options.ManualFlushing = true;
for (int i = 0; i < treeCount; i++)
{
using (var tx = env.WriteTransaction())
{
string name = "tree/" + i;
treeNames.Add(name);
var tree = tx.CreateTree(name);
for (int j = 0; j < recordCount; j++)
{
tree.Add(string.Format("{0}/items/{1}", name, j), j % 2 == 0 ? value1 : value2);
}
tx.Commit();
}
}
env.FlushLogToDataFile();
}
// Lets corrupt something
using (var options = StorageEnvironmentOptions.ForPath(DataDir))
using (var pager = LinuxTestUtils.GetNewPager(options, DataDir, "Raven.Voron"))
using (var tempTX = new TempPagerTransaction())
{
var writePtr = pager.AcquirePagePointer(tempTX, 2) + PageHeader.SizeOf + 43; // just some random place on page #2
for (byte i = 0; i < 8; i++)
{
writePtr[i] = i;
}
}
// Now lets try to read it all back and hope we get an exception
try
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(DataDir)))
{
using (var tx = env.ReadTransaction())
{
foreach (var treeName in treeNames)
{
var tree = tx.CreateTree(treeName);
for (int i = 0; i < recordCount; i++)
{
var readResult = tree.Read(string.Format("{0}/items/{1}", treeName, i));
Assert.NotNull(readResult);
if (i % 2 == 0)
{
var readBytes = new byte[value1.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
}
else
{
var readBytes = new byte[value2.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
}
}
}
}
}
}
catch (Exception e)
{
Assert.True(e is InvalidOperationException || e is InvalidDataException);
}
}
private void Restore(StorageEnvironment env, string backupPath)
{
using (env.Journal.Applicator.TakeFlushingLock())
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(txw);
}
List <string> journalNames;
using (var package = ZipFile.Open(backupPath, ZipArchiveMode.Read))
{
journalNames = package.Entries.Select(x => x.Name).ToList();
}
if (journalNames.Count == 0)
{
return;
}
var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;
var toDispose = new List <IDisposable>();
try
{
ZipFile.ExtractToDirectory(backupPath, tempDir);
TransactionHeader *lastTxHeader = null;
var pagesToWrite = new Dictionary <long, Func <Page> >();
long journalNumber = -1;
foreach (var journalName in journalNames)
{
var pager = new Win32MemoryMapPager(Path.Combine(tempDir, journalName));
toDispose.Add(pager);
if (long.TryParse(journalName.Replace(".journal", string.Empty), out journalNumber) == false)
{
throw new InvalidOperationException("Cannot parse journal file number");
}
var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
toDispose.Add(recoveryPager);
var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);
while (reader.ReadOneTransaction(env.Options))
{
lastTxHeader = reader.LastTransactionHeader;
}
foreach (var translation in reader.TransactionPageTranslation)
{
var pageInJournal = translation.Value.JournalPos;
pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
}
}
var sortedPages = pagesToWrite.OrderBy(x => x.Key)
.Select(x => x.Value())
.ToList();
var last = sortedPages.Last();
env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
last.IsOverflow
? env.Options.DataPager.GetNumberOfOverflowPages(
last.OverflowSize)
: 1);
foreach (var page in sortedPages)
{
env.Options.DataPager.Write(page);
}
env.Options.DataPager.Sync();
txw.State.Root = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root);
txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace);
txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
txw.State.Root.Name = Constants.RootTreeName;
txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;
env.Journal.Clear(txw);
txw.Commit();
env.HeaderAccessor.Modify(header =>
{
header->TransactionId = lastTxHeader->TransactionId;
header->LastPageNumber = lastTxHeader->LastPageNumber;
header->Journal.LastSyncedJournal = journalNumber;
header->Journal.LastSyncedTransactionId = lastTxHeader->TransactionId;
header->Root = lastTxHeader->Root;
header->FreeSpace = lastTxHeader->FreeSpace;
header->Journal.CurrentJournal = journalNumber + 1;
header->Journal.JournalFilesCount = 0;
});
}
finally
{
toDispose.ForEach(x => x.Dispose());
Directory.Delete(tempDir, true);
}
}
}
}
public void ScratchPagesShouldNotBeReleasedUntilNotUsed()
{
var directory = "Test2";
if (Directory.Exists(directory))
Directory.Delete(directory, true);
var options = StorageEnvironmentOptions.ForPath(directory);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
CreateTrees(env, 2, "tree");
for (int a = 0; a < 3; a++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
tx.Environment.State.GetTree(tx, "tree0").Add(string.Format("key/{0}/{1}/1", new string('0', 1000), a), new MemoryStream());
tx.Environment.State.GetTree(tx, "tree0").Add(string.Format("key/{0}/{1}/2", new string('0', 1000), a), new MemoryStream());
tx.Commit();
}
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
tx.Environment.State.GetTree(tx, "tree1").Add("yek/1", new MemoryStream());
tx.Commit();
}
using (var txr = env.NewTransaction(TransactionFlags.Read))
{
using (var iterator = txr.Environment.State.GetTree(txr, "tree0").Iterate())
{
Assert.True(iterator.Seek(Slice.BeforeAllKeys)); // all pages are from scratch (one from position 11)
var currentKey = iterator.CurrentKey.ToString();
env.FlushLogToDataFile(); // frees pages from scratch (including the one at position 11)
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.State.GetTree(txw, "tree1");
tree.Add(string.Format("yek/{0}/0/0", new string('0', 1000)), new MemoryStream()); // allocates new page from scratch (position 11)
txw.Commit();
}
Assert.Equal(currentKey, iterator.CurrentKey.ToString());
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.State.GetTree(txw, "tree1");
tree.Add("fake", new MemoryStream());
txw.Commit();
}
Assert.Equal(currentKey, iterator.CurrentKey.ToString());
var count = 0;
do
{
currentKey = iterator.CurrentKey.ToString();
count++;
Assert.Contains("key/", currentKey);
}
while (iterator.MoveNext());
Assert.Equal(6, count);
}
}
}
}
public void ScratchPagesShouldNotBeReleasedUntilNotUsed()
{
var options = StorageEnvironmentOptions.ForPath(DataDir);
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
CreateTrees(env, 2, "tree");
for (int a = 0; a < 3; a++)
{
using (var tx = env.WriteTransaction())
{
tx.CreateTree("tree0").Add(string.Format("key/{0}/{1}/1", new string('0', 1000), a), new MemoryStream());
tx.CreateTree("tree0").Add(string.Format("key/{0}/{1}/2", new string('0', 1000), a), new MemoryStream());
tx.Commit();
}
}
using (var tx = env.WriteTransaction())
{
tx.CreateTree("tree1").Add("yek/1", new MemoryStream());
tx.Commit();
}
using (var txr = env.ReadTransaction())
{
using (var iterator = txr.CreateTree("tree0").Iterate(false))
{
Assert.True(iterator.Seek(Slices.BeforeAllKeys)); // all pages are from scratch (one from position 11)
var currentKey = iterator.CurrentKey.ToString();
env.FlushLogToDataFile(); // frees pages from scratch (including the one at position 11)
using (var txw = env.WriteTransaction())
{
var tree = txw.CreateTree("tree1");
tree.Add(string.Format("yek/{0}/0/0", new string('0', 1000)), new MemoryStream()); // allocates new page from scratch (position 11)
txw.Commit();
}
Assert.Equal(currentKey, iterator.CurrentKey.ToString());
using (var txw = env.WriteTransaction())
{
var tree = txw.CreateTree("tree1");
tree.Add("fake", new MemoryStream());
txw.Commit();
}
Assert.Equal(currentKey, iterator.CurrentKey.ToString());
var count = 0;
do
{
currentKey = iterator.CurrentKey.ToString();
count++;
Assert.Contains("key/", currentKey);
}while (iterator.MoveNext());
Assert.Equal(6, count);
}
}
}
}
private static long CopyVariableSizeTree(StorageEnvironment compactedEnv, Action <StorageCompactionProgress> progressReport, Transaction txr, string treeName, long copiedTrees, long totalTreesCount, TransactionPersistentContext context, CancellationToken token)
{
var existingTree = txr.ReadTree(treeName);
Report(copiedTrees, totalTreesCount, 0, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: 0/{existingTree.State.NumberOfEntries} entries.", treeName);
using (var existingTreeIterator = existingTree.Iterate(true))
{
if (existingTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
return(copiedTrees);
}
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
if (existingTree.IsLeafCompressionSupported)
{
txw.CreateTree(treeName, flags: TreeFlags.LeafsCompressed);
}
else
{
txw.CreateTree(treeName);
}
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
token.ThrowIfCancellationRequested();
using (var txw = compactedEnv.WriteTransaction(context))
{
var newTree = txw.ReadTree(treeName);
do
{
token.ThrowIfCancellationRequested();
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == TreeNodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slices.BeforeAllKeys) == false)
{
continue;
}
do
{
token.ThrowIfCancellationRequested();
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else if (existingTree.IsLeafCompressionSupported)
{
using (var read = existingTree.ReadDecompressed(key))
{
var value = read.Reader.AsStream();
newTree.Add(key, value);
transactionSize += value.Length;
}
}
else if (existingTree.State.Flags == (TreeFlags.FixedSizeTrees | TreeFlags.Streams))
{
var tag = existingTree.GetStreamTag(key);
using (var stream = existingTree.ReadStream(key))
{
if (tag != null)
{
Slice tagStr;
using (Slice.From(txw.Allocator, tag, out tagStr))
newTree.AddStream(key, stream, tagStr);
}
else
{
newTree.AddStream(key, stream);
}
transactionSize += stream.Length;
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
var reportRate = existingTree.State.NumberOfEntries / 33 + 1;
if (copiedEntries % reportRate == 0)
{
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
} while (transactionSize < compactedEnv.Options.MaxScratchBufferSize / 2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.NumberOfEntries)
{
copiedTrees++;
Report(copiedTrees, totalTreesCount, copiedEntries, existingTree.State.NumberOfEntries, progressReport, $"Finished copying variable size tree '{treeName}'. Progress: {copiedEntries}/{existingTree.State.NumberOfEntries} entries.", treeName);
}
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
return(copiedTrees);
}