public long GetEntriesCount(TableBase table)
{
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
return(env.CreateTree(tx, table.TableName).State.EntriesCount);
}
}
public void StorageRecoveryShouldWorkWhenThereAreCommitedAndUncommitedTransactions2()
{
var path = "test2.data";
DeleteDirectory(path);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "atree");
env.CreateTree(tx, "btree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < 10000; i++)
{
tx.Environment.State.GetTree(tx, "atree").Add("a" + i, new MemoryStream());
tx.Environment.State.GetTree(tx, "btree").MultiAdd("a" + i, "a" + i);
}
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
}
DeleteDirectory(path);
}
public FullTextIndex(StorageEnvironmentOptions options, IAnalyzer analyzer)
{
Analyzer = analyzer;
Conventions = new IndexingConventions();
BufferPool = new BufferPool();
StorageEnvironment = new StorageEnvironment(options);
using (var tx = StorageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
StorageEnvironment.CreateTree(tx, "@terms", keysPrefixing: true);
StorageEnvironment.CreateTree(tx, "deletes", keysPrefixing: true);
var docs = StorageEnvironment.CreateTree(tx, "docs", keysPrefixing: true);
var metadata = StorageEnvironment.CreateTree(tx, "$metadata");
var idVal = metadata.Read("id");
if (idVal == null)
{
Id = Guid.NewGuid();
metadata.Add("id", Id.ToByteArray());
}
else
{
int _;
Id = new Guid(idVal.Reader.ReadBytes(16, out _));
}
using (var it = docs.Iterate())
{
_lastDocumentId = it.Seek(Slice.AfterAllKeys) == false ?
0 :
it.CurrentKey.CreateReader().ReadBigEndianInt64();
}
tx.Commit();
}
}
public FreeDbQueries(string path)
{
_storageEnvironment = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path));
using (Transaction tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
_storageEnvironment.CreateTree(tx, "albums");
_storageEnvironment.CreateTree(tx, "ix_diskids");
_storageEnvironment.CreateTree(tx, "ix_artists");
_storageEnvironment.CreateTree(tx, "ix_titles");
tx.Commit();
}
}
public VoronDisksDestination()
{
_storageEnvironment = new StorageEnvironment(StorageEnvironmentOptions.ForPath("FreeDB"));
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
_storageEnvironment.CreateTree(tx, "albums");
_storageEnvironment.CreateTree(tx, "ix_diskids");
_storageEnvironment.CreateTree(tx, "ix_artists");
_storageEnvironment.CreateTree(tx, "ix_titles");
tx.Commit();
}
_currentBatch = new WriteBatch();
}
public void IterationShouldNotFindAnyRecordsAndShouldNotThrowWhenNumberOfEntriesOnPageIs1AndKeyDoesNotMatch()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.ReadTree("tree");
tree.Add(@"Raven\Database\1", StreamFor("123"));
tx.Commit();
}
using (var snapshot = env.CreateSnapshot())
using (var iterator = snapshot.Iterate("tree"))
{
Assert.False(iterator.Seek(@"Raven\Filesystem\"));
}
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.CreateTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
// we keep track of the pages in scratch for one additional transaction, to avoid race
// condition with FlushLogToDataFile concurrently with new read transactions
Assert.Equal(2, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void ShouldWork()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var s = new string('0', 500);
var tree = env.CreateTree(tx, "data");
for (int i = 0; i < 10; i++)
{
tree.Add("users-" + i + "-" + s, new byte[0]);
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("data");
using (var it = tree.Iterate())
{
Assert.True(it.Seek("users-7"));
for (int i = 0; i < 10; i++)
{
Assert.True(it.Seek("users-" + i), i.ToString());
}
}
}
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.CreateTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
// we keep track of the pages in scratch for one additional transaction, to avoid race
// condition with FlushLogToDataFile concurrently with new read transactions
Assert.Equal(2, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void DataIsKeptAfterRestartForSubTrees()
{
using (var pureMemoryPager = new PureMemoryPager())
{
using (var env = new StorageEnvironment(pureMemoryPager, ownsPager: false))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.GetTree("test").Add(tx, "test", Stream.Null);
tx.Commit();
}
}
using (var env = new StorageEnvironment(pureMemoryPager))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.NotNull(env.GetTree("test").Read(tx, "test"));
tx.Commit();
}
}
}
}
public void ShouldWork()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var s = new string('0', 500);
var tree = env.CreateTree(tx, "data");
for (int i = 0; i < 10; i++)
{
tree.Add("users-" + i + "-" + s, new byte[0]);
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.State.GetTree(tx, "data");
using (var it = tree.Iterate())
{
Assert.True(it.Seek("users-7"));
for (int i = 0; i < 10; i++)
{
Assert.True(it.Seek("users-"+i),i.ToString());
}
}
}
}
}
public KeyValueStateMachine(StorageEnvironmentOptions options)
{
options.IncrementalBackupEnabled = true;
_storageEnvironment = new StorageEnvironment(options);
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
_storageEnvironment.CreateTree(tx, "items");
var metadata = _storageEnvironment.CreateTree(tx, "$metadata");
var readResult = metadata.Read("last-index");
if (readResult != null)
{
LastAppliedIndex = readResult.Reader.ReadLittleEndianInt64();
}
tx.Commit();
}
}
public VoronOdbBackend(string voronDataPath)
{
if (voronDataPath == null)
{
throw new ArgumentNullException("voronDataPath");
}
_env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(voronDataPath));
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
_env.CreateTree(tx, Index);
_env.CreateTree(tx, Objects);
tx.Commit();
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.State.GetTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
Assert.Equal(0, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void IterationShouldNotFindAnyRecordsAndShouldNotThrowWhenNumberOfEntriesOnPageIs1AndKeyDoesNotMatch()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.ReadTree("tree");
tree.Add(@"Raven\Database\1", StreamFor("123"));
tx.Commit();
}
using (var snapshot = env.CreateSnapshot())
using (var iterator = snapshot.Iterate("tree"))
{
Assert.False(iterator.Seek(@"Raven\Filesystem\"));
}
}
}
public void AllScratchPagesShouldBeReleased()
{
var options = StorageEnvironmentOptions.CreateMemoryOnly();
options.ManualFlushing = true;
using (var env = new StorageEnvironment(options))
{
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(txw, "test");
txw.Commit();
}
using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = txw.Environment.State.GetTree(txw, "test");
tree.Add("key/1", new MemoryStream(new byte[100]));
tree.Add("key/1", new MemoryStream(new byte[200]));
txw.Commit();
}
env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch
Assert.Equal(0, env.ScratchBufferPool.GetNumberOfAllocations(0));
}
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_new_trees_no_flushing()
{
var transactionsToShip = new ConcurrentBag <TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
tx.Commit();
}
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
{
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
}
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
snapshot.Read("TestTree", "Foo");
}
}
}
public void Flush()
{
_bufferSize = 0;
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
foreach (var kvp in _buffer)
{
var data = _storageEnvironment.CreateTree(tx, "channel:" + kvp.Key);
var buffer = new byte[16];
var key = new Slice(buffer);
var ms = new MemoryStream();
var bw = new BinaryWriter(ms);
foreach (var item in kvp.Value)
{
var date = item.Timestamp;
EndianBitConverter.Big.CopyBytes(date.Ticks, buffer, 0);
EndianBitConverter.Big.CopyBytes(_last++, buffer, 8);
ms.SetLength(0);
bw.Write(item.Value);
ms.Position = 0;
data.Add(tx, key, ms);
}
}
tx.State.Root.Add(tx, _lastKey, new MemoryStream(BitConverter.GetBytes(_last)));
tx.Commit();
}
_buffer.Clear();
}
public void CannotCompactStorageIfIncrementalBackupEnabled()
{
var envOptions = StorageEnvironmentOptions.ForPath(CompactionTestsData);
envOptions.IncrementalBackupEnabled = true;
using (var env = new StorageEnvironment(envOptions))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
tree.Add("record/1", new byte[9]);
tree.Add("record/2", new byte[9]);
tx.Commit();
}
}
var srcOptions = StorageEnvironmentOptions.ForPath(CompactionTestsData);
srcOptions.IncrementalBackupEnabled = true;
var invalidOperationException = Assert.Throws <InvalidOperationException>(() => StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData)));
Assert.Equal(StorageCompaction.CannotCompactBecauseOfIncrementalBackup, invalidOperationException.Message);
}
private void ReplayWriteAction(WriteActivityEntry activityEntry, ref Transaction tx)
{
switch (activityEntry.ActionType)
{
case DebugActionType.Add:
tx.ReadTree(activityEntry.TreeName).Add(activityEntry.Key, activityEntry.ValueStream);
break;
case DebugActionType.Delete:
tx.ReadTree(activityEntry.TreeName).Delete(activityEntry.Key);
break;
case DebugActionType.MultiAdd:
tx.ReadTree(activityEntry.TreeName).MultiAdd(activityEntry.Key, new Slice(Encoding.UTF8.GetBytes(activityEntry.Value.ToString())));
break;
case DebugActionType.MultiDelete:
tx.ReadTree(activityEntry.TreeName).MultiDelete(activityEntry.Key, new Slice(Encoding.UTF8.GetBytes(activityEntry.Value.ToString())));
break;
case DebugActionType.CreateTree:
_env.CreateTree(tx, activityEntry.TreeName);
break;
case DebugActionType.Increment:
var buffer = new byte[sizeof(long)];
activityEntry.ValueStream.Read(buffer, 0, buffer.Length);
var delta = EndianBitConverter.Little.ToInt64(buffer, 0);
tx.ReadTree(activityEntry.TreeName).Increment(activityEntry.Key, delta);
break;
default: //precaution against newly added action types
throw new InvalidOperationException("unsupported tree action type");
}
}
public void Can_split_merged_transaction_to_multiple_tx()
{
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
storageEnvironmentOptions.MaxNumberOfPagesInMergedTransaction = 8;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
for (int xi = 0; xi < 100; xi++)
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < 1000; i++)
{
tree.Add("users/" + i, "john doe/" + i);
}
tx.Commit();
}
}
var snapshotWriter = new MinimalIncrementalBackup();
var backupPath = Path.Combine(_tempDir, "1.snapshot");
snapshotWriter.ToFile(envToSnapshot, backupPath);
using (var stream = File.OpenRead(backupPath))
using (var zip = new ZipArchive(stream, ZipArchiveMode.Read))
{
Assert.True(zip.Entries.Count > 1);
}
}
}
public void ShouldDeleteCurrentJournalEvenThoughItHasAvailableSpace()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
tx.Commit();
}
}
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
var compactedDir = new DirectoryInfo(CompactedData);
var journalsAfterCompaction = compactedDir.GetFiles("*.journal").Select(x => x.Name).ToList();
Assert.Equal(0, journalsAfterCompaction.Count);
// ensure it can write more data
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactedData)))
{
using (var tx = compacted.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = compacted.CreateTree(tx, "fruits");
tree.Add("peach", new byte[144]);
}
}
}
public void Mixed_small_and_overflow_changes()
{
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
tree.Add("users/1", "john doe");
tree.Add("users/2", new String('a', 5000));
tx.Commit();
}
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
tree.Add("users/2", "jane darling");
tree.Add("users/3", new String('b', 5000));
tx.Commit();
}
var snapshotWriter = new MinimalIncrementalBackup();
snapshotWriter.ToFile(envToSnapshot, Path.Combine(_tempDir, "1.snapshot"));
var restoredOptions = StorageEnvironmentOptions.ForPath(Path.Combine(_tempDir, "restored"));
new IncrementalBackup().Restore(restoredOptions, new[] { Path.Combine(_tempDir, "1.snapshot") });
using (var snapshotRestoreEnv = new StorageEnvironment(restoredOptions))
{
using (var tx = snapshotRestoreEnv.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("test");
Assert.NotNull(tree);
Assert.Equal("john doe", tree.Read("users/1").Reader.ToStringValue());
Assert.Equal("jane darling", tree.Read("users/2").Reader.ToStringValue());
Assert.Equal(new String('b', 5000), tree.Read("users/3").Reader.ToStringValue());
}
}
}
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_LOTS_of_transactions()
{
var transactionsToShip = new List <TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
shippingSourceEnv.CreateTree(tx, "TestTree2");
tx.Commit();
}
WriteLotsOfTestDataForTree("TestTree", shippingSourceEnv);
WriteLotsOfTestDataForTree("TestTree2", shippingSourceEnv);
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
{
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
}
shippingDestinationEnv.FlushLogToDataFile();
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
ValidateLotsOfTestDataForTree(snapshot, "TestTree");
ValidateLotsOfTestDataForTree(snapshot, "TestTree2");
}
}
}
private void InitializeDatabase()
{
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
_env.CreateTree(tx, LogsTreeName);
_env.CreateTree(tx, EntryTermsTreeName);
var metadata = _env.CreateTree(tx, MetadataTreeName);
var versionReadResult = metadata.Read("version");
if (versionReadResult == null) // new db
{
metadata.Add("version", Encoding.UTF8.GetBytes(CurrentVersion));
DbId = Guid.NewGuid();
metadata.Add("db-id", DbId.ToByteArray());
metadata.Add("current-term", EndianBitConverter.Little.GetBytes(0L));
metadata.Add("voted-for", Encoding.UTF8.GetBytes(string.Empty));
metadata.Add("voted-for-term", EndianBitConverter.Little.GetBytes(-1L));
metadata.Add("is-leader-potential", EndianBitConverter.Little.GetBytes(0));
}
else
{
var dbVersion = versionReadResult.Reader.ToStringValue();
if (dbVersion != CurrentVersion)
{
throw new InvalidOperationException("Cannot open db because its version is " + dbVersion +
" but the library expects version " + CurrentVersion);
}
int used;
var bytes = metadata.Read("db-id").Reader.ReadBytes(16, out used).Take(16).ToArray();
DbId = new Guid(bytes);
CurrentTerm = metadata.Read("current-term").Reader.ReadLittleEndianInt64();
var votedFor = metadata.Read("voted-for");
VotedFor = votedFor.Reader.Length == 0 ? null : votedFor.Reader.ToStringValue();
var votedForTerm = metadata.Read("voted-for-term");
VotedForTerm = votedForTerm.Reader.ReadLittleEndianInt64();
}
tx.Commit();
}
}
public void Can_make_multiple_min_inc_backups_and_then_restore()
{
const int UserCount = 5000;
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
int index = 0;
for (int xi = 0; xi < 5; xi++)
{
for (int yi = 0; yi < 2; yi++)
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < UserCount / 10; i++)
{
tree.Add("users/" + index, "john doe/" + index);
index++;
}
tx.Commit();
}
}
var snapshotWriter = new MinimalIncrementalBackup();
snapshotWriter.ToFile(envToSnapshot, Path.Combine(_tempDir, xi + ".snapshot"));
}
}
var incremental = new IncrementalBackup();
var restoredOptions = StorageEnvironmentOptions.ForPath(Path.Combine(_tempDir, "restored"));
incremental.Restore(restoredOptions, Enumerable.Range(0, 5).Select(i => Path.Combine(_tempDir, i + ".snapshot")));
using (var snapshotRestoreEnv = new StorageEnvironment(restoredOptions))
{
using (var tx = snapshotRestoreEnv.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("test");
Assert.NotNull(tree);
for (int i = 0; i < UserCount; i++)
{
var readResult = tree.Read("users/" + i);
Assert.NotNull(readResult);
Assert.Equal("john doe/" + i, readResult.Reader.ToStringValue());
}
}
}
}
public void ShouldOccupyLessSpace()
{
var r = new Random();
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 100; i++)
{
var bytes = new byte[r.Next(10, 2 * 1024 * 1024)];
r.NextBytes(bytes);
tree.Add("record/" + i, bytes);
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 50; i++)
{
tree.Delete("record/" + r.Next(0, 100));
}
tx.Commit();
}
}
var oldSize = GetDirSize(new DirectoryInfo(CompactionTestsData));
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
var newSize = GetDirSize(new DirectoryInfo(CompactedData));
Assert.True(newSize < oldSize, string.Format("Old size: {0:#,#;;0} MB, new size {1:#,#;;0} MB", oldSize / 1024 / 1024, newSize / 1024 / 1024));
}
public void ShouldBeAbleToWriteValuesGreaterThanLogAndRecoverThem()
{
DeleteDirectory("test2.data");
var random = new Random(1234);
var buffer = new byte[1024 * 512];
random.NextBytes(buffer);
var options = StorageEnvironmentOptions.ForPath("test2.data");
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
tx.Environment.CreateTree(tx, "tree").Add("key1", new MemoryStream(buffer));
tx.Commit();
}
}
options = StorageEnvironmentOptions.ForPath("test2.data");
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var read = tx.Environment.CreateTree(tx, "tree").Read("key1");
Assert.NotNull(read);
{
Assert.Equal(buffer.Length, read.Reader.Length);
int used;
Assert.Equal(buffer, read.Reader.ReadBytes(read.Reader.Length, out used).Take(used).ToArray());
}
}
}
DeleteDirectory("test2.data");
}
public void ShouldReportProgress()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
var tree2 = env.CreateTree(tx, "vegetables");
tree2.Add("carrot", new byte[123]);
tree2.Add("potato", new byte[99]);
var tree3 = env.CreateTree(tx, "multi");
tree3.MultiAdd("fruits", "apple");
tree3.MultiAdd("fruits", "orange");
tree3.MultiAdd("vegetables", "carrot");
tree3.MultiAdd("vegetables", "carrot");
tx.Commit();
}
}
var progressReport = new List <string>();
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData), x => progressReport.Add(string.Format("Copied {0} of {1} records in '{2}' tree. Copied {3} of {4} trees.", x.CopiedTreeRecords, x.TotalTreeRecordsCount, x.TreeName, x.CopiedTrees, x.TotalTreeCount)));
Assert.NotEmpty(progressReport);
Assert.Contains("Copied 0 of 2 records in 'fruits' tree. Copied 0 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'fruits' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'multi' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'multi' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'vegetables' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'vegetables' tree. Copied 3 of 3 trees.", progressReport);
}
private void Initialize()
{
using (var tx = storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
var serverNamesToIds = storageEnvironment.CreateTree(tx, "serverNames->Ids");
var serverIdsToNames = storageEnvironment.CreateTree(tx, "Ids->serverNames");
storageEnvironment.CreateTree(tx, "servers->lastEtag");
storageEnvironment.CreateTree(tx, "counters");
storageEnvironment.CreateTree(tx, "countersGroups");
var etags = storageEnvironment.CreateTree(tx, "etags->counters");
storageEnvironment.CreateTree(tx, "counters->etags");
var metadata = tx.State.GetTree(tx, "$metadata");
var id = metadata.Read("id");
if (id == null) // new counter db
{
var serverIdBytes = EndianBitConverter.Big.GetBytes(ServerId);
var serverIdSlice = new Slice(serverIdBytes);
serverNamesToIds.Add(CounterStorageUrl, serverIdSlice);
serverIdsToNames.Add(serverIdSlice, CounterStorageUrl);
Id = Guid.NewGuid();
metadata.Add("id", Id.ToByteArray());
metadata.Add("name", Encoding.UTF8.GetBytes(Name));
tx.Commit();
}
else // existing counter db
{
int used;
Id = new Guid(id.Reader.ReadBytes(16, out used));
var nameResult = metadata.Read("name");
if (nameResult == null)
{
throw new InvalidOperationException("Could not read name from the store, something bad happened");
}
var storedName = new StreamReader(nameResult.Reader.AsStream()).ReadToEnd();
if (storedName != Name)
{
throw new InvalidOperationException("The stored name " + storedName + " does not match the given name " + Name);
}
using (var it = etags.Iterate())
{
if (it.Seek(Slice.AfterAllKeys))
{
LastEtag = it.CurrentKey.CreateReader().ReadBigEndianInt64();
}
}
}
ReplicationTask.StartReplication();
}
}
public void ShouldBeAbleToWriteValuesGreaterThanLogAndRecoverThem()
{
DeleteDirectory("test2.data");
var random = new Random(1234);
var buffer = new byte[1024 * 512];
random.NextBytes(buffer);
var options = StorageEnvironmentOptions.ForPath("test2.data");
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
tx.Environment.State.GetTree(tx,"tree").Add("key1", new MemoryStream(buffer));
tx.Commit();
}
}
options = StorageEnvironmentOptions.ForPath("test2.data");
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var read = tx.Environment.State.GetTree(tx,"tree").Read("key1");
Assert.NotNull(read);
{
Assert.Equal(buffer.Length, read.Reader.Length);
int used;
Assert.Equal(buffer, read.Reader.ReadBytes(read.Reader.Length, out used).Take(used).ToArray());
}
}
}
DeleteDirectory("test2.data");
}
public void Can_make_multiple_min_inc_backups_and_then_restore()
{
const int UserCount = 5000;
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
int index = 0;
for (int xi = 0; xi < 5; xi++)
{
for (int yi = 0; yi < 2; yi++)
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < UserCount / 10; i++)
{
tree.Add("users/" + index, "john doe/" + index);
index++;
}
tx.Commit();
}
}
var snapshotWriter = new MinimalIncrementalBackup();
snapshotWriter.ToFile(envToSnapshot, Path.Combine(_tempDir, xi + ".snapshot"));
}
}
var incremental = new IncrementalBackup();
var restoredOptions = StorageEnvironmentOptions.ForPath(Path.Combine(_tempDir, "restored"));
incremental.Restore(restoredOptions, Enumerable.Range(0, 5).Select(i => Path.Combine(_tempDir, i + ".snapshot")));
using (var snapshotRestoreEnv = new StorageEnvironment(restoredOptions))
{
using (var tx = snapshotRestoreEnv.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("test");
Assert.NotNull(tree);
for (int i = 0; i < UserCount; i++)
{
var readResult = tree.Read("users/" + i);
Assert.NotNull(readResult);
Assert.Equal("john doe/" + i, readResult.Reader.ToStringValue());
}
}
}
}
public void Replay()
{
var wasDebugRecording = _env.IsDebugRecording;
_env.IsDebugRecording = false;
using (var writeBatch = new WriteBatch())
{
ActivityEntry entry;
while (WriteQueue.TryDequeue(out entry))
{
switch (entry.ActionType)
{
case DebugActionType.Add:
writeBatch.Add(entry.Key, entry.ValueStream, entry.TreeName);
break;
case DebugActionType.Delete:
writeBatch.Delete(entry.Key, entry.TreeName);
break;
case DebugActionType.MultiAdd:
writeBatch.MultiAdd(entry.Key, new Slice(Encoding.UTF8.GetBytes(entry.Value.ToString())), entry.TreeName);
break;
case DebugActionType.MultiDelete:
writeBatch.MultiDelete(entry.Key, new Slice(Encoding.UTF8.GetBytes(entry.Value.ToString())), entry.TreeName);
break;
case DebugActionType.CreateTree:
using (var tx = _env.NewTransaction(TransactionFlags.ReadWrite))
{
_env.CreateTree(tx, entry.TreeName);
tx.Commit();
}
break;
case DebugActionType.Increment:
//TODO : make sure this is correct here
writeBatch.Increment(entry.Key, entry.ValueStream.ReadByte(), entry.TreeName);
break;
default: //precaution against newly added action types
throw new InvalidOperationException("unsupported tree action type");
}
}
_env.Writer.Write(writeBatch);
}
_env.IsDebugRecording = wasDebugRecording; //restore the state as it was
}
public DateTimeSeries(string path)
{
_lastKey = "last-key";
_storageEnvironment = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path));
using (var tx = _storageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
_storageEnvironment.CreateTree(tx, "data");
var read = tx.State.Root.Read(tx, _lastKey);
_last = read != null?read.Reader.ReadInt64() : 1;
tx.Commit();
}
}
protected IList <string> CreateTrees(StorageEnvironment env, int number, string prefix)
{
var results = new List <string>();
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < number; i++)
{
results.Add(env.CreateTree(tx, prefix + i).Name);
}
tx.Commit();
}
return(results);
}
public FullTextIndex(StorageEnvironmentOptions options, IAnalyzer analyzer)
{
Analyzer = analyzer;
Conventions = new IndexingConventions();
BufferPool = new BufferPool();
StorageEnvironment = new StorageEnvironment(options);
using (var tx = StorageEnvironment.NewTransaction(TransactionFlags.ReadWrite))
{
StorageEnvironment.CreateTree(tx, "TermPositions");
ReadMetadata(tx);
ReadLastDocumentId(tx);
ReadFields(tx);
tx.Commit();
}
}
public void SurviveRestart()
{
using (var options = StorageEnvironmentOptions.CreateMemoryOnly())
{
options.OwnsPagers = false;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "events");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
tx.Environment.State.GetTree(tx,"events").Add("test", new MemoryStream(0));
tx.Commit();
}
}
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "events");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.Environment.State.GetTree(tx,"events");
var readResult = tree.Read("test");
Assert.NotNull(readResult);
tx.Commit();
}
}
}
}
public void StorageRecoveryShouldWorkWhenThereSingleTransactionToRecoverFromLog()
{
var path = "test2.data";
DeleteDirectory(path);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "tree");
for (var i = 0; i < 100; i++)
{
tree.Add("key" + i, new MemoryStream());
}
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.Environment.State.GetTree(tx,"tree");
for (var i = 0; i < 100; i++)
{
Assert.NotNull(tree.Read("key" + i));
}
}
}
DeleteDirectory(path);
}
public void DataIsKeptAfterRestartForSubTrees()
{
using (var pureMemoryPager = StorageEnvironmentOptions.CreateMemoryOnly())
{
pureMemoryPager.OwnsPagers = false;
using (var env = new StorageEnvironment(pureMemoryPager))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.Environment.CreateTree(tx,"test");
tree.Add("test", Stream.Null);
tx.Commit();
Assert.NotNull(tree.Read("test"));
}
}
using (var env = new StorageEnvironment(pureMemoryPager))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "test");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.Environment.CreateTree(tx,"test");
Assert.NotNull(tree.Read("test"));
tx.Commit();
}
}
}
}
public void MultiAdds_And_MultiDeletes_After_Causing_PageSplit_DoNot_Fail(int size)
{
using (var Env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
var inputData = new List<byte[]>();
for (int i = 0; i < size; i++)
{
inputData.Add(Encoding.UTF8.GetBytes(RandomString(1024)));
}
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
Env.CreateTree(tx, "foo");
tx.Commit();
}
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.Environment.State.GetTree(tx,"foo");
foreach (var buffer in inputData)
{
Assert.DoesNotThrow(() => tree.MultiAdd(tx, "ChildTreeKey", new Slice(buffer)));
}
tx.Commit();
}
using (var tx = Env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.Environment.State.GetTree(tx,"foo");
for (int i = 0; i < inputData.Count; i++)
{
var buffer = inputData[i];
Assert.DoesNotThrow(() => tree.MultiDelete(tx, "ChildTreeKey", new Slice(buffer)));
}
tx.Commit();
}
}
}
public void ShouldCorrectlyLoadAfterRestartIfIncrementalBackupWasDone()
{
var bytes = new byte[1024];
new Random().NextBytes(bytes);
using (var env = new StorageEnvironment(ModifyOptions(StorageEnvironmentOptions.ForPath("Data"))))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "items");
tx.Commit();
}
for (int j = 0; j < 100; j++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = tx.ReadTree("items");
for (int i = 0; i < 100; i++)
{
tree.Add("items/" + i, bytes);
}
tx.Commit();
}
}
BackupMethods.Incremental.ToFile(env, IncrementalBackupTestUtils.IncrementalBackupFile(0));
}
// restart
using (var env = new StorageEnvironment(ModifyOptions(StorageEnvironmentOptions.ForPath("Data"))))
{
}
}
public void CannotCompactStorageIfIncrementalBackupEnabled()
{
var envOptions = StorageEnvironmentOptions.ForPath(CompactionTestsData);
envOptions.IncrementalBackupEnabled = true;
using (var env = new StorageEnvironment(envOptions))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
tree.Add("record/1", new byte[9]);
tree.Add("record/2", new byte[9]);
tx.Commit();
}
}
var srcOptions = StorageEnvironmentOptions.ForPath(CompactionTestsData);
srcOptions.IncrementalBackupEnabled = true;
var invalidOperationException = Assert.Throws<InvalidOperationException>(() => StorageCompaction.Execute(srcOptions, (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData)));
Assert.Equal(StorageCompaction.CannotCompactBecauseOfIncrementalBackup, invalidOperationException.Message);
}
public void StorageRecoveryShouldWorkForSplitTransactions()
{
var random = new Random(1234);
var buffer = new byte[4096];
random.NextBytes(buffer);
var path = "test2.data";
var count = 1000;
DeleteDirectory(path);
var options = StorageEnvironmentOptions.ForPath(path);
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "atree");
env.CreateTree(tx, "btree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var aTree = tx.Environment.State.GetTree(tx,"atree");
var bTree = tx.Environment.State.GetTree(tx,"btree");
for (var i = 0; i < count; i++)
{
aTree.Add("a" + i, new MemoryStream(buffer));
bTree.MultiAdd("a", "a" + i);
}
tx.Commit();
}
}
var expectedString = Encoding.UTF8.GetString(buffer);
options = StorageEnvironmentOptions.ForPath(path);
options.MaxLogFileSize = 10 * AbstractPager.PageSize;
using (var env = new StorageEnvironment(options))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "atree");
env.CreateTree(tx, "btree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var aTree = tx.Environment.State.GetTree(tx,"atree");
var bTree = tx.Environment.State.GetTree(tx,"btree");
for (var i = 0; i < count; i++)
{
var read = aTree.Read("a" + i);
Assert.NotNull(read);
Assert.Equal(expectedString, read.Reader.ToStringValue());
}
using (var iterator = bTree.MultiRead("a"))
{
Assert.True(iterator.Seek(Slice.BeforeAllKeys));
var keys = new HashSet<string>();
do
{
keys.Add(iterator.CurrentKey.ToString());
}
while (iterator.MoveNext());
Assert.Equal(count, keys.Count);
}
}
}
DeleteDirectory(path);
}
public unsafe void Min_inc_backup_is_smaller_than_normal_inc_backup()
{
const int UserCount = 5000;
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
for (int xi = 0; xi < 10; xi++)
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < UserCount / 10; i++)
{
tree.Add("users/" + i, "john doe/" + i);
}
tx.Commit();
}
}
var incrementalBackupInfo = envToSnapshot.HeaderAccessor.Get(ptr => ptr->IncrementalBackup);
var snapshotWriter = new MinimalIncrementalBackup();
snapshotWriter.ToFile(envToSnapshot, Path.Combine(_tempDir, "1.snapshot"));
// reset the incremental backup stuff
envToSnapshot.HeaderAccessor.Modify(ptr => ptr->IncrementalBackup = incrementalBackupInfo);
var incBackup = new IncrementalBackup();
incBackup.ToFile(envToSnapshot, Path.Combine(_tempDir, "2.snapshot"));
var incLen = new FileInfo(Path.Combine(_tempDir, "2.snapshot")).Length;
var minInLen = new FileInfo(Path.Combine(_tempDir, "1.snapshot")).Length;
Assert.True(incLen > minInLen);
}
}
public void Can_use_full_back_then_full_min_backup()
{
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < 1000; i++)
{
tree.Add("users/" + i, "first/" + i);
}
tx.Commit();
}
new FullBackup().ToFile(envToSnapshot, Path.Combine(_tempDir, "full.backup"));
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < 500; i++)
{
tree.Add("users/" + i, "second/" + (i * 2));
}
for (int i = 0; i < 500; i++)
{
tree.Add("users/" + (i + 10000), "third/" + i);
}
tx.Commit();
}
new MinimalIncrementalBackup().ToFile(envToSnapshot, Path.Combine(_tempDir, "1.backup"));
}
new FullBackup().Restore(Path.Combine(_tempDir, "full.backup"), Path.Combine(_tempDir, "restored"));
var restoredOptions = StorageEnvironmentOptions.ForPath(Path.Combine(_tempDir, "restored"));
new IncrementalBackup().Restore(restoredOptions, new[] { Path.Combine(_tempDir, "1.backup") });
using (var snapshotRestoreEnv = new StorageEnvironment(restoredOptions))
{
using (var tx = snapshotRestoreEnv.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("test");
Assert.NotNull(tree);
for (int i = 0; i < 500; i++)
{
var readResult = tree.Read("users/" + i);
Assert.NotNull(readResult);
Assert.Equal("second/" + (i * 2), readResult.Reader.ToStringValue());
}
for (int i = 0; i < 500; i++)
{
var readResult = tree.Read("users/" + (i + 10000));
Assert.NotNull(readResult);
Assert.Equal("third/" + i, readResult.Reader.ToStringValue());
}
for (int i = 0; i < 500; i++)
{
var readResult = tree.Read("users/" + (i + 500));
Assert.NotNull(readResult);
Assert.Equal("first/" + (i+500), readResult.Reader.ToStringValue());
}
}
}
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_LOTS_of_transactions()
{
var transactionsToShip = new List<TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
shippingSourceEnv.CreateTree(tx, "TestTree2");
tx.Commit();
}
WriteLotsOfTestDataForTree("TestTree", shippingSourceEnv);
WriteLotsOfTestDataForTree("TestTree2", shippingSourceEnv);
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
shippingDestinationEnv.FlushLogToDataFile();
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
ValidateLotsOfTestDataForTree(snapshot, "TestTree");
ValidateLotsOfTestDataForTree(snapshot, "TestTree2");
}
}
}
public void CompactionMustNotLooseAnyData()
{
var treeNames = new List<string>();
var multiValueTreeNames = new List<string>();
var random = new Random();
var value1 = new byte[random.Next(1024*1024*2)];
var value2 = new byte[random.Next(1024*1024*2)];
random.NextBytes(value1);
random.NextBytes(value2);
const int treeCount = 5;
const int recordCount = 6;
const int multiValueTreeCount = 7;
const int multiValueRecordsCount = 4;
const int multiValuesCount = 3;
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
for (int i = 0; i < treeCount; i++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
string name = "tree/" + i;
treeNames.Add(name);
var tree = env.State.GetTree(tx, name);
for (int j = 0; j < recordCount; j++)
{
tree.Add(string.Format("{0}/items/{1}", name, j), j%2 == 0 ? value1 : value2);
}
tx.Commit();
}
}
for (int i = 0; i < multiValueTreeCount; i++)
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var name = "multiValueTree/" + i;
multiValueTreeNames.Add(name);
var tree = env.CreateTree(tx, name);
for (int j = 0; j < multiValueRecordsCount; j++)
{
for (int k = 0; k < multiValuesCount; k++)
{
tree.MultiAdd("record/" + j, "value/" + k);
}
}
tx.Commit();
}
}
}
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactedData)))
{
using (var tx = compacted.NewTransaction(TransactionFlags.Read))
{
foreach (var treeName in treeNames)
{
var tree = compacted.State.GetTree(tx, treeName);
for (int i = 0; i < recordCount; i++)
{
var readResult = tree.Read(string.Format("{0}/items/{1}", treeName, i));
Assert.NotNull(readResult);
if (i%2 == 0)
{
var readBytes = new byte[value1.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
Assert.Equal(value1, readBytes);
}
else
{
var readBytes = new byte[value2.Length];
readResult.Reader.Read(readBytes, 0, readBytes.Length);
Assert.Equal(value2, readBytes);
}
}
}
foreach (var treeName in multiValueTreeNames)
{
var tree = compacted.State.GetTree(tx, treeName);
for (int i = 0; i < multiValueRecordsCount; i++)
{
var multiRead = tree.MultiRead("record/" + i);
Assert.True(multiRead.Seek(Slice.BeforeAllKeys));
int count = 0;
do
{
Assert.Equal("value/" + count, multiRead.CurrentKey.ToString());
count++;
} while (multiRead.MoveNext());
Assert.Equal(multiValuesCount, count);
}
}
}
}
}
public void ShouldOccupyLessSpace()
{
var r = new Random();
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 100; i++)
{
var bytes = new byte[r.Next(10, 2*1024*1024)];
r.NextBytes(bytes);
tree.Add("record/" + i, bytes);
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "records");
for (int i = 0; i < 50; i++)
{
tree.Delete("record/" + r.Next(0, 100));
}
tx.Commit();
}
}
var oldSize = GetDirSize(new DirectoryInfo(CompactionTestsData));
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions) StorageEnvironmentOptions.ForPath(CompactedData));
var newSize = GetDirSize(new DirectoryInfo(CompactedData));
Assert.True(newSize < oldSize, string.Format("Old size: {0:#,#;;0} MB, new size {1:#,#;;0} MB", oldSize / 1024 / 1024, newSize / 1024 / 1024));
}
public void Record_debug_journal_and_replay_it_with_manual_flushing()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
env.Writer.Write(writeBatch);
}
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
using (env.Options.AllowManualFlushing())
{
env.FlushLogToDataFile(tx);
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
}
}
}
private static void CopyTrees(StorageEnvironment existingEnv, StorageEnvironment compactedEnv, Action<CompactionProgress> progressReport = null)
{
using (var rootIterator = existingEnv.State.Root.Iterate())
{
if (rootIterator.Seek(Slice.BeforeAllKeys) == false)
return;
var totalTreesCount = existingEnv.State.Root.State.EntriesCount;
var copiedTrees = 0L;
do
{
var treeName = rootIterator.CurrentKey.ToString();
using (var txr = existingEnv.NewTransaction(TransactionFlags.Read))
{
var existingTree = existingEnv.State.GetTree(txr, treeName);
Report(treeName, copiedTrees, totalTreesCount, 0, existingTree.State.EntriesCount, progressReport);
using (var existingTreeIterator = existingTree.Iterate())
{
if (existingTreeIterator.Seek(Slice.BeforeAllKeys) == false)
continue;
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
compactedEnv.CreateTree(txw, treeName);
txw.Commit();
}
var copiedEntries = 0L;
do
{
var transactionSize = 0L;
using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
{
var newTree = txw.ReadTree(treeName);
do
{
var key = existingTreeIterator.CurrentKey;
if (existingTreeIterator.Current->Flags == NodeFlags.MultiValuePageRef)
{
using (var multiTreeIterator = existingTree.MultiRead(key))
{
if (multiTreeIterator.Seek(Slice.BeforeAllKeys) == false)
continue;
do
{
var multiValue = multiTreeIterator.CurrentKey;
newTree.MultiAdd(key, multiValue);
transactionSize += multiValue.Size;
} while (multiTreeIterator.MoveNext());
}
}
else
{
using (var value = existingTree.Read(key).Reader.AsStream())
{
newTree.Add(key, value);
transactionSize += value.Length;
}
}
copiedEntries++;
} while (transactionSize < compactedEnv.Options.MaxLogFileSize/2 && existingTreeIterator.MoveNext());
txw.Commit();
}
if (copiedEntries == existingTree.State.EntriesCount)
copiedTrees++;
Report(treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.EntriesCount, progressReport);
compactedEnv.FlushLogToDataFile();
} while (existingTreeIterator.MoveNext());
}
}
} while (rootIterator.MoveNext());
}
}
public void StorageRecoveryShouldWorkWhenThereAreMultipleCommitedTransactions2()
{
var path = "test2.data";
DeleteDirectory(path);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "atree");
for (var i = 0; i < 1000; i++)
{
tree.Add("key" + i, new MemoryStream());
}
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "btree");
for (var i = 0; i < 5; i++)
{
tree.Add("key" + i, new MemoryStream());
}
tx.Commit();
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "atree");
env.CreateTree(tx, "btree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var aTree = tx.Environment.CreateTree(tx,"atree");
var bTree = tx.Environment.CreateTree(tx,"btree");
for (var i = 0; i < 1000; i++)
{
Assert.NotNull(aTree.Read("key" + i));
}
for (var i = 0; i < 5; i++)
{
Assert.NotNull(bTree.Read("key" + i));
}
}
}
DeleteDirectory(path);
}
public void StorageRecoveryShouldWorkWhenThereAreCommitedAndUncommitedTransactions()
{
var path = "test2.data";
DeleteDirectory(path);
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "tree");
tx.Commit();
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < 10000; i++)
{
tx.Environment.State.GetTree(tx,"tree").Add("a" + i, new MemoryStream());
}
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(path)))
{
}
DeleteDirectory(path);
}
public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_new_trees_no_flushing()
{
var transactionsToShip = new ConcurrentBag<TransactionToShip>();
using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
shippingSourceEnv.Journal.OnTransactionCommit += tx =>
{
tx.CreatePagesSnapshot();
transactionsToShip.Add(tx);
};
using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
{
shippingSourceEnv.CreateTree(tx, "TestTree");
tx.Commit();
}
}
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
{
foreach (var tx in transactionsToShip)
shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);
using (var snapshot = shippingDestinationEnv.CreateSnapshot())
{
snapshot.Read("TestTree", "Foo");
}
}
}
public void SplitterIssue2()
{
var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
storageEnvironmentOptions.ManualFlushing = true;
using (var env = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "multi");
tx.Commit();
}
var batch = new WriteBatch();
batch.MultiAdd("0", "1", "multi");
batch.MultiAdd("1", "1", "multi");
batch.MultiAdd("2", "1", "multi");
batch.MultiAdd("3", "1", "multi");
batch.MultiAdd("4", "1", "multi");
batch.MultiAdd("5", "1", "multi");
env.Writer.Write(batch);
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.Environment.State.GetTree(tx,"multi");
using (var iterator = tree.MultiRead(tx, "0"))
{
Assert.True(iterator.Seek(Slice.BeforeAllKeys));
var count = 0;
do
{
count++;
} while (iterator.MoveNext());
Assert.Equal(1, count);
}
}
batch = new WriteBatch();
batch.MultiAdd("0", "2", "multi");
batch.MultiAdd("1", "2", "multi");
batch.MultiAdd("2", "2", "multi");
batch.MultiAdd("3", "2", "multi");
batch.MultiAdd("4", "2", "multi");
batch.MultiAdd("5", "2", "multi");
env.Writer.Write(batch);
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
var tree = tx.Environment.State.GetTree(tx,"multi");
using (var iterator = tree.MultiRead(tx, "0"))
{
Assert.True(iterator.Seek(Slice.BeforeAllKeys));
var count = 0;
do
{
count++;
} while (iterator.MoveNext());
Assert.Equal(2, count);
}
}
}
}
public void ShouldDeleteCurrentJournalEvenThoughItHasAvailableSpace()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
tx.Commit();
}
}
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData));
var compactedDir = new DirectoryInfo(CompactedData);
var journalsAfterCompaction = compactedDir.GetFiles("*.journal").Select(x => x.Name).ToList();
Assert.Equal(0, journalsAfterCompaction.Count);
// ensure it can write more data
using (var compacted = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactedData)))
{
using (var tx = compacted.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = compacted.CreateTree(tx, "fruits");
tree.Add("peach", new byte[144]);
}
}
}
private IList<string> CreateTrees(StorageEnvironment env, int number, string prefix)
{
var results = new List<string>();
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
for (var i = 0; i < number; i++)
{
results.Add(env.CreateTree(tx, prefix + i).Name);
}
tx.Commit();
}
return results;
}
public void ShouldReportProgress()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.ForPath(CompactionTestsData)))
{
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = env.CreateTree(tx, "fruits");
tree.Add("apple", new byte[123]);
tree.Add("orange", new byte[99]);
var tree2 = env.CreateTree(tx, "vegetables");
tree2.Add("carrot", new byte[123]);
tree2.Add("potato", new byte[99]);
var tree3 = env.CreateTree(tx, "multi");
tree3.MultiAdd("fruits", "apple");
tree3.MultiAdd("fruits", "orange");
tree3.MultiAdd("vegetables", "carrot");
tree3.MultiAdd("vegetables", "carrot");
tx.Commit();
}
}
var progressReport = new List<string>();
StorageCompaction.Execute(StorageEnvironmentOptions.ForPath(CompactionTestsData), (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(CompactedData), x => progressReport.Add(string.Format("Copied {0} of {1} records in '{2}' tree. Copied {3} of {4} trees.", x.CopiedTreeRecords, x.TotalTreeRecordsCount, x.TreeName, x.CopiedTrees, x.TotalTreeCount)));
Assert.NotEmpty(progressReport);
Assert.Contains("Copied 0 of 2 records in 'fruits' tree. Copied 0 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'fruits' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'multi' tree. Copied 1 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'multi' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 0 of 2 records in 'vegetables' tree. Copied 2 of 3 trees.", progressReport);
Assert.Contains("Copied 2 of 2 records in 'vegetables' tree. Copied 3 of 3 trees.", progressReport);
}
public void Record_debug_journal_and_replay_it()
{
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = new DebugJournal(debugJouralName, env, true);
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
writeBatch.Add("foo", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree");
writeBatch.MultiAdd("multi-foo", "AA", "test-tree");
env.Writer.Write(writeBatch);
}
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
using (var writeBatch = new WriteBatch())
{
writeBatch.Increment("incr-key", 5, "test-tree");
env.Writer.Write(writeBatch);
}
Assert.Equal(5, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var tx = env.NewTransaction(TransactionFlags.Read))
{
Assert.Equal(10, tx.ReadTree("test-tree").Read("incr-key").Reader.ReadLittleEndianInt64());
}
using (var writeBatch = new WriteBatch())
{
writeBatch.MultiAdd("multi-foo", "BB", "test-tree");
writeBatch.MultiAdd("multi-foo", "CC", "test-tree");
writeBatch.Delete("foo-bar", "test-tree");
env.Writer.Write(writeBatch);
}
using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
{
env.CreateTree(tx, "test-tree2");
tx.Commit();
}
using (var writeBatch = new WriteBatch())
{
var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1!"));
writeBatch.Add("foo", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2!"));
writeBatch.Add("bar", valueBuffer, "test-tree2");
valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("testing testing 1 2 3!"));
writeBatch.Add("foo-bar", valueBuffer, "test-tree2");
env.Writer.Write(writeBatch);
}
}
using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
{
env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
env.DebugJournal.Replay();
using (var snapshot = env.CreateSnapshot())
{
Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1!", snapshot.Read("test-tree2", "foo").Reader.ToStringValue());
Assert.Equal("testing testing 1 2!", snapshot.Read("test-tree2", "bar").Reader.ToStringValue());
Assert.Equal("testing testing 1 2 3!", snapshot.Read("test-tree2", "foo-bar").Reader.ToStringValue());
Assert.Equal(10, snapshot.Read("test-tree", "incr-key").Reader.ReadLittleEndianInt64());
Assert.Equal(0,snapshot.ReadVersion("test-tree","foo-bar"));
using (var iter = snapshot.MultiRead("test-tree","multi-foo"))
{
iter.Seek(Slice.BeforeAllKeys);
Assert.Equal("AA",iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("BB",iter.CurrentKey.ToString());
Assert.DoesNotThrow(() => iter.MoveNext());
Assert.Equal("CC",iter.CurrentKey.ToString());
}
}
}
}
public void Mixed_small_and_overflow_changes()
{
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
tree.Add("users/1", "john doe");
tree.Add("users/2", new String('a', 5000));
tx.Commit();
}
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
tree.Add("users/2", "jane darling");
tree.Add("users/3", new String('b', 5000));
tx.Commit();
}
var snapshotWriter = new MinimalIncrementalBackup();
snapshotWriter.ToFile(envToSnapshot, Path.Combine(_tempDir, "1.snapshot"));
var restoredOptions = StorageEnvironmentOptions.ForPath(Path.Combine(_tempDir, "restored"));
new IncrementalBackup().Restore(restoredOptions, new[] { Path.Combine(_tempDir, "1.snapshot") });
using (var snapshotRestoreEnv = new StorageEnvironment(restoredOptions))
{
using (var tx = snapshotRestoreEnv.NewTransaction(TransactionFlags.Read))
{
var tree = tx.ReadTree("test");
Assert.NotNull(tree);
Assert.Equal("john doe", tree.Read("users/1").Reader.ToStringValue());
Assert.Equal("jane darling", tree.Read("users/2").Reader.ToStringValue());
Assert.Equal(new String('b', 5000), tree.Read("users/3").Reader.ToStringValue());
}
}
}
}
public void Can_split_merged_transaction_to_multiple_tx()
{
_tempDir = Guid.NewGuid().ToString();
var storageEnvironmentOptions = StorageEnvironmentOptions.ForPath(_tempDir);
storageEnvironmentOptions.IncrementalBackupEnabled = true;
storageEnvironmentOptions.MaxNumberOfPagesInMergedTransaction = 8;
using (var envToSnapshot = new StorageEnvironment(storageEnvironmentOptions))
{
for (int xi = 0; xi < 100; xi++)
{
using (var tx = envToSnapshot.NewTransaction(TransactionFlags.ReadWrite))
{
var tree = envToSnapshot.CreateTree(tx, "test");
for (int i = 0; i < 1000; i++)
{
tree.Add("users/" + i, "john doe/" + i);
}
tx.Commit();
}
}
var snapshotWriter = new MinimalIncrementalBackup();
var backupPath = Path.Combine(_tempDir, "1.snapshot");
snapshotWriter.ToFile(envToSnapshot, backupPath);
using (var stream = File.OpenRead(backupPath))
using (var zip = new ZipArchive(stream, ZipArchiveMode.Read))
{
Assert.True(zip.Entries.Count > 1);
}
}
}