public void Write_the_same_object_twice()
{
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(0, storage.BlockCount);
storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150);
storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150);
Assert.AreEqual(1, storage.BlockCount);
Assert.AreEqual(0, storage.InactiveBlockCount);
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(1, storage.BlockCount);
Assert.AreEqual(0, storage.InactiveBlockCount);
}
}
public void Write_then_reload_object()
{
var data = new byte[] { 1, 2, 3 };
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
storage.StoreBlock(data, "a1", 150);
Assert.AreEqual(1, storage.BlockCount);
}
var processor = new NullProcessor();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
Assert.AreEqual(1, storage.BlockCount);
}
Assert.AreEqual(1, processor.ProcessedBlocks.Count);
CollectionAssert.AreEqual(data, processor.ProcessedBlocks[0]);
}
public void Persistence_of_simple_put_transaction_with_one_object()
{
var transaction1 = MakeTransaction(new Trade(1, 5465, "TATA", DateTime.Now.Date, 150));
var engine = new PersistenceEngine();
engine.Start();
engine.NewTransaction(transaction1);
// wait for the transaction log to be processed
engine.WaitForPendingTransactions();
engine.Stop();
var processor = new TestProcessor();
//reload data from persistent storage
var unused = new ReliableStorage(processor);
unused.LoadPersistentData();
unused.Dispose();
Assert.AreEqual(1, processor.LoadedObjects.Count);
var reloaded = CachedObject.Unpack <Trade>(processor.LoadedObjects[0]);
Assert.AreEqual("TATA", reloaded.Folder);
}
public void Test_recovery_of_corrupted_storage()
{
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(0, storage.BlockCount);
storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150);
storage.StoreBlock(new byte[] { 1, 2, 3, 4, 5 }, "a2", 151);
storage.StoreBlock(new byte[] { 21, 22, 23, 24, 25 }, "a3", 152);
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(0, storage.InactiveBlockCount);
storage.MakeCorruptedBlock("a2");
}
// here we apply recovery wile loading
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(2, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
Assert.AreEqual(2, storage.Keys.Count);
Assert.AreEqual(1, storage.CorruptedBlocks);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a3"));
}
// here it is clean again (one block was lost)
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(2, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
Assert.AreEqual(0, storage.CorruptedBlocks);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a3"));
}
}
Add_two_objects_then_update_one_of_them_then_check_the_persistent_storage_contains_the_updated_value()
{
var transaction1 = MakeTransaction(
new Trade(1, 5465, "TATA", DateTime.Now.Date, 150),
new Trade(2, 5467, "TATA", DateTime.Now.Date, 180)
);
var transaction2 = MakeTransaction(
new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190)
);
var deleteTransaction = MakeDeleteTransaction(
new Trade(1, 5465, "TATA", DateTime.Now.Date, 150)
);
var engine = new PersistenceEngine();
engine.Start();
engine.NewTransaction(transaction1);
engine.NewTransaction(transaction2);
engine.NewTransaction(deleteTransaction);
// wait for the transaction log to be processed
engine.WaitForPendingTransactions();
engine.Stop();
var processor = new TestProcessor();
//reload data from persistent storage
var unused = new ReliableStorage(processor);
unused.LoadPersistentData();
unused.Dispose();
Assert.AreEqual(1, processor.LoadedObjects.Count);
var reloaded = processor.LoadedObjects.Select(CachedObject.Unpack <Trade>).First(t => t.Id == 2);
Assert.AreEqual("TOTO", reloaded.Folder);
//reload data from transaction log. Check no more pending transactions
var log = new TransactionLog();
Assert.AreEqual(0, log.PendingTransactionsCount);
log.Dispose();
}
public void Simulate_failure_during_transaction_processing()
{
var transaction1 = MakeTransaction(
new Trade(1, 5465, "TATA", DateTime.Now.Date, 150),
new Trade(2, 5467, "TATA", DateTime.Now.Date, 180)
);
var transaction2 = MakeTransaction(
new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190)
);
var log = new TransactionLog();
log.NewTransaction(
SerializationHelper.ObjectToBytes(transaction1, SerializationMode.ProtocolBuffers, null));
log.NewTransaction(
SerializationHelper.ObjectToBytes(transaction2, SerializationMode.ProtocolBuffers, null));
var dummy = log.StartProcessing(); // call StartProcessing and close before EndProcessing
log.Dispose();
var engine = new PersistenceEngine();
engine.Start();
engine.WaitForPendingTransactions();
engine.Stop();
var processor = new TestProcessor();
//reload data from persistent storage
var unused = new ReliableStorage(processor);
unused.LoadPersistentData();
unused.Dispose();
Assert.AreEqual(2, processor.LoadedObjects.Count);
var reloaded = processor.LoadedObjects.Select(CachedObject.Unpack <Trade>).First(t => t.Id == 2);
Assert.AreEqual("TOTO", reloaded.Folder);
//reload data from transaction log. Check no more pending transactions
log = new TransactionLog();
Assert.AreEqual(0, log.PendingTransactionsCount);
log.Dispose();
}
public void Start_the_persistence_engine_with_a_non_empty_transaction_log()
{
var transaction1 = MakeTransaction(
new Trade(1, 5465, "TATA", DateTime.Now.Date, 150),
new Trade(2, 5467, "TATA", DateTime.Now.Date, 180)
);
var transaction2 = MakeTransaction(
new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190)
);
var log = new TransactionLog();
log.NewTransaction(
SerializationHelper.ObjectToBytes(transaction1, SerializationMode.ProtocolBuffers, false));
log.NewTransaction(
SerializationHelper.ObjectToBytes(transaction2, SerializationMode.ProtocolBuffers, false));
log.Dispose();
var engine = new PersistenceEngine();
engine.Start();
engine.WaitForPendingTransactions();
engine.Stop();
var processor = new TestProcessor();
//reload data from persistent storage
var unused = new ReliableStorage(processor);
unused.LoadPersistentData();
unused.Dispose();
Assert.AreEqual(2, processor.LoadedObjects.Count);
var reloaded = processor.LoadedObjects.Select(PackedObject.Unpack <Trade>).First(t => t.Id == 2);
Assert.AreEqual("TOTO", reloaded.Folder);
//reload data from transaction log. Check no more pending transactions
log = new TransactionLog();
Assert.AreEqual(0, log.PendingTransactionsCount);
log.Dispose();
}
public void Write_reload_test_with_one_million_objects()
{
var data = new byte[1000];
for (var i = 0; i < 1000; i++)
{
data[i] = (byte)(i % 255);
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
const int count = 1000000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
{
var w = new Stopwatch();
var processor = new NullProcessor();
w.Start();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
}
w.Stop();
Console.WriteLine(
$"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds");
}
}
public void Create_then_delete_object()
{
var data = new byte[] { 1, 2, 3 };
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
storage.StoreBlock(data, "a1", 150);
storage.StoreBlock(data, "a2", 150);
Assert.AreEqual(2, storage.BlockCount);
}
var processor = new NullProcessor();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
storage.LoadPersistentData();
Assert.AreEqual(2, storage.BlockCount);
storage.DeleteBlock("a1", 44);
}
processor = new NullProcessor();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
Assert.AreEqual(2, storage.BlockCount); // deleted blocks are counted too
}
Assert.AreEqual(1, processor.ProcessedBlocks.Count);
CollectionAssert.AreEqual(data, processor.ProcessedBlocks[0]);
}
public void Write_blocks_to_new_storage()
{
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(0, storage.BlockCount);
storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150);
storage.StoreBlock(new byte[] { 11, 12, 13, 14 }, "a2", 151);
Assert.AreEqual(2, storage.BlockCount);
}
// load add a new one and do in-place update
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(2, storage.BlockCount);
storage.StoreBlock(new byte[] { 21, 22, 23 }, "a3", 150); // new block
storage.StoreBlock(new byte[] { 11, 12, 13, 14, 15 }, "a2", 155); // in place update of old block
Assert.AreEqual(3, storage.BlockCount);
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(0, storage.InactiveBlockCount);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
}
// load again and make an update that can not be done in-place
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(3, storage.BlockCount);
storage.StoreBlock(new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 }, "a2", 155);
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
}
//reload and check data
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
}
}
public void Test_with_backup_storage()
{
var data = new byte[1000];
for (var i = 0; i < 1000; i++)
{
data[i] = (byte)(i % 255);
}
var data1 = new byte[2000];
for (var i = 0; i < 2000; i++)
{
data1[i] = (byte)(i % 255);
}
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
{
var w = new Stopwatch();
var processor = new NullProcessor();
w.Start();
using (var storage = new ReliableStorage(processor, "backup"))
{
storage.LoadPersistentData();
}
w.Stop();
Console.WriteLine(
$"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds");
}
using (var storage = new ReliableStorage(new NullProcessor(), "backup"))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"In-place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data1, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"out of place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
var info1 = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName));
var info2 = new FileInfo(Path.Combine("backup", ReliableStorage.StorageFileName));
Assert.AreEqual(info1.Length, info2.Length);
}
public void Test_recovery_of_corrupted_storage_with_backup_available()
{
// add two new blocks
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
Assert.AreEqual(0, storage.BlockCount);
storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150);
storage.StoreBlock(new byte[] { 1, 2, 3, 4, 5 }, "a2", 151);
storage.StoreBlock(new byte[] { 21, 22, 23, 24, 25 }, "a3", 152);
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(0, storage.InactiveBlockCount);
storage.MakeCorruptedBlock("a2");
}
// here we apply recovery wile loading
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
Assert.AreEqual(3, storage.Keys.Count);
Assert.AreEqual(1, storage.CorruptedBlocks);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
}
// here it is clean again (lost block was recoveres)
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
Assert.AreEqual(3, storage.BlockCount);
Assert.AreEqual(1, storage.InactiveBlockCount);
Assert.AreEqual(0, storage.CorruptedBlocks);
Assert.IsTrue(storage.Keys.Contains("a1"));
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
// do some updates after recovery
storage.StoreBlock(new byte[] { 121, 122, 123, 124, 125, 126 }, "a4", 152);
storage.DeleteBlock("a1", 455);
}
// check the main storage
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
Assert.AreEqual(4, storage.BlockCount); // deleted blocks are counted too
Assert.AreEqual(2, storage.InactiveBlockCount);
Assert.AreEqual(0, storage.CorruptedBlocks);
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
Assert.IsTrue(storage.Keys.Contains("a4"));
}
// check the backup storage
using (var storage = new ReliableStorage(new NullProcessor(), null, "backup"))
{
storage.LoadPersistentData();
Assert.AreEqual(4, storage.BlockCount); // deleted blocks are counted too
Assert.IsTrue(storage.Keys.Contains("a2"));
Assert.IsTrue(storage.Keys.Contains("a3"));
Assert.IsTrue(storage.Keys.Contains("a4"));
}
}
public void Performance_test()
{
var data = new byte[1000];
for (var i = 0; i < 1000; i++)
{
data[i] = (byte)(i % 255);
}
var data1 = new byte[2000];
for (var i = 0; i < 2000; i++)
{
data1[i] = (byte)(i % 255);
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
{
var w = new Stopwatch();
var processor = new NullProcessor();
w.Start();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
}
w.Stop();
Console.WriteLine(
$"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds");
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"In-place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
using (var storage = new ReliableStorage(new NullProcessor()))
{
storage.LoadPersistentData();
const int count = 10000;
var sw = new Stopwatch();
sw.Start();
for (var i = 0; i < count; i++)
{
storage.StoreBlock(data1, i.ToString(), 155);
}
sw.Stop();
Console.WriteLine(
$"out of place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds");
}
{
var w = new Stopwatch();
var processor = new NullProcessor();
w.Start();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
w.Stop();
Assert.AreEqual(10000, storage.InactiveBlockCount);
}
Console.WriteLine(
$"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage with dirty blocks took {w.ElapsedMilliseconds} milliseconds");
}
var info = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName));
Console.WriteLine($"Before cleaning the file size was {info.Length}");
// clean the storage (remove all dirty blocks)
{
var w = new Stopwatch();
var processor = new NullProcessor();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
w.Start();
storage.CleanStorage();
w.Stop();
Assert.AreEqual(0, storage.InactiveBlockCount);
Console.WriteLine(
$"Storage cleaning took {w.ElapsedMilliseconds} milliseconds");
}
}
info = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName));
Console.WriteLine($"After cleaning the file size was {info.Length}");
{
var w = new Stopwatch();
var processor = new NullProcessor();
w.Start();
using (var storage = new ReliableStorage(processor))
{
storage.LoadPersistentData();
w.Stop();
Assert.AreEqual(0, storage.InactiveBlockCount);
}
Console.WriteLine(
$"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage after cleaning took {w.ElapsedMilliseconds} milliseconds");
}
}
