public void Write_the_same_object_twice() { // add two new blocks using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(0, storage.BlockCount); storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150); storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150); Assert.AreEqual(1, storage.BlockCount); Assert.AreEqual(0, storage.InactiveBlockCount); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(1, storage.BlockCount); Assert.AreEqual(0, storage.InactiveBlockCount); } }
public void Write_then_reload_object() { var data = new byte[] { 1, 2, 3 }; // add two new blocks using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); storage.StoreBlock(data, "a1", 150); Assert.AreEqual(1, storage.BlockCount); } var processor = new NullProcessor(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); Assert.AreEqual(1, storage.BlockCount); } Assert.AreEqual(1, processor.ProcessedBlocks.Count); CollectionAssert.AreEqual(data, processor.ProcessedBlocks[0]); }
public void Persistence_of_simple_put_transaction_with_one_object() { var transaction1 = MakeTransaction(new Trade(1, 5465, "TATA", DateTime.Now.Date, 150)); var engine = new PersistenceEngine(); engine.Start(); engine.NewTransaction(transaction1); // wait for the transaction log to be processed engine.WaitForPendingTransactions(); engine.Stop(); var processor = new TestProcessor(); //reload data from persistent storage var unused = new ReliableStorage(processor); unused.LoadPersistentData(); unused.Dispose(); Assert.AreEqual(1, processor.LoadedObjects.Count); var reloaded = CachedObject.Unpack <Trade>(processor.LoadedObjects[0]); Assert.AreEqual("TATA", reloaded.Folder); }
public void Test_recovery_of_corrupted_storage() { // add two new blocks using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(0, storage.BlockCount); storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150); storage.StoreBlock(new byte[] { 1, 2, 3, 4, 5 }, "a2", 151); storage.StoreBlock(new byte[] { 21, 22, 23, 24, 25 }, "a3", 152); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(0, storage.InactiveBlockCount); storage.MakeCorruptedBlock("a2"); } // here we apply recovery wile loading using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(2, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); Assert.AreEqual(2, storage.Keys.Count); Assert.AreEqual(1, storage.CorruptedBlocks); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a3")); } // here it is clean again (one block was lost) using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(2, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); Assert.AreEqual(0, storage.CorruptedBlocks); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a3")); } }
public void Create_then_delete_object() { var data = new byte[] { 1, 2, 3 }; // add two new blocks using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); storage.StoreBlock(data, "a1", 150); storage.StoreBlock(data, "a2", 150); Assert.AreEqual(2, storage.BlockCount); } var processor = new NullProcessor(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); storage.LoadPersistentData(); Assert.AreEqual(2, storage.BlockCount); storage.DeleteBlock("a1", 44); } processor = new NullProcessor(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); Assert.AreEqual(2, storage.BlockCount); // deleted blocks are counted too } Assert.AreEqual(1, processor.ProcessedBlocks.Count); CollectionAssert.AreEqual(data, processor.ProcessedBlocks[0]); }
Add_two_objects_then_update_one_of_them_then_check_the_persistent_storage_contains_the_updated_value() { var transaction1 = MakeTransaction( new Trade(1, 5465, "TATA", DateTime.Now.Date, 150), new Trade(2, 5467, "TATA", DateTime.Now.Date, 180) ); var transaction2 = MakeTransaction( new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190) ); var deleteTransaction = MakeDeleteTransaction( new Trade(1, 5465, "TATA", DateTime.Now.Date, 150) ); var engine = new PersistenceEngine(); engine.Start(); engine.NewTransaction(transaction1); engine.NewTransaction(transaction2); engine.NewTransaction(deleteTransaction); // wait for the transaction log to be processed engine.WaitForPendingTransactions(); engine.Stop(); var processor = new TestProcessor(); //reload data from persistent storage var unused = new ReliableStorage(processor); unused.LoadPersistentData(); unused.Dispose(); Assert.AreEqual(1, processor.LoadedObjects.Count); var reloaded = processor.LoadedObjects.Select(CachedObject.Unpack <Trade>).First(t => t.Id == 2); Assert.AreEqual("TOTO", reloaded.Folder); //reload data from transaction log. Check no more pending transactions var log = new TransactionLog(); Assert.AreEqual(0, log.PendingTransactionsCount); log.Dispose(); }
public void Simulate_failure_during_transaction_processing() { var transaction1 = MakeTransaction( new Trade(1, 5465, "TATA", DateTime.Now.Date, 150), new Trade(2, 5467, "TATA", DateTime.Now.Date, 180) ); var transaction2 = MakeTransaction( new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190) ); var log = new TransactionLog(); log.NewTransaction( SerializationHelper.ObjectToBytes(transaction1, SerializationMode.ProtocolBuffers, null)); log.NewTransaction( SerializationHelper.ObjectToBytes(transaction2, SerializationMode.ProtocolBuffers, null)); var dummy = log.StartProcessing(); // call StartProcessing and close before EndProcessing log.Dispose(); var engine = new PersistenceEngine(); engine.Start(); engine.WaitForPendingTransactions(); engine.Stop(); var processor = new TestProcessor(); //reload data from persistent storage var unused = new ReliableStorage(processor); unused.LoadPersistentData(); unused.Dispose(); Assert.AreEqual(2, processor.LoadedObjects.Count); var reloaded = processor.LoadedObjects.Select(CachedObject.Unpack <Trade>).First(t => t.Id == 2); Assert.AreEqual("TOTO", reloaded.Folder); //reload data from transaction log. Check no more pending transactions log = new TransactionLog(); Assert.AreEqual(0, log.PendingTransactionsCount); log.Dispose(); }
public void Start_the_persistence_engine_with_a_non_empty_transaction_log() { var transaction1 = MakeTransaction( new Trade(1, 5465, "TATA", DateTime.Now.Date, 150), new Trade(2, 5467, "TATA", DateTime.Now.Date, 180) ); var transaction2 = MakeTransaction( new Trade(2, 5467, "TOTO", DateTime.Now.Date, 190) ); var log = new TransactionLog(); log.NewTransaction( SerializationHelper.ObjectToBytes(transaction1, SerializationMode.ProtocolBuffers, false)); log.NewTransaction( SerializationHelper.ObjectToBytes(transaction2, SerializationMode.ProtocolBuffers, false)); log.Dispose(); var engine = new PersistenceEngine(); engine.Start(); engine.WaitForPendingTransactions(); engine.Stop(); var processor = new TestProcessor(); //reload data from persistent storage var unused = new ReliableStorage(processor); unused.LoadPersistentData(); unused.Dispose(); Assert.AreEqual(2, processor.LoadedObjects.Count); var reloaded = processor.LoadedObjects.Select(PackedObject.Unpack <Trade>).First(t => t.Id == 2); Assert.AreEqual("TOTO", reloaded.Folder); //reload data from transaction log. Check no more pending transactions log = new TransactionLog(); Assert.AreEqual(0, log.PendingTransactionsCount); log.Dispose(); }
public void Write_reload_test_with_one_million_objects() { var data = new byte[1000]; for (var i = 0; i < 1000; i++) { data[i] = (byte)(i % 255); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); const int count = 1000000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } { var w = new Stopwatch(); var processor = new NullProcessor(); w.Start(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); } w.Stop(); Console.WriteLine( $"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds"); } }
public void Write_blocks_to_new_storage() { // add two new blocks using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(0, storage.BlockCount); storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150); storage.StoreBlock(new byte[] { 11, 12, 13, 14 }, "a2", 151); Assert.AreEqual(2, storage.BlockCount); } // load add a new one and do in-place update using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(2, storage.BlockCount); storage.StoreBlock(new byte[] { 21, 22, 23 }, "a3", 150); // new block storage.StoreBlock(new byte[] { 11, 12, 13, 14, 15 }, "a2", 155); // in place update of old block Assert.AreEqual(3, storage.BlockCount); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(0, storage.InactiveBlockCount); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); } // load again and make an update that can not be done in-place using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(3, storage.BlockCount); storage.StoreBlock(new byte[] { 11, 12, 13, 14, 15, 16, 17, 18 }, "a2", 155); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); } //reload and check data using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); } }
public void Test_with_backup_storage() { var data = new byte[1000]; for (var i = 0; i < 1000; i++) { data[i] = (byte)(i % 255); } var data1 = new byte[2000]; for (var i = 0; i < 2000; i++) { data1[i] = (byte)(i % 255); } using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } { var w = new Stopwatch(); var processor = new NullProcessor(); w.Start(); using (var storage = new ReliableStorage(processor, "backup")) { storage.LoadPersistentData(); } w.Stop(); Console.WriteLine( $"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds"); } using (var storage = new ReliableStorage(new NullProcessor(), "backup")) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"In-place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data1, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"out of place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } var info1 = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName)); var info2 = new FileInfo(Path.Combine("backup", ReliableStorage.StorageFileName)); Assert.AreEqual(info1.Length, info2.Length); }
public void Test_recovery_of_corrupted_storage_with_backup_available() { // add two new blocks using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); Assert.AreEqual(0, storage.BlockCount); storage.StoreBlock(new byte[] { 1, 2, 3 }, "a1", 150); storage.StoreBlock(new byte[] { 1, 2, 3, 4, 5 }, "a2", 151); storage.StoreBlock(new byte[] { 21, 22, 23, 24, 25 }, "a3", 152); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(0, storage.InactiveBlockCount); storage.MakeCorruptedBlock("a2"); } // here we apply recovery wile loading using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); Assert.AreEqual(3, storage.Keys.Count); Assert.AreEqual(1, storage.CorruptedBlocks); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); } // here it is clean again (lost block was recoveres) using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); Assert.AreEqual(3, storage.BlockCount); Assert.AreEqual(1, storage.InactiveBlockCount); Assert.AreEqual(0, storage.CorruptedBlocks); Assert.IsTrue(storage.Keys.Contains("a1")); Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); // do some updates after recovery storage.StoreBlock(new byte[] { 121, 122, 123, 124, 125, 126 }, "a4", 152); storage.DeleteBlock("a1", 455); } // check the main storage using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); Assert.AreEqual(4, storage.BlockCount); // deleted blocks are counted too Assert.AreEqual(2, storage.InactiveBlockCount); Assert.AreEqual(0, storage.CorruptedBlocks); Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); Assert.IsTrue(storage.Keys.Contains("a4")); } // check the backup storage using (var storage = new ReliableStorage(new NullProcessor(), null, "backup")) { storage.LoadPersistentData(); Assert.AreEqual(4, storage.BlockCount); // deleted blocks are counted too Assert.IsTrue(storage.Keys.Contains("a2")); Assert.IsTrue(storage.Keys.Contains("a3")); Assert.IsTrue(storage.Keys.Contains("a4")); } }
public void Performance_test() { var data = new byte[1000]; for (var i = 0; i < 1000; i++) { data[i] = (byte)(i % 255); } var data1 = new byte[2000]; for (var i = 0; i < 2000; i++) { data1[i] = (byte)(i % 255); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"Writing {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } { var w = new Stopwatch(); var processor = new NullProcessor(); w.Start(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); } w.Stop(); Console.WriteLine( $"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage took {w.ElapsedMilliseconds} milliseconds"); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"In-place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } using (var storage = new ReliableStorage(new NullProcessor())) { storage.LoadPersistentData(); const int count = 10000; var sw = new Stopwatch(); sw.Start(); for (var i = 0; i < count; i++) { storage.StoreBlock(data1, i.ToString(), 155); } sw.Stop(); Console.WriteLine( $"out of place updating {count} blocks to persistent storage took {sw.ElapsedMilliseconds} milliseconds"); } { var w = new Stopwatch(); var processor = new NullProcessor(); w.Start(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); w.Stop(); Assert.AreEqual(10000, storage.InactiveBlockCount); } Console.WriteLine( $"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage with dirty blocks took {w.ElapsedMilliseconds} milliseconds"); } var info = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName)); Console.WriteLine($"Before cleaning the file size was {info.Length}"); // clean the storage (remove all dirty blocks) { var w = new Stopwatch(); var processor = new NullProcessor(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); w.Start(); storage.CleanStorage(); w.Stop(); Assert.AreEqual(0, storage.InactiveBlockCount); Console.WriteLine( $"Storage cleaning took {w.ElapsedMilliseconds} milliseconds"); } } info = new FileInfo(Path.Combine(Constants.DataPath, ReliableStorage.StorageFileName)); Console.WriteLine($"After cleaning the file size was {info.Length}"); { var w = new Stopwatch(); var processor = new NullProcessor(); w.Start(); using (var storage = new ReliableStorage(processor)) { storage.LoadPersistentData(); w.Stop(); Assert.AreEqual(0, storage.InactiveBlockCount); } Console.WriteLine( $"Loading {processor.ProcessedBlocks.Count} blocks from persistent storage after cleaning took {w.ElapsedMilliseconds} milliseconds"); } }