public void LargeDataEnumerateTest() { string path = Path.GetFullPath("TestData\\LargeDataEnumerateTest"); int totalSize = 0; int num_items = 500; var timer = new Stopwatch(); using (var db = new KeyValueStore(path)) { db.Truncate(); // Generate a data value that is larger than the block size. var value = ByteArray.Random(Config.SortedBlockSize + 256); // Do it enough times to ensure a roll-over for (int i = 0; i < num_items; i++) { var key = BitConverter.GetBytes(i).Reverse().ToArray(); // this has to be little endian to sort in an obvious way db.Set(key, value.InternalBytes); totalSize += value.InternalBytes.Length; } int j=0; timer.Start(); foreach (var pair in db.Enumerate()) { var key = BitConverter.GetBytes(j).Reverse().ToArray(); Assert.AreEqual(key, pair.Key); Assert.AreEqual(value.InternalBytes, pair.Value); j++; } timer.Stop(); Console.WriteLine("Randomized read throughput of {0} MB/s (avg {1} ms per lookup)", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)num_items); } }
public void CleanIndex(string indexName) { KeyValueStore indexStore = GetSecondaryIndex(indexName); var allValueStoreItems = new HashSet <ByteArray>(this.Enumerate().Select(item => new ByteArray(item.Key))); foreach (var indexItem in indexStore.Enumerate()) { byte[] itemKey = KeyValueStore.ItemKeyFromIndex(indexItem); if (!allValueStoreItems.Contains(new ByteArray(itemKey))) { indexStore.Delete(indexItem.Key); } } }
public void CrashTestBeforeMerge() { string path = Path.GetFullPath("TestData\\CrashTestBeforeMerge"); using (var db = new KeyValueStore(path)) { db.Truncate(); } var doneSetting = new EventWaitHandle(false, EventResetMode.ManualReset, "CrashTestBeforeMerge"); doneSetting.Reset(); string testPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().GetName().CodeBase), "RazorTest.exe"); var process = Process.Start(testPath, "CrashTestBeforeMerge"); doneSetting.WaitOne(30000); process.Refresh(); if (!process.HasExited) { try { process.Kill(); } catch { } process.WaitForExit(); } // Open the database created by the other program using (var db = new KeyValueStore(path)) { db.Manifest.Logger = (msg) => Console.WriteLine(msg); Console.WriteLine("Begin enumeration."); ByteArray lastKey = new ByteArray(); int ct = 0; foreach (var pair in db.Enumerate()) { ByteArray k = new ByteArray(pair.Key); Assert.True(lastKey.CompareTo(k) < 0); lastKey = k; ct++; } Assert.AreEqual(10000, ct); Console.WriteLine("Found {0} items in the crashed database.", ct); } }
public int CountIndex(string indexName) { KeyValueStore indexStore = GetSecondaryIndex(indexName); return(indexStore.Enumerate().Count()); }
public void DumpKeySpaceUsed() { double valueBytes=0L; double keyBytes = 0L; double dupBytes = 0L; double totalRecords = 0L; Action<string> dumpFolderBytes = (folder) => { double tableValBytes = 0L; double tableKeyBytes = 0L; double tableDupBytes = 0L; double tableRecords = 0L; byte[] lastkey = null; using (var kvs = new KeyValueStore(folder)) { foreach (var pair in kvs.Enumerate()) { tableRecords++; tableValBytes += pair.Value.Length; tableKeyBytes += pair.Key.Length; if (lastkey != null) { int i = 0; for (i = 0; i < lastkey.Length && i < pair.Key.Length; i++) if (lastkey[i] != pair.Key[i]) continue; tableDupBytes += i; } lastkey = pair.Key; } } valueBytes += tableValBytes; keyBytes += tableKeyBytes; dupBytes += tableDupBytes; totalRecords += tableRecords; Console.WriteLine("{0} Total Bytes: {1}", folder, tableValBytes + tableKeyBytes); Console.WriteLine(" #Records: {0}", tableRecords); Console.WriteLine(" Key Bytes: {0}", tableKeyBytes); Console.WriteLine(" Value Bytes: {0}", tableValBytes); Console.WriteLine(" Dupl. Bytes: {0}", tableDupBytes); Console.WriteLine(" %Savings in keys: {0}%", tableDupBytes / tableKeyBytes * 100); Console.WriteLine(" %Savings overall: {0}%", tableDupBytes / (tableValBytes + tableKeyBytes) * 100); Console.WriteLine(); }; var baseFolder = @"d:\ncoverdata\ncover"; foreach (var folder in Directory.GetDirectories(baseFolder, "*", SearchOption.AllDirectories)) dumpFolderBytes(folder); Console.WriteLine("Total KeyValueStore Bytes: {0}", valueBytes + keyBytes); Console.WriteLine(" #Records: {0}", totalRecords); Console.WriteLine(" Key Bytes: {0}", keyBytes); Console.WriteLine(" Value Bytes: {0}", valueBytes); Console.WriteLine(" Dupl. Bytes: {0}", dupBytes); Console.WriteLine(" %Savings in keys: {0}%", dupBytes / keyBytes * 100); Console.WriteLine(" %Savings overall: {0}%", dupBytes / (valueBytes + keyBytes) * 100); }
public void BulkSetEnumerateAllWithMissingSBT_ThrowAll() { string path = Path.GetFullPath("TestData\\BulkSetEnumerateAllWithMissingSBT_ThrowAll"+DateTime.Now.Ticks.ToString()); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; Action<string> logger = (msg) => { Console.WriteLine(msg); }; using (var db = new KeyValueStore(path)) { db.Truncate(); timer.Start(); for (int i = 0; i < 500000; i++) { var randomKey = BitConverter.GetBytes(i); var randomValue = BitConverter.GetBytes(i); db.Set(randomKey, randomValue); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); } // delete the sbt files var files = Directory.GetFiles(path, "*.sbt"); foreach(var fname in files) File.Delete(fname); // Close and re-open the database to force all the sstable merging to complete. Console.WriteLine("Begin enumeration."); RazorDB.Config.ExceptionHandling = ExceptionHandling.ThrowAll; Assert.Throws(typeof(FileNotFoundException), () => { using (var db = new KeyValueStore(path)) { foreach (var pair in db.Enumerate()); } }); }
public void BulkSetEnumerateAllWithMissingSBT_AttemptRecovery() { try { RazorDB.Config.ExceptionHandling = ExceptionHandling.AttemptRecovery; string path = Path.GetFullPath("TestData\\BulkSetEnumerateAllWithMissingSBT_AttemptRecovery"); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; Action<string> logger = (msg) => { Console.WriteLine(msg); }; using (var db = new KeyValueStore(path)) { db.Truncate(); timer.Start(); for (int i = 0; i < 500000; i++) { var randomKey = BitConverter.GetBytes(i); var randomValue = BitConverter.GetBytes(i); db.Set(randomKey, randomValue); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); } // delete the sbt files var files = Directory.GetFiles(path, "*.sbt"); foreach (var fname in files) File.Delete(fname); // Close and re-open the database to force all the sstable merging to complete. Console.WriteLine("Begin enumeration."); using (var db = new KeyValueStore(path)) { timer.Reset(); timer.Start(); ByteArray lastKey = ByteArray.Empty; int ct = 0; foreach (var pair in db.Enumerate()) { try { ByteArray k = new ByteArray(pair.Key); ByteArray v = new ByteArray(pair.Value); Assert.AreEqual(k, v); Assert.True(lastKey.CompareTo(k) < 0); lastKey = k; ct++; } catch (Exception /*e*/) { //Console.WriteLine("Key: {0}\n{1}",insertedItem.Key,e); //Debugger.Launch(); //db.Get(insertedItem.Key.InternalBytes); //db.Manifest.LogContents(); throw; } } timer.Stop(); Assert.AreEqual(80568, ct); Console.WriteLine("Enumerated read throughput of {0} MB/s (avg {1} ms per 1000 items)", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)105); } // add some more records after deleting files using (var db = new KeyValueStore(path)) { timer.Start(); // add 1,000,000 new keys for (int i = 1000000; i < 3000000; i++) { var randomKey = BitConverter.GetBytes(i); var randomValue = BitConverter.GetBytes(i); db.Set(randomKey, randomValue); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); } // Close and re-open the database to force all the sstable merging to complete. Console.WriteLine("Begin enumeration."); using (var db = new KeyValueStore(path)) { timer.Reset(); timer.Start(); ByteArray lastKey = ByteArray.Empty; int ct = 0; foreach (var pair in db.Enumerate()) { try { ByteArray k = new ByteArray(pair.Key); ByteArray v = new ByteArray(pair.Value); Assert.AreEqual(k, v); Assert.True(lastKey.CompareTo(k) < 0); lastKey = k; ct++; } catch (Exception /*e*/) { //Console.WriteLine("Key: {0}\n{1}",insertedItem.Key,e); //Debugger.Launch(); //db.Get(insertedItem.Key.InternalBytes); //db.Manifest.LogContents(); throw; } } timer.Stop(); Assert.AreEqual(2080568, ct); Console.WriteLine("Enumerated read throughput of {0} MB/s (avg {1} ms per 1000 items)", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)105); } } finally { RazorDB.Config.ExceptionHandling = ExceptionHandling.ThrowAll; } }
public void BulkSetEnumerateAll2() { string path = Path.GetFullPath("TestData\\BulkSetEnumerateAll2"); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = (msg) => Console.WriteLine(msg); timer.Start(); for (int i = 0; i < 105000; i++) { var randomKey = BitConverter.GetBytes(i); var randomValue = BitConverter.GetBytes(i); db.Set(randomKey, randomValue); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); Console.WriteLine("Begin enumeration."); timer.Start(); ByteArray lastKey = ByteArray.Empty; int ct = 0; foreach (var pair in db.Enumerate()) { try { ByteArray k = new ByteArray(pair.Key); ByteArray v = new ByteArray(pair.Value); Assert.AreEqual(k, v); Assert.True(lastKey.CompareTo(k) < 0); lastKey = k; ct++; } catch (Exception /*e*/) { //Console.WriteLine("Key: {0}\n{1}",insertedItem.Key,e); //Debugger.Launch(); //db.Get(insertedItem.Key.InternalBytes); //db.Manifest.LogContents(); throw; } } timer.Stop(); Assert.AreEqual(105000, ct, "105000 items should be enumerated."); Console.WriteLine("Enumerated read throughput of {0} MB/s (avg {1} ms per 1000 items)", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)105); } }
public void BulkSetBulkEnumerateWithCache() { string path = Path.GetFullPath("TestData\\BulkSetBulkEnumerateWithCache"); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; int num_items = 100000; using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = (msg) => Console.WriteLine(msg); timer.Start(); for (int i = 0; i < num_items; i++) { var randomKey = ByteArray.Random(40); var randomValue = ByteArray.Random(256); db.Set(randomKey.InternalBytes, randomValue.InternalBytes); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); timer.Start(); Assert.AreEqual(num_items, db.Enumerate().Count()); timer.Stop(); Console.WriteLine("Enumerated read throughput of {0} MB/s", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); timer.Start(); Assert.AreEqual(num_items, db.Enumerate().Count()); timer.Stop(); Console.WriteLine("Enumerated (second pass) read throughput of {0} MB/s", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); } }
public void BulkSetBulkEnumerateWhileMerging() { string path = Path.GetFullPath("TestData\\BulkSetBulkEnumerateWhileMerging"); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; int num_items = 100000; using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = (msg) => Console.WriteLine(msg); timer.Start(); for (int i = 0; i < num_items; i++) { var randomKey = ByteArray.Random(40); var randomValue = ByteArray.Random(256); db.Set(randomKey.InternalBytes, randomValue.InternalBytes); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); Console.WriteLine("Begin enumeration."); timer.Start(); ByteArray lastKey = new ByteArray(); int ct = 0; foreach (var pair in db.Enumerate()) { try { ByteArray k = new ByteArray(pair.Key); Assert.True(lastKey.CompareTo(k) < 0); lastKey = k; ct++; } catch (Exception e) { Console.WriteLine("Key: {0}\n{1}",pair.Key,e); Debugger.Launch(); throw; } } timer.Stop(); Assert.AreEqual(num_items, ct, num_items.ToString() + " items should be enumerated."); Console.WriteLine("Enumerated read throughput of {0} MB/s", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); } }
public void TestJournalFileGrowth() { string path = Path.GetFullPath("TestData\\TestJournalFileGrowth"); // Open database and store enough data to cause a page split using (var db = new KeyValueStore(path)) { db.Truncate(); ByteArray key = ByteArray.Random(40); for (int i = 0; i < 100000; i++) { ByteArray value = ByteArray.Random(400); db.Set(key.InternalBytes, value.InternalBytes); } var journalfileName = Config.JournalFile(db.Manifest.BaseFileName, db.Manifest.CurrentVersion(0)); var journalLength = new FileInfo(journalfileName).Length; // Make sure the journal is smaller than the max memtable size. Assert.LessOrEqual(journalLength, Config.MaxMemTableSize); // Double check to be sure that the contents of the database are correct in this case. int count = 0; foreach (var value in db.Enumerate()) { Assert.AreEqual(key.InternalBytes, value.Key); count++; } Assert.AreEqual(1, count); } }
public void V1ReadLargeObjectsFromDatastore() { using (var db = new KeyValueStore(@"..\FormatTestData\V1LargeObjectStore")) { foreach (var pair in db.Enumerate()) { Assert.AreEqual(40, pair.Key.Length); Assert.AreEqual(Config.MaxSmallValueSize * 100, pair.Value.Length); } } }
public void IndexClean() { string path = Path.GetFullPath("TestData\\IndexClean"); using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = msg => Console.WriteLine(msg); db.Set(Encoding.UTF8.GetBytes("KeyA"), Encoding.UTF8.GetBytes("ValueA:1"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("1") } }); db.Set(Encoding.UTF8.GetBytes("KeyB"), Encoding.UTF8.GetBytes("ValueB:2"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("2") } }); db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:3"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("3") } }); var lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("3")).Single(); Assert.AreEqual("ValueC:3", Encoding.UTF8.GetString(lookupValue.Value)); Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key)); db.Delete(Encoding.UTF8.GetBytes("KeyC")); } // Open the index directly and confirm that the lookup key is still there using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) { Assert.AreEqual(3, db.Enumerate().Count()); } using (var db = new KeyValueStore(path)) { db.CleanIndex("Idx"); } // Open the index directly and confirm that the lookup key is now gone using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) { Assert.AreEqual(2, db.Enumerate().Count()); } }