public void CleanIndex(string indexName) { KeyValueStore indexStore = GetSecondaryIndex(indexName); var allValueStoreItems = new HashSet <ByteArray>(this.Enumerate().Select(item => new ByteArray(item.Key))); foreach (var indexItem in indexStore.Enumerate()) { byte[] itemKey = KeyValueStore.ItemKeyFromIndex(indexItem); if (!allValueStoreItems.Contains(new ByteArray(itemKey))) { indexStore.Delete(indexItem.Key); } } }
public void RemoveFromIndex(byte[] key, IDictionary <string, byte[]> indexedValues) { foreach (var pair in indexedValues) { string IndexName = pair.Key; // Construct Index key by concatenating the indexed value and the target key byte[] indexValue = pair.Value; byte[] indexKey = new byte[key.Length + indexValue.Length]; indexValue.CopyTo(indexKey, 0); key.CopyTo(indexKey, indexValue.Length); KeyValueStore indexStore = GetSecondaryIndex(IndexName); indexStore.Delete(indexKey); } }
public void RemoveIndexRangeForValue(string indexName, byte[] startAt, byte[] value) { KeyValueStore indexStore = GetSecondaryIndex(indexName); var pairs = indexStore.EnumerateFromKey(startAt); foreach (var pair in pairs) { var itemKey = KeyValueStore.ItemKeyFromIndex(pair); if (ByteArray.CompareMemCmp(itemKey, value) == 0) { indexStore.Delete(pair.Key); } if (ByteArray.CompareMemCmp(startAt, 0, pair.Key, 0, startAt.Length) == 0) { continue; } break; } }
public void GetAndSetWithDelete() { string path = Path.GetFullPath("TestData\\GetAndSetWithDelete"); using (var db = new KeyValueStore(path)) { db.Truncate(); for (int i = 0; i < 10; i++) { byte[] key = BitConverter.GetBytes(i); byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString()); db.Set(key, value); } db.Delete(BitConverter.GetBytes(3)); db.Delete(BitConverter.GetBytes(30)); db.Delete(BitConverter.GetBytes(7)); db.Delete(BitConverter.GetBytes(1)); db.Delete(BitConverter.GetBytes(3)); } using (var db = new KeyValueStore(path)) { for (int j = 0; j < 15; j++) { byte[] key = BitConverter.GetBytes(j); byte[] value = db.Get(key); if (j == 3 || j == 1 || j == 7) { Assert.IsNull(value); } else if (j < 10) { Assert.AreEqual(Encoding.UTF8.GetBytes("Number " + j.ToString()), value); } else { Assert.IsNull(value); } } } }
public void BulkSetWithDelete() { int numItems = 100000; string path = Path.GetFullPath("TestData\\BulkSetWithDelete"); if (Directory.Exists(path)) Directory.Delete(path, true); Directory.CreateDirectory(path); using (var db = new KeyValueStore(path)) { db.Manifest.Logger = msg => Console.WriteLine(msg); db.Truncate(); Stopwatch timer = new Stopwatch(); timer.Start(); for (int i = 0; i < numItems; i++) { byte[] key = BitConverter.GetBytes(i); byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString()); db.Set(key, value); } timer.Stop(); Console.WriteLine("Wrote {0} items in {1}s", numItems, timer.Elapsed.TotalSeconds); int skip = 1000; timer.Reset(); timer.Start(); // Delete every skip-th item in reverse order, for (int j = numItems; j >= 0; j--) { if (j % skip == 0) { byte[] key = BitConverter.GetBytes(j); db.Delete(key); } } timer.Stop(); Console.WriteLine("Deleted every {0}-th item in {1}s", skip, timer.Elapsed.TotalSeconds); // Now check all the results timer.Reset(); timer.Start(); for (int k = 0; k < numItems; k++) { byte[] key = BitConverter.GetBytes(k); byte[] value = db.Get(key); if (k % skip == 0) { Assert.IsNull(value); } else { Assert.AreEqual(Encoding.UTF8.GetBytes("Number " + k.ToString()), value, string.Format("{0}", k)); } } timer.Stop(); Console.WriteLine("Read and check every item in {0}s", timer.Elapsed.TotalSeconds); } }
public void RemoveDeletedValuesFromIndex() { string path = Path.GetFullPath("TestData\\RemoveDeletedValuesFromIndex"); var timer = new Stopwatch(); using (var db = new KeyValueStore(path)) { db.Truncate(); int totalSize = 0; db.Manifest.Logger = msg => Console.WriteLine(msg); var indexed = new SortedDictionary<string, byte[]>(); int num_items = 1000; timer.Start(); for (int i = 0; i < num_items; i++) { indexed["Mod"] = BitConverter.GetBytes(i % 100); db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i), indexed); totalSize += 8 + 4; } timer.Stop(); Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); timer.Start(); var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count(); timer.Stop(); Assert.AreEqual(10, ctModZeros); Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds); } // Open the index directly and see if the data is there using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) { int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0)); Assert.AreEqual(10, num_vals); } // Re-open the main key-value store and delete the value at 30 using (var db = new KeyValueStore(path)) { db.Delete(BitConverter.GetBytes(200)); // Clean the data from the index db.RemoveFromIndex(BitConverter.GetBytes(200), new Dictionary<string, byte[]> { { "Mod", BitConverter.GetBytes(200 % 100) } }); } // Open the index again directly and confirm that the lookup key is gone now as well using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) { int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0)); Assert.AreEqual(9, num_vals); } }
public void IndexClean() { string path = Path.GetFullPath("TestData\\IndexClean"); using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = msg => Console.WriteLine(msg); db.Set(Encoding.UTF8.GetBytes("KeyA"), Encoding.UTF8.GetBytes("ValueA:1"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("1") } }); db.Set(Encoding.UTF8.GetBytes("KeyB"), Encoding.UTF8.GetBytes("ValueB:2"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("2") } }); db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:3"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("3") } }); var lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("3")).Single(); Assert.AreEqual("ValueC:3", Encoding.UTF8.GetString(lookupValue.Value)); Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key)); db.Delete(Encoding.UTF8.GetBytes("KeyC")); } // Open the index directly and confirm that the lookup key is still there using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) { Assert.AreEqual(3, db.Enumerate().Count()); } using (var db = new KeyValueStore(path)) { db.CleanIndex("Idx"); } // Open the index directly and confirm that the lookup key is now gone using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) { Assert.AreEqual(2, db.Enumerate().Count()); } }