public void CleanIndex(string indexName)
{
KeyValueStore indexStore = GetSecondaryIndex(indexName);
var allValueStoreItems = new HashSet <ByteArray>(this.Enumerate().Select(item => new ByteArray(item.Key)));
foreach (var indexItem in indexStore.Enumerate())
{
byte[] itemKey = KeyValueStore.ItemKeyFromIndex(indexItem);
if (!allValueStoreItems.Contains(new ByteArray(itemKey)))
{
indexStore.Delete(indexItem.Key);
}
}
}
public void RemoveFromIndex(byte[] key, IDictionary <string, byte[]> indexedValues)
{
foreach (var pair in indexedValues)
{
string IndexName = pair.Key;
// Construct Index key by concatenating the indexed value and the target key
byte[] indexValue = pair.Value;
byte[] indexKey = new byte[key.Length + indexValue.Length];
indexValue.CopyTo(indexKey, 0);
key.CopyTo(indexKey, indexValue.Length);
KeyValueStore indexStore = GetSecondaryIndex(IndexName);
indexStore.Delete(indexKey);
}
}
public void RemoveIndexRangeForValue(string indexName, byte[] startAt, byte[] value)
{
KeyValueStore indexStore = GetSecondaryIndex(indexName);
var pairs = indexStore.EnumerateFromKey(startAt);
foreach (var pair in pairs)
{
var itemKey = KeyValueStore.ItemKeyFromIndex(pair);
if (ByteArray.CompareMemCmp(itemKey, value) == 0)
{
indexStore.Delete(pair.Key);
}
if (ByteArray.CompareMemCmp(startAt, 0, pair.Key, 0, startAt.Length) == 0)
{
continue;
}
break;
}
}
public void GetAndSetWithDelete()
{
string path = Path.GetFullPath("TestData\\GetAndSetWithDelete");
using (var db = new KeyValueStore(path)) {
db.Truncate();
for (int i = 0; i < 10; i++) {
byte[] key = BitConverter.GetBytes(i);
byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString());
db.Set(key, value);
}
db.Delete(BitConverter.GetBytes(3));
db.Delete(BitConverter.GetBytes(30));
db.Delete(BitConverter.GetBytes(7));
db.Delete(BitConverter.GetBytes(1));
db.Delete(BitConverter.GetBytes(3));
}
using (var db = new KeyValueStore(path)) {
for (int j = 0; j < 15; j++) {
byte[] key = BitConverter.GetBytes(j);
byte[] value = db.Get(key);
if (j == 3 || j == 1 || j == 7) {
Assert.IsNull(value);
} else if (j < 10) {
Assert.AreEqual(Encoding.UTF8.GetBytes("Number " + j.ToString()), value);
} else {
Assert.IsNull(value);
}
}
}
}
public void BulkSetWithDelete()
{
int numItems = 100000;
string path = Path.GetFullPath("TestData\\BulkSetWithDelete");
if (Directory.Exists(path))
Directory.Delete(path, true);
Directory.CreateDirectory(path);
using (var db = new KeyValueStore(path)) {
db.Manifest.Logger = msg => Console.WriteLine(msg);
db.Truncate();
Stopwatch timer = new Stopwatch();
timer.Start();
for (int i = 0; i < numItems; i++) {
byte[] key = BitConverter.GetBytes(i);
byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString());
db.Set(key, value);
}
timer.Stop();
Console.WriteLine("Wrote {0} items in {1}s", numItems, timer.Elapsed.TotalSeconds);
int skip = 1000;
timer.Reset();
timer.Start();
// Delete every skip-th item in reverse order,
for (int j = numItems; j >= 0; j--) {
if (j % skip == 0) {
byte[] key = BitConverter.GetBytes(j);
db.Delete(key);
}
}
timer.Stop();
Console.WriteLine("Deleted every {0}-th item in {1}s", skip, timer.Elapsed.TotalSeconds);
// Now check all the results
timer.Reset();
timer.Start();
for (int k = 0; k < numItems; k++) {
byte[] key = BitConverter.GetBytes(k);
byte[] value = db.Get(key);
if (k % skip == 0) {
Assert.IsNull(value);
} else {
Assert.AreEqual(Encoding.UTF8.GetBytes("Number " + k.ToString()), value, string.Format("{0}", k));
}
}
timer.Stop();
Console.WriteLine("Read and check every item in {0}s", timer.Elapsed.TotalSeconds);
}
}
public void RemoveDeletedValuesFromIndex()
{
string path = Path.GetFullPath("TestData\\RemoveDeletedValuesFromIndex");
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
int totalSize = 0;
db.Manifest.Logger = msg => Console.WriteLine(msg);
var indexed = new SortedDictionary<string, byte[]>();
int num_items = 1000;
timer.Start();
for (int i = 0; i < num_items; i++) {
indexed["Mod"] = BitConverter.GetBytes(i % 100);
db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i), indexed);
totalSize += 8 + 4;
}
timer.Stop();
Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
timer.Start();
var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count();
timer.Stop();
Assert.AreEqual(10, ctModZeros);
Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds);
}
// Open the index directly and see if the data is there
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(10, num_vals);
}
// Re-open the main key-value store and delete the value at 30
using (var db = new KeyValueStore(path)) {
db.Delete(BitConverter.GetBytes(200));
// Clean the data from the index
db.RemoveFromIndex(BitConverter.GetBytes(200), new Dictionary<string, byte[]> { { "Mod", BitConverter.GetBytes(200 % 100) } });
}
// Open the index again directly and confirm that the lookup key is gone now as well
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(9, num_vals);
}
}
public void IndexClean()
{
string path = Path.GetFullPath("TestData\\IndexClean");
using (var db = new KeyValueStore(path)) {
db.Truncate();
db.Manifest.Logger = msg => Console.WriteLine(msg);
db.Set(Encoding.UTF8.GetBytes("KeyA"), Encoding.UTF8.GetBytes("ValueA:1"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("1") } });
db.Set(Encoding.UTF8.GetBytes("KeyB"), Encoding.UTF8.GetBytes("ValueB:2"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("2") } });
db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:3"), new Dictionary<string, byte[]> { { "Idx", Encoding.UTF8.GetBytes("3") } });
var lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("3")).Single();
Assert.AreEqual("ValueC:3", Encoding.UTF8.GetString(lookupValue.Value));
Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key));
db.Delete(Encoding.UTF8.GetBytes("KeyC"));
}
// Open the index directly and confirm that the lookup key is still there
using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) {
Assert.AreEqual(3, db.Enumerate().Count());
}
using (var db = new KeyValueStore(path)) {
db.CleanIndex("Idx");
}
// Open the index directly and confirm that the lookup key is now gone
using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) {
Assert.AreEqual(2, db.Enumerate().Count());
}
}
