public EnumerateFromKey ( byte startingKey ) : byte[]>>.IEnumerable |
||
startingKey | byte | |
리턴 | byte[]>>.IEnumerable |
public IEnumerable <KeyValuePair <byte[], byte[]> > Find(string indexName, byte[] lookupValue) { KeyValueStore indexStore = GetSecondaryIndex(indexName); // Loop over the values foreach (var pair in indexStore.EnumerateFromKey(lookupValue)) { var key = pair.Key; var value = pair.Value; // construct our index key pattern (lookupvalue | key) if (ByteArray.CompareMemCmp(key, 0, lookupValue, 0, lookupValue.Length) == 0) { int offset = 0; byte[] objectKey = null; if (indexStore.RazorFormatVersion < 2) { if (ByteArray.CompareMemCmp(key, key.Length - value.Length, value, 0, value.Length) == 0) { objectKey = pair.Value; } } else { int indexKeyLen = Helper.Decode7BitInt(pair.Value, ref offset); if (lookupValue.Length == indexKeyLen) { // Lookup the value of the actual object using the key that was found // get the object key from the index value tail objectKey = ItemKeyFromIndex(pair, indexKeyLen); } } if (objectKey != null) { var primaryValue = this.Get(objectKey); if (primaryValue != null) { yield return(new KeyValuePair <byte[], byte[]>(objectKey, primaryValue)); } } } else { // if the above condition was not met then we must have enumerated past the end of the indexed value yield break; } } }
public void RemoveIndexRangeForValue(string indexName, byte[] startAt, byte[] value) { KeyValueStore indexStore = GetSecondaryIndex(indexName); var pairs = indexStore.EnumerateFromKey(startAt); foreach (var pair in pairs) { var itemKey = KeyValueStore.ItemKeyFromIndex(pair); if (ByteArray.CompareMemCmp(itemKey, value) == 0) { indexStore.Delete(pair.Key); } if (ByteArray.CompareMemCmp(startAt, 0, pair.Key, 0, startAt.Length) == 0) { continue; } break; } }
/// <summary> /// Return only the bytes for the key linked to the index (record key in this case is the index value) /// </summary> /// <param name="indexName"></param> /// <param name="lookupValue"></param> /// <returns></returns> public IEnumerable <KeyValuePair <byte[], byte[]> > FindKeysByIndexStartsWith(string indexName, byte[] lookupValue) { KeyValueStore indexStore = GetSecondaryIndex(indexName); // Loop over the values foreach (var pair in indexStore.EnumerateFromKey(lookupValue)) { // construct our index key pattern (lookupvalue | key) if (ByteArray.CompareMemCmp(pair.Key, 0, lookupValue, 0, lookupValue.Length) == 0) { int offset = 0; if (Manifest.RazorFormatVersion < 2) { if (ByteArray.CompareMemCmp(pair.Key, pair.Key.Length - pair.Value.Length, pair.Value, 0, pair.Value.Length) == 0) { yield return(new KeyValuePair <byte[], byte[]>(pair.Key, pair.Value)); } } else { int indexKeyLen = Helper.Decode7BitInt(pair.Value, ref offset); if (lookupValue.Length <= indexKeyLen) { var objectKey = ItemKeyFromIndex(pair, indexKeyLen); Helper.BlockCopy(pair.Key, indexKeyLen, objectKey, 0, pair.Key.Length - indexKeyLen); yield return(new KeyValuePair <byte[], byte[]>(pair.Key, objectKey)); } } } else { // if the above condition was not met then we must have enumerated past the end of the indexed value yield break; } } }
public void BulkSetEnumerateFromKey() { string path = Path.GetFullPath("TestData\\BulkSetEnumerateFromKey"); var timer = new Stopwatch(); int totalSize = 0; int readSize = 0; using (var db = new KeyValueStore(path)) { db.Truncate(); db.Manifest.Logger = (msg) => Console.WriteLine(msg); timer.Start(); for (int i = 0; i < 105000; i++) { var randomKey = BitConverter.GetBytes(i).Reverse().ToArray(); var randomValue = BitConverter.GetBytes(i); db.Set(randomKey, randomValue); readSize += randomKey.Length + randomValue.Length; totalSize += randomKey.Length + randomValue.Length; } timer.Stop(); Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); Console.WriteLine("Begin enumeration."); timer.Start(); int lastKeyNum = 0; int ct = 0; int sum = 0; var searchKey = BitConverter.GetBytes(50000).Reverse().ToArray(); foreach (var pair in db.EnumerateFromKey( searchKey )) { try { int num = BitConverter.ToInt32(pair.Key.Reverse().ToArray(),0); Assert.GreaterOrEqual(num, 50000); sum += num; Assert.Less(lastKeyNum, num); lastKeyNum = num; ct++; } catch (Exception /*e*/) { //Console.WriteLine("Key: {0}\n{1}",insertedItem.Key,e); //Debugger.Launch(); //db.Get(insertedItem.Key.InternalBytes); //db.Manifest.LogContents(); throw; } } timer.Stop(); Assert.AreEqual(55000, ct, "55000 items should be enumerated."); Console.WriteLine("Enumerated read throughput of {0} MB/s (avg {1} ms per 1000 items)", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)105); } }
public void BulkSetEnumerateAll3() { string path = Path.GetFullPath("TestData\\BulkSetEnumerateAll3"); var timer = new Stopwatch(); using (var db = new KeyValueStore(path)) { db.Truncate(); int totalSize = 0; db.Manifest.Logger = msg => Console.WriteLine(msg); int num_items = 1000000; timer.Start(); for (int i = 0; i < num_items; i++) { byte[] key = new byte[8]; BitConverter.GetBytes(i % 100).CopyTo(key,0); BitConverter.GetBytes(i).CopyTo(key,4); byte[] value = BitConverter.GetBytes(i); db.Set(key, value); totalSize += 8 + 4; } timer.Stop(); Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); timer.Start(); var ctModZeros = db.EnumerateFromKey(BitConverter.GetBytes(0)).Count(); timer.Stop(); Console.WriteLine("Scanned index at a throughput of {0} items/s", (double) ctModZeros / timer.Elapsed.TotalSeconds); } }
public void RotationBulkRead() { string path = Path.GetFullPath("TestData\\RotationBulkReadRace"); using (var db = new KeyValueStore(path)) { db.Truncate(); int num_items = 30000; byte[] split = BitConverter.GetBytes(num_items >> 2); int number_we_should_scan = 0; for (int i = 0; i < num_items; i++) { byte[] key = BitConverter.GetBytes(i); if (ByteArray.CompareMemCmp(split, key) <= 0) number_we_should_scan++; } Console.WriteLine("Number to Scan: {0}", number_we_should_scan); Console.WriteLine("Writing {0} items.", num_items); for (int i = 0; i < num_items; i++) { byte[] key = BitConverter.GetBytes(i); byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString()); db.Set(key, value); } Assert.AreEqual(number_we_should_scan, db.EnumerateFromKey(split).Count()); } }
public void RemoveUpdatedValuesFromIndex2() { string path = Path.GetFullPath("TestData\\RemoveUpdatedValuesFromIndex2"); var timer = new Stopwatch(); using (var db = new KeyValueStore(path)) { db.Truncate(); int totalSize = 0; db.Manifest.Logger = msg => Console.WriteLine(msg); var indexed = new SortedDictionary<string, byte[]>(); int num_items = 1000; timer.Start(); for (int i = 0; i < num_items; i++) { indexed["Mod"] = BitConverter.GetBytes(i % 100); db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i), indexed); totalSize += 8 + 4; } timer.Stop(); Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0)); timer.Reset(); timer.Start(); var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count(); timer.Stop(); Assert.AreEqual(10, ctModZeros); Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds); } // Open the index directly and see if the data is there using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) { int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0)); Assert.AreEqual(10, num_vals); } // Re-open the main key-value store and update the value at 30 using (var db = new KeyValueStore(path)) { var indexed = new SortedDictionary<string, byte[]>(); indexed["Mod"] = BitConverter.GetBytes(201 % 100); db.Set(BitConverter.GetBytes(200), BitConverter.GetBytes(200), indexed); // Clean the data from the index db.RemoveFromIndex(BitConverter.GetBytes(200), new Dictionary<string, byte[]> { { "Mod", BitConverter.GetBytes(200 % 100) } }); } // Open the index again directly and confirm that the lookup key is gone now as well using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) { int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0)); Assert.AreEqual(9, num_vals); } }