|
public EnumerateFromKey ( byte startingKey ) : byte[]>>.IEnumerable |
||
| startingKey | byte | |
| return | byte[]>>.IEnumerable | |
public IEnumerable <KeyValuePair <byte[], byte[]> > Find(string indexName, byte[] lookupValue)
{
KeyValueStore indexStore = GetSecondaryIndex(indexName);
// Loop over the values
foreach (var pair in indexStore.EnumerateFromKey(lookupValue))
{
var key = pair.Key;
var value = pair.Value;
// construct our index key pattern (lookupvalue | key)
if (ByteArray.CompareMemCmp(key, 0, lookupValue, 0, lookupValue.Length) == 0)
{
int offset = 0;
byte[] objectKey = null;
if (indexStore.RazorFormatVersion < 2)
{
if (ByteArray.CompareMemCmp(key, key.Length - value.Length, value, 0, value.Length) == 0)
{
objectKey = pair.Value;
}
}
else
{
int indexKeyLen = Helper.Decode7BitInt(pair.Value, ref offset);
if (lookupValue.Length == indexKeyLen)
{
// Lookup the value of the actual object using the key that was found
// get the object key from the index value tail
objectKey = ItemKeyFromIndex(pair, indexKeyLen);
}
}
if (objectKey != null)
{
var primaryValue = this.Get(objectKey);
if (primaryValue != null)
{
yield return(new KeyValuePair <byte[], byte[]>(objectKey, primaryValue));
}
}
}
else
{
// if the above condition was not met then we must have enumerated past the end of the indexed value
yield break;
}
}
}
public void RemoveIndexRangeForValue(string indexName, byte[] startAt, byte[] value)
{
KeyValueStore indexStore = GetSecondaryIndex(indexName);
var pairs = indexStore.EnumerateFromKey(startAt);
foreach (var pair in pairs)
{
var itemKey = KeyValueStore.ItemKeyFromIndex(pair);
if (ByteArray.CompareMemCmp(itemKey, value) == 0)
{
indexStore.Delete(pair.Key);
}
if (ByteArray.CompareMemCmp(startAt, 0, pair.Key, 0, startAt.Length) == 0)
{
continue;
}
break;
}
}
/// <summary>
/// Return only the bytes for the key linked to the index (record key in this case is the index value)
/// </summary>
/// <param name="indexName"></param>
/// <param name="lookupValue"></param>
/// <returns></returns>
public IEnumerable <KeyValuePair <byte[], byte[]> > FindKeysByIndexStartsWith(string indexName, byte[] lookupValue)
{
KeyValueStore indexStore = GetSecondaryIndex(indexName);
// Loop over the values
foreach (var pair in indexStore.EnumerateFromKey(lookupValue))
{
// construct our index key pattern (lookupvalue | key)
if (ByteArray.CompareMemCmp(pair.Key, 0, lookupValue, 0, lookupValue.Length) == 0)
{
int offset = 0;
if (Manifest.RazorFormatVersion < 2)
{
if (ByteArray.CompareMemCmp(pair.Key, pair.Key.Length - pair.Value.Length, pair.Value, 0, pair.Value.Length) == 0)
{
yield return(new KeyValuePair <byte[], byte[]>(pair.Key, pair.Value));
}
}
else
{
int indexKeyLen = Helper.Decode7BitInt(pair.Value, ref offset);
if (lookupValue.Length <= indexKeyLen)
{
var objectKey = ItemKeyFromIndex(pair, indexKeyLen);
Helper.BlockCopy(pair.Key, indexKeyLen, objectKey, 0, pair.Key.Length - indexKeyLen);
yield return(new KeyValuePair <byte[], byte[]>(pair.Key, objectKey));
}
}
}
else
{
// if the above condition was not met then we must have enumerated past the end of the indexed value
yield break;
}
}
}
public void BulkSetEnumerateFromKey()
{
string path = Path.GetFullPath("TestData\\BulkSetEnumerateFromKey");
var timer = new Stopwatch();
int totalSize = 0;
int readSize = 0;
using (var db = new KeyValueStore(path)) {
db.Truncate();
db.Manifest.Logger = (msg) => Console.WriteLine(msg);
timer.Start();
for (int i = 0; i < 105000; i++) {
var randomKey = BitConverter.GetBytes(i).Reverse().ToArray();
var randomValue = BitConverter.GetBytes(i);
db.Set(randomKey, randomValue);
readSize += randomKey.Length + randomValue.Length;
totalSize += randomKey.Length + randomValue.Length;
}
timer.Stop();
Console.WriteLine("Wrote sorted table at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
Console.WriteLine("Begin enumeration.");
timer.Start();
int lastKeyNum = 0;
int ct = 0;
int sum = 0;
var searchKey = BitConverter.GetBytes(50000).Reverse().ToArray();
foreach (var pair in db.EnumerateFromKey( searchKey )) {
try {
int num = BitConverter.ToInt32(pair.Key.Reverse().ToArray(),0);
Assert.GreaterOrEqual(num, 50000);
sum += num;
Assert.Less(lastKeyNum, num);
lastKeyNum = num;
ct++;
} catch (Exception /*e*/) {
//Console.WriteLine("Key: {0}\n{1}",insertedItem.Key,e);
//Debugger.Launch();
//db.Get(insertedItem.Key.InternalBytes);
//db.Manifest.LogContents();
throw;
}
}
timer.Stop();
Assert.AreEqual(55000, ct, "55000 items should be enumerated.");
Console.WriteLine("Enumerated read throughput of {0} MB/s (avg {1} ms per 1000 items)", (double)readSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)105);
}
}
public void BulkSetEnumerateAll3()
{
string path = Path.GetFullPath("TestData\\BulkSetEnumerateAll3");
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
int totalSize = 0;
db.Manifest.Logger = msg => Console.WriteLine(msg);
int num_items = 1000000;
timer.Start();
for (int i = 0; i < num_items; i++) {
byte[] key = new byte[8];
BitConverter.GetBytes(i % 100).CopyTo(key,0);
BitConverter.GetBytes(i).CopyTo(key,4);
byte[] value = BitConverter.GetBytes(i);
db.Set(key, value);
totalSize += 8 + 4;
}
timer.Stop();
Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
timer.Start();
var ctModZeros = db.EnumerateFromKey(BitConverter.GetBytes(0)).Count();
timer.Stop();
Console.WriteLine("Scanned index at a throughput of {0} items/s", (double) ctModZeros / timer.Elapsed.TotalSeconds);
}
}
public void RotationBulkRead()
{
string path = Path.GetFullPath("TestData\\RotationBulkReadRace");
using (var db = new KeyValueStore(path)) {
db.Truncate();
int num_items = 30000;
byte[] split = BitConverter.GetBytes(num_items >> 2);
int number_we_should_scan = 0;
for (int i = 0; i < num_items; i++) {
byte[] key = BitConverter.GetBytes(i);
if (ByteArray.CompareMemCmp(split, key) <= 0)
number_we_should_scan++;
}
Console.WriteLine("Number to Scan: {0}", number_we_should_scan);
Console.WriteLine("Writing {0} items.", num_items);
for (int i = 0; i < num_items; i++) {
byte[] key = BitConverter.GetBytes(i);
byte[] value = Encoding.UTF8.GetBytes("Number " + i.ToString());
db.Set(key, value);
}
Assert.AreEqual(number_we_should_scan, db.EnumerateFromKey(split).Count());
}
}
public void RemoveUpdatedValuesFromIndex2()
{
string path = Path.GetFullPath("TestData\\RemoveUpdatedValuesFromIndex2");
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
int totalSize = 0;
db.Manifest.Logger = msg => Console.WriteLine(msg);
var indexed = new SortedDictionary<string, byte[]>();
int num_items = 1000;
timer.Start();
for (int i = 0; i < num_items; i++) {
indexed["Mod"] = BitConverter.GetBytes(i % 100);
db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i), indexed);
totalSize += 8 + 4;
}
timer.Stop();
Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
timer.Start();
var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count();
timer.Stop();
Assert.AreEqual(10, ctModZeros);
Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds);
}
// Open the index directly and see if the data is there
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(10, num_vals);
}
// Re-open the main key-value store and update the value at 30
using (var db = new KeyValueStore(path)) {
var indexed = new SortedDictionary<string, byte[]>();
indexed["Mod"] = BitConverter.GetBytes(201 % 100);
db.Set(BitConverter.GetBytes(200), BitConverter.GetBytes(200), indexed);
// Clean the data from the index
db.RemoveFromIndex(BitConverter.GetBytes(200), new Dictionary<string, byte[]> { { "Mod", BitConverter.GetBytes(200 % 100) } });
}
// Open the index again directly and confirm that the lookup key is gone now as well
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(9, num_vals);
}
}