public void AddObjectsAndLookup()
{
string path = Path.GetFullPath("TestData\\AddObjectsAndLookup");
using (var db = new KeyValueStore(path)) {
db.Truncate();
var indexed = new SortedDictionary <string, byte[]>();
indexed["NumberType"] = Encoding.UTF8.GetBytes("Fib");
db.Set(BitConverter.GetBytes(112), Encoding.UTF8.GetBytes("112"), indexed);
db.Set(BitConverter.GetBytes(1123), Encoding.UTF8.GetBytes("1123"), indexed);
db.Set(BitConverter.GetBytes(11235), Encoding.UTF8.GetBytes("11235"), indexed);
db.Set(BitConverter.GetBytes(112358), Encoding.UTF8.GetBytes("112358"), indexed);
indexed["NumberType"] = Encoding.UTF8.GetBytes("Seq");
db.Set(BitConverter.GetBytes(1), Encoding.UTF8.GetBytes("1"), indexed);
db.Set(BitConverter.GetBytes(2), Encoding.UTF8.GetBytes("2"), indexed);
db.Set(BitConverter.GetBytes(3), Encoding.UTF8.GetBytes("3"), indexed);
db.Set(BitConverter.GetBytes(4), Encoding.UTF8.GetBytes("4"), indexed);
indexed["NumberType"] = Encoding.UTF8.GetBytes("Zero");
db.Set(BitConverter.GetBytes(0), Encoding.UTF8.GetBytes("0"), indexed);
}
using (var db = new KeyValueStore(path)) {
var zeros = db.Find("NumberType", Encoding.UTF8.GetBytes("Zero")).ToList();
Assert.AreEqual(1, zeros.Count());
Assert.AreEqual("0", Encoding.UTF8.GetString(zeros[0].Value));
var seqs = db.Find("NumberType", Encoding.UTF8.GetBytes("Seq")).ToList();
Assert.AreEqual(4, seqs.Count());
Assert.AreEqual("1", Encoding.UTF8.GetString(seqs[0].Value));
Assert.AreEqual("2", Encoding.UTF8.GetString(seqs[1].Value));
Assert.AreEqual("3", Encoding.UTF8.GetString(seqs[2].Value));
Assert.AreEqual("4", Encoding.UTF8.GetString(seqs[3].Value));
var fib = db.Find("NumberType", Encoding.UTF8.GetBytes("Fib")).ToList();
Assert.AreEqual(4, seqs.Count());
Assert.AreEqual("1123", Encoding.UTF8.GetString(fib[0].Value));
Assert.AreEqual("112", Encoding.UTF8.GetString(fib[1].Value));
Assert.AreEqual("11235", Encoding.UTF8.GetString(fib[2].Value));
Assert.AreEqual("112358", Encoding.UTF8.GetString(fib[3].Value));
var non = db.Find("NoIndex", new byte[] { 23 }).ToList();
Assert.AreEqual(0, non.Count());
non = db.Find("NumberType", Encoding.UTF8.GetBytes("Unfound")).ToList();
Assert.AreEqual(0, non.Count());
}
}
public void LargeDataSetGetWithIndexTest()
{
string path = Path.GetFullPath("TestData\\LargeDataSetGetWithIndexTest");
int totalSize = 0;
int num_items = 500;
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
// Generate a data value that is larger than the block size.
var value = ByteArray.Random(Config.SortedBlockSize + 256);
var indexed = new SortedDictionary <string, byte[]>();
// Do it enough times to ensure a roll-over
for (int i = 0; i < num_items; i++)
{
var key = BitConverter.GetBytes(i);
indexed["Index"] = BitConverter.GetBytes(i + 10);
db.Set(key, value.InternalBytes, indexed);
totalSize += value.InternalBytes.Length;
}
timer.Start();
for (int i = 0; i < num_items; i++)
{
var items = db.Find("Index", BitConverter.GetBytes(i + 10));
var val = items.First();
Assert.AreEqual(value.InternalBytes, val.Value);
}
timer.Stop();
Console.WriteLine("Randomized read throughput of {0} MB/s (avg {1} ms per lookup)", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0), (double)timer.Elapsed.TotalSeconds / (double)num_items);
}
}
public void AddObjectsAndLookupWhileMerging()
{
string path = Path.GetFullPath("TestData\\AddObjectsAndLookupWhileMerging");
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
int totalSize = 0;
db.Manifest.Logger = msg => Console.WriteLine(msg);
var indexed = new SortedDictionary <string, byte[]>();
int num_items = 1000000;
timer.Start();
for (int i = 0; i < num_items; i++)
{
indexed["Mod"] = BitConverter.GetBytes(i % 100);
db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i * 1000), indexed);
totalSize += 8 + 4;
}
timer.Stop();
Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
timer.Start();
var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count();
timer.Stop();
Assert.AreEqual(10000, ctModZeros);
Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds);
}
}
public void FindStartsWith()
{
string path = Path.GetFullPath("TestData\\FindStartsWith");
using (var db = new KeyValueStore(path)) {
db.Truncate();
var indexed = new SortedDictionary <string, byte[]>();
indexed["Bytes"] = Encoding.UTF8.GetBytes("112");
db.Set(BitConverter.GetBytes(112), Encoding.UTF8.GetBytes("112"), indexed);
indexed["Bytes"] = Encoding.UTF8.GetBytes("1123");
db.Set(BitConverter.GetBytes(1123), Encoding.UTF8.GetBytes("1123"), indexed);
indexed["Bytes"] = Encoding.UTF8.GetBytes("11235");
db.Set(BitConverter.GetBytes(11235), Encoding.UTF8.GetBytes("11235"), indexed);
indexed["Bytes"] = Encoding.UTF8.GetBytes("112358");
db.Set(BitConverter.GetBytes(112358), Encoding.UTF8.GetBytes("112358"), indexed);
}
using (var db = new KeyValueStore(path)) {
var exact = db.Find("Bytes", Encoding.UTF8.GetBytes("1123")).ToList();
Assert.AreEqual(1, exact.Count());
Assert.AreEqual("1123", Encoding.UTF8.GetString(exact[0].Value));
var startsWith = db.FindStartsWith("Bytes", Encoding.UTF8.GetBytes("1123")).ToList();
Assert.AreEqual(3, startsWith.Count());
Assert.AreEqual("112358", Encoding.UTF8.GetString(startsWith[0].Value));
Assert.AreEqual("11235", Encoding.UTF8.GetString(startsWith[1].Value));
Assert.AreEqual("1123", Encoding.UTF8.GetString(startsWith[2].Value));
}
}
public void IndexClean()
{
string path = Path.GetFullPath("TestData\\IndexClean");
using (var db = new KeyValueStore(path)) {
db.Truncate();
db.Manifest.Logger = msg => Console.WriteLine(msg);
db.Set(Encoding.UTF8.GetBytes("KeyA"), Encoding.UTF8.GetBytes("ValueA:1"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("1") }
});
db.Set(Encoding.UTF8.GetBytes("KeyB"), Encoding.UTF8.GetBytes("ValueB:2"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("2") }
});
db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:3"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("3") }
});
var lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("3")).Single();
Assert.AreEqual("ValueC:3", Encoding.UTF8.GetString(lookupValue.Value));
Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key));
db.Delete(Encoding.UTF8.GetBytes("KeyC"));
}
// Open the index directly and confirm that the lookup key is still there
using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) Assert.AreEqual(3, db.Enumerate().Count());
using (var db = new KeyValueStore(path)) db.CleanIndex("Idx");
// Open the index directly and confirm that the lookup key is now gone
using (var db = new KeyValueStore(Path.Combine(path, "Idx"))) Assert.AreEqual(2, db.Enumerate().Count());
}
public void RemoveUpdatedValuesFromIndex2()
{
string path = Path.GetFullPath("TestData\\RemoveUpdatedValuesFromIndex2");
var timer = new Stopwatch();
using (var db = new KeyValueStore(path)) {
db.Truncate();
int totalSize = 0;
db.Manifest.Logger = msg => Console.WriteLine(msg);
var indexed = new SortedDictionary <string, byte[]>();
int num_items = 1000;
timer.Start();
for (int i = 0; i < num_items; i++)
{
indexed["Mod"] = BitConverter.GetBytes(i % 100);
db.Set(BitConverter.GetBytes(i), BitConverter.GetBytes(i), indexed);
totalSize += 8 + 4;
}
timer.Stop();
Console.WriteLine("Wrote data (with indexing) at a throughput of {0} MB/s", (double)totalSize / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
timer.Reset();
timer.Start();
var ctModZeros = db.Find("Mod", BitConverter.GetBytes((int)0)).Count();
timer.Stop();
Assert.AreEqual(10, ctModZeros);
Console.WriteLine("Scanned index at a throughput of {0} items/s", (double)ctModZeros / timer.Elapsed.TotalSeconds);
}
// Open the index directly and see if the data is there
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(10, num_vals);
}
// Re-open the main key-value store and update the value at 30
using (var db = new KeyValueStore(path)) {
var indexed = new SortedDictionary <string, byte[]>();
indexed["Mod"] = BitConverter.GetBytes(201 % 100);
db.Set(BitConverter.GetBytes(200), BitConverter.GetBytes(200), indexed);
// Clean the data from the index
db.RemoveFromIndex(BitConverter.GetBytes(200), new Dictionary <string, byte[]> {
{ "Mod", BitConverter.GetBytes(200 % 100) }
});
}
// Open the index again directly and confirm that the lookup key is gone now as well
using (var db = new KeyValueStore(Path.Combine(path, "Mod"))) {
int num_vals = db.EnumerateFromKey(BitConverter.GetBytes((int)0)).Count(pair => pair.Key.Take(4).All(b => b == 0));
Assert.AreEqual(9, num_vals);
}
}
public void LookupOldDataFromIndex()
{
string path = Path.GetFullPath("TestData\\LookupOldDataFromIndex");
using (var db = new KeyValueStore(path)) {
db.Truncate();
db.Manifest.Logger = msg => Console.WriteLine(msg);
db.Set(Encoding.UTF8.GetBytes("KeyA"), Encoding.UTF8.GetBytes("ValueA:1"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("1") }
});
db.Set(Encoding.UTF8.GetBytes("KeyB"), Encoding.UTF8.GetBytes("ValueB:2"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("2") }
});
db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:3"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("3") }
});
var lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("3")).Single();
Assert.AreEqual("ValueC:3", Encoding.UTF8.GetString(lookupValue.Value));
Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key));
db.Set(Encoding.UTF8.GetBytes("KeyC"), Encoding.UTF8.GetBytes("ValueC:4"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("4") }
});
lookupValue = db.Find("Idx", Encoding.UTF8.GetBytes("4")).Single();
Assert.AreEqual("ValueC:4", Encoding.UTF8.GetString(lookupValue.Value));
Assert.AreEqual("KeyC", Encoding.UTF8.GetString(lookupValue.Key));
Assert.True(db.Find("Idx", Encoding.UTF8.GetBytes("3")).Any());
db.RemoveFromIndex(Encoding.UTF8.GetBytes("KeyC"), new Dictionary <string, byte[]> {
{ "Idx", Encoding.UTF8.GetBytes("3") }
});
Assert.False(db.Find("Idx", Encoding.UTF8.GetBytes("3")).Any());
}
}
} // create
public dict activate(IContext ctx, idstring userid, varchar activationkey)
{
dict Return = new dict();
// your code goes here; add return values to 'Return'
bool status = false;
long pver;
dict u = KeyValueStore.Find("user:"******"activationkey"].Equals(activationkey.str))
{
u["activationkey"] = "";
status = KeyValueStore.Store("user:"******"status", status);
return(Return);
} // activate
public dict create(IContext ctx, idstring userid, password password, varchar realname, email email)
{
dict Return = new dict();
// your code goes here; add return values to 'Return'
bool status = false;
string message = "unknown error";
long pver;
if (userid.str.Length < 5 || !Helpers.IsValidId(userid.str))
{
message = "bad user id format: non-special characters, at least length 5 required";
}
else if (KeyValueStore.Find("user:"******"the user id " + userid + " already exists";
}
else if (KeyValueStore.Find("email:" + email, out pver) != null)
{
message = "the email address " + email + " is already tied to another user id";
}
else
{
dict user = new dict();
user.Add("userid", userid.str);
user.Add("password", password.str);
user.Add("realname", realname.str);
user.Add("email", email.str);
user.Add("activationkey", Helpers.RandomString(24));
if (KeyValueStore.Store("user:"******"user " + userid + " created";
status = true;
}
else
{
message = "database error when trying to create user " + userid;
}
}
Return.Add("status", status);
Return.Add("message", message);
return(Return);
} // create