public void T12_RecordKeyDecodePerfTest()
{
RecordKey rk = new RecordKey().appendParsedKey(".data/test/unpack/with/lots/of/parts");
int NUM_ITERATIONS = 100000;
// encode test
{
GC.Collect();
DateTime start = DateTime.Now;
for (int x = 0; x < NUM_ITERATIONS; x++) {
byte[] data = rk.encode();
}
DateTime end = DateTime.Now;
double elapsed_s = (end - start).TotalMilliseconds / 1000.0;
double rec_per_s = (double)NUM_ITERATIONS / elapsed_s;
Console.WriteLine("packed {0} record keys in {1} seconds, {2} keys/second",
NUM_ITERATIONS, elapsed_s, rec_per_s);
Assert.Less(500000, rec_per_s, "minimum records per second of pack");
}
// decode test
{
byte[] data = rk.encode();
GC.Collect();
DateTime start = DateTime.Now;
for (int x = 0; x < NUM_ITERATIONS; x++) {
RecordKey unpacked = new RecordKey(data);
}
DateTime end = DateTime.Now;
double elapsed_s = (end - start).TotalMilliseconds / 1000.0;
double rec_per_s = (double)NUM_ITERATIONS / elapsed_s;
Console.WriteLine("unpacked {0} record keys in {1} seconds, {2} keys/second",
NUM_ITERATIONS, elapsed_s, rec_per_s);
Assert.Less(500000, rec_per_s, "minimum records per second of unpack");
}
}
public static void Block_Perftest(IBlockTestFactory factory)
{
// iterate through blocksizes, randomly generating input data, and then doing some
// random key queries to see how fast retrieval is
int[] block_sizes = { 2 * 1024, 40 * 1024, 100 * 1024, 512 * 1025, 2 * 1024 * 1024 };
int[] value_sizes = { 10, 30, 100, 1000, 10000 };
int[] num_levels = { 2, 3, 4 };
int[,] perf_results = new int[block_sizes.Length, value_sizes.Length];
int READ_COUNT = 3000;
Random rnd = new Random((int)DateTime.Now.ToBinary());
foreach (int key_part_count in num_levels) {
System.Console.WriteLine("--");
foreach (int block_size in block_sizes) {
foreach (int value_size in value_sizes) {
if (value_size > (block_size / 8)) {
// we want at least 8 values
continue;
}
System.GC.Collect();
// setup the block for encoding
ISegmentBlockEncoder enc = factory.makeEncoder();
MemoryStream ms = new MemoryStream();
enc.setStream(ms);
int curblock_size = 0;
// do the sorted block create.. we nest it so we can dispose the SkipList
{
var sorted_input = new BDSkipList<RecordKey, RecordUpdate>();
// first create the sorted input
while (curblock_size < block_size) {
// generate a random key
RecordKey key = new RecordKey();
for (int i = 0; i < key_part_count; i++) {
key.appendParsedKey("" + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF));
}
// generate a random value
byte[] data = new byte[value_size];
for (int i = 0; i < value_size; i++) {
data[i] = (byte)rnd.Next(40, 50);
}
RecordUpdate upd = RecordUpdate.WithPayload(data);
curblock_size += key.encode().Length;
curblock_size += value_size;
sorted_input.Add(key, upd);
}
// encode the block
foreach (var kvp in sorted_input) {
enc.add(kvp.Key, kvp.Value);
}
enc.flush();
sorted_input = null; // free the skiplist
}
// init the decoder
ISegmentBlockDecoder dec = factory.makeDecoder(new BlockAccessor(ms.ToArray()));
int num_misses = 0;
System.GC.Collect(); // force GC so it may not happen during the test
// perform random access test
DateTime start = DateTime.Now;
for (int i = 0; i < READ_COUNT; i++) {
RecordKey key = new RecordKey();
for (int ki = 0; ki < key_part_count; ki++) {
key.appendParsedKey("" + rnd.Next(8) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF));
}
try {
dec.FindNext(key, true);
} catch (KeyNotFoundException) {
num_misses++;
// System.Console.WriteLine("misfetch: {0}", key);
// no problem, but this shouuld be small
}
}
double duration_ms = (DateTime.Now - start).TotalMilliseconds;
double reads_per_second = (READ_COUNT * 1000.0) / (duration_ms);
System.Console.WriteLine("BlockSize src{0,10} final{6,10} ratio ({7:0.000}), ValueSize {1,6}, Keyparts {5,3}, {2,6} reads in {3,10:0.0}ms, {8,6} misses, {4,9:0.00} read/sec",
curblock_size, value_size, READ_COUNT, duration_ms, reads_per_second,
key_part_count, ms.Length, ((double)ms.Length / (double)curblock_size) * (double)100.0, num_misses);
}
}
}
}
public void add(RecordKey key, RecordUpdate data)
{
byte[] keybytes = key.encode();
byte[] databytes = data.encode();
RecordInfo ri;
ri.record_start_pos = _curPos();
ri.key_len = keybytes.Length;
ri.data_len = databytes.Length;
output.Write(keybytes, 0, keybytes.Length);
output.Write(databytes, 0, databytes.Length);
record_offsets.Add(ri);
}
public void add(RecordKey key, RecordUpdate data)
{
byte[] keybytes = key.encode();
byte[] databytes = data.encode();
writeEncoded(output,keybytes);
output.WriteByte(KEY_VAL_SEP);
writeEncoded(output,databytes);
output.WriteByte(END_OF_LINE);
}
public void T06_RecordKeyDelimiterEscape()
{
string DELIM = new String(RecordKey.PRINTED_DELIMITER, 1);
RecordKey key1 = new RecordKey();
key1.appendKeyParts("1", "2", "3");
Assert.AreEqual(3, key1.numParts());
RecordKey dkey1 = new RecordKey(key1.encode());
Assert.AreEqual(3, dkey1.numParts(), "dkey1 delimiter decode");
RecordKey key2 = new RecordKey();
key2.appendKeyPart("1" + DELIM + "2" + DELIM + "3");
Assert.AreEqual(1, key2.numParts());
RecordKey dkey2 = new RecordKey(key2.encode());
Assert.AreEqual(1, dkey2.numParts(), "dkey2 delimiter decode");
// key2 > key1
Assert.AreEqual(1, key2.CompareTo(key1));
Assert.AreEqual(-1, key1.CompareTo(key2));
}
public void T10_RecordKeyEncodeDecodeBugTest()
{
// test encode/decode with byte[] parts
// 92 43 0
byte[] chars = { 92, 43, 0 };
{
System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding();
String keystring = enc.GetString(chars);
enc.GetBytes(keystring);
Assert.AreEqual(chars, enc.GetBytes(keystring), "string encoding not reversible");
RecordKey key = new RecordKey();
key.appendKeyPart(chars);
byte[] data = key.encode();
Assert.AreEqual(key, new RecordKey(data), "check encode/decode with binary data");
// check nested key
var wrap_key = new RecordKey().appendKeyPart(key.encode());
byte[] wrap_encoded = wrap_key.encode();
RecordKey wrap_decoded = new RecordKey(wrap_encoded);
}
}
public void T02b_RecordKeyNesting()
{
String[] parts1 = { "aaaa", "bbbb" };
String[] parts2 = { "xxxx", "yyyy", "zzzz" };
RecordKey nestedkey = new RecordKey();
nestedkey.appendKeyParts(parts2);
RecordKey parentkey = new RecordKey();
parentkey.appendKeyPart(parts1[0]); // "aaaa"
parentkey.appendKeyPart(nestedkey.encode()); // (xxxx,yyyy,zzzz)
parentkey.appendKeyPart(parts1[1]); // "bbbb"
RecordKey decodedkey = new RecordKey(parentkey.encode());
Assert.AreEqual(decodedkey.ToString(), parentkey.ToString(), "tostring versions of keys don't match");
Assert.AreEqual(decodedkey.numParts(), parentkey.numParts(), "nested delimiters are changing number of keyparts");
Assert.AreEqual(decodedkey, parentkey, "nested key encode/decode error");
}
public void T01_RecordKey()
{
String[] parts1 = { "test", "test2", "blah" };
{
RecordKey key = new RecordKey();
key.appendKeyParts(parts1);
byte[] data = key.encode();
// decode
RecordKey key2 = new RecordKey(data);
// verify tostring matches
Assert.AreEqual(key.ToString(), key2.ToString());
// verify comparison
Assert.AreEqual(0, key.CompareTo(key2));
// verify individual parts
}
// verify hierarchial sorting
{
RecordKey key1 = new RecordKey().appendParsedKey(".ROOT/GEN/000");
RecordKey key2 = new RecordKey().appendParsedKey(".ROOT/GEN/000/</aaaa"); // ord('<') -> 60
RecordKey key3 = new RecordKey().appendParsedKey(".ROOT/GEN/000/>"); // ord('>') -> 62
RecordKey key4 = new RecordKey().appendParsedKey(".ROOT/GEN");
Assert.AreEqual(true, ">".CompareTo("<") > 0, "expect '>' > '<'");
Assert.AreEqual(true, key1.CompareTo(key2) < 0, "prefix key should be considered earlier");
Assert.AreEqual(true, key3.CompareTo(key2) > 0, "but parts are considered individually before using length ord('_') > ord('<')");
IComparable<RecordKey> prefixend = RecordKey.AfterPrefix(key1);
Assert.AreEqual(true, prefixend.CompareTo(key1) > 0, "prefix1");
Assert.AreEqual(true, prefixend.CompareTo(key2) > 0, "prefix2");
Assert.AreEqual(true, prefixend.CompareTo(key3) > 0, "prefix3");
Assert.AreEqual(true, prefixend.CompareTo(key4) > 0, "prefix4");
}
}
