public void T12_RecordKeyDecodePerfTest() { RecordKey rk = new RecordKey().appendParsedKey(".data/test/unpack/with/lots/of/parts"); int NUM_ITERATIONS = 100000; // encode test { GC.Collect(); DateTime start = DateTime.Now; for (int x = 0; x < NUM_ITERATIONS; x++) { byte[] data = rk.encode(); } DateTime end = DateTime.Now; double elapsed_s = (end - start).TotalMilliseconds / 1000.0; double rec_per_s = (double)NUM_ITERATIONS / elapsed_s; Console.WriteLine("packed {0} record keys in {1} seconds, {2} keys/second", NUM_ITERATIONS, elapsed_s, rec_per_s); Assert.Less(500000, rec_per_s, "minimum records per second of pack"); } // decode test { byte[] data = rk.encode(); GC.Collect(); DateTime start = DateTime.Now; for (int x = 0; x < NUM_ITERATIONS; x++) { RecordKey unpacked = new RecordKey(data); } DateTime end = DateTime.Now; double elapsed_s = (end - start).TotalMilliseconds / 1000.0; double rec_per_s = (double)NUM_ITERATIONS / elapsed_s; Console.WriteLine("unpacked {0} record keys in {1} seconds, {2} keys/second", NUM_ITERATIONS, elapsed_s, rec_per_s); Assert.Less(500000, rec_per_s, "minimum records per second of unpack"); } }
public static void Block_Perftest(IBlockTestFactory factory) { // iterate through blocksizes, randomly generating input data, and then doing some // random key queries to see how fast retrieval is int[] block_sizes = { 2 * 1024, 40 * 1024, 100 * 1024, 512 * 1025, 2 * 1024 * 1024 }; int[] value_sizes = { 10, 30, 100, 1000, 10000 }; int[] num_levels = { 2, 3, 4 }; int[,] perf_results = new int[block_sizes.Length, value_sizes.Length]; int READ_COUNT = 3000; Random rnd = new Random((int)DateTime.Now.ToBinary()); foreach (int key_part_count in num_levels) { System.Console.WriteLine("--"); foreach (int block_size in block_sizes) { foreach (int value_size in value_sizes) { if (value_size > (block_size / 8)) { // we want at least 8 values continue; } System.GC.Collect(); // setup the block for encoding ISegmentBlockEncoder enc = factory.makeEncoder(); MemoryStream ms = new MemoryStream(); enc.setStream(ms); int curblock_size = 0; // do the sorted block create.. we nest it so we can dispose the SkipList { var sorted_input = new BDSkipList<RecordKey, RecordUpdate>(); // first create the sorted input while (curblock_size < block_size) { // generate a random key RecordKey key = new RecordKey(); for (int i = 0; i < key_part_count; i++) { key.appendParsedKey("" + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF)); } // generate a random value byte[] data = new byte[value_size]; for (int i = 0; i < value_size; i++) { data[i] = (byte)rnd.Next(40, 50); } RecordUpdate upd = RecordUpdate.WithPayload(data); curblock_size += key.encode().Length; curblock_size += value_size; sorted_input.Add(key, upd); } // encode the block foreach (var kvp in sorted_input) { enc.add(kvp.Key, kvp.Value); } enc.flush(); sorted_input = null; // free the skiplist } // init the decoder ISegmentBlockDecoder dec = factory.makeDecoder(new BlockAccessor(ms.ToArray())); int num_misses = 0; System.GC.Collect(); // force GC so it may not happen during the test // perform random access test DateTime start = DateTime.Now; for (int i = 0; i < READ_COUNT; i++) { RecordKey key = new RecordKey(); for (int ki = 0; ki < key_part_count; ki++) { key.appendParsedKey("" + rnd.Next(8) + rnd.Next(0xFFFFFF) + rnd.Next(0xFFFFFF)); } try { dec.FindNext(key, true); } catch (KeyNotFoundException) { num_misses++; // System.Console.WriteLine("misfetch: {0}", key); // no problem, but this shouuld be small } } double duration_ms = (DateTime.Now - start).TotalMilliseconds; double reads_per_second = (READ_COUNT * 1000.0) / (duration_ms); System.Console.WriteLine("BlockSize src{0,10} final{6,10} ratio ({7:0.000}), ValueSize {1,6}, Keyparts {5,3}, {2,6} reads in {3,10:0.0}ms, {8,6} misses, {4,9:0.00} read/sec", curblock_size, value_size, READ_COUNT, duration_ms, reads_per_second, key_part_count, ms.Length, ((double)ms.Length / (double)curblock_size) * (double)100.0, num_misses); } } } }
public void add(RecordKey key, RecordUpdate data) { byte[] keybytes = key.encode(); byte[] databytes = data.encode(); RecordInfo ri; ri.record_start_pos = _curPos(); ri.key_len = keybytes.Length; ri.data_len = databytes.Length; output.Write(keybytes, 0, keybytes.Length); output.Write(databytes, 0, databytes.Length); record_offsets.Add(ri); }
public void add(RecordKey key, RecordUpdate data) { byte[] keybytes = key.encode(); byte[] databytes = data.encode(); writeEncoded(output,keybytes); output.WriteByte(KEY_VAL_SEP); writeEncoded(output,databytes); output.WriteByte(END_OF_LINE); }
public void T06_RecordKeyDelimiterEscape() { string DELIM = new String(RecordKey.PRINTED_DELIMITER, 1); RecordKey key1 = new RecordKey(); key1.appendKeyParts("1", "2", "3"); Assert.AreEqual(3, key1.numParts()); RecordKey dkey1 = new RecordKey(key1.encode()); Assert.AreEqual(3, dkey1.numParts(), "dkey1 delimiter decode"); RecordKey key2 = new RecordKey(); key2.appendKeyPart("1" + DELIM + "2" + DELIM + "3"); Assert.AreEqual(1, key2.numParts()); RecordKey dkey2 = new RecordKey(key2.encode()); Assert.AreEqual(1, dkey2.numParts(), "dkey2 delimiter decode"); // key2 > key1 Assert.AreEqual(1, key2.CompareTo(key1)); Assert.AreEqual(-1, key1.CompareTo(key2)); }
public void T10_RecordKeyEncodeDecodeBugTest() { // test encode/decode with byte[] parts // 92 43 0 byte[] chars = { 92, 43, 0 }; { System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); String keystring = enc.GetString(chars); enc.GetBytes(keystring); Assert.AreEqual(chars, enc.GetBytes(keystring), "string encoding not reversible"); RecordKey key = new RecordKey(); key.appendKeyPart(chars); byte[] data = key.encode(); Assert.AreEqual(key, new RecordKey(data), "check encode/decode with binary data"); // check nested key var wrap_key = new RecordKey().appendKeyPart(key.encode()); byte[] wrap_encoded = wrap_key.encode(); RecordKey wrap_decoded = new RecordKey(wrap_encoded); } }
public void T02b_RecordKeyNesting() { String[] parts1 = { "aaaa", "bbbb" }; String[] parts2 = { "xxxx", "yyyy", "zzzz" }; RecordKey nestedkey = new RecordKey(); nestedkey.appendKeyParts(parts2); RecordKey parentkey = new RecordKey(); parentkey.appendKeyPart(parts1[0]); // "aaaa" parentkey.appendKeyPart(nestedkey.encode()); // (xxxx,yyyy,zzzz) parentkey.appendKeyPart(parts1[1]); // "bbbb" RecordKey decodedkey = new RecordKey(parentkey.encode()); Assert.AreEqual(decodedkey.ToString(), parentkey.ToString(), "tostring versions of keys don't match"); Assert.AreEqual(decodedkey.numParts(), parentkey.numParts(), "nested delimiters are changing number of keyparts"); Assert.AreEqual(decodedkey, parentkey, "nested key encode/decode error"); }
public void T01_RecordKey() { String[] parts1 = { "test", "test2", "blah" }; { RecordKey key = new RecordKey(); key.appendKeyParts(parts1); byte[] data = key.encode(); // decode RecordKey key2 = new RecordKey(data); // verify tostring matches Assert.AreEqual(key.ToString(), key2.ToString()); // verify comparison Assert.AreEqual(0, key.CompareTo(key2)); // verify individual parts } // verify hierarchial sorting { RecordKey key1 = new RecordKey().appendParsedKey(".ROOT/GEN/000"); RecordKey key2 = new RecordKey().appendParsedKey(".ROOT/GEN/000/</aaaa"); // ord('<') -> 60 RecordKey key3 = new RecordKey().appendParsedKey(".ROOT/GEN/000/>"); // ord('>') -> 62 RecordKey key4 = new RecordKey().appendParsedKey(".ROOT/GEN"); Assert.AreEqual(true, ">".CompareTo("<") > 0, "expect '>' > '<'"); Assert.AreEqual(true, key1.CompareTo(key2) < 0, "prefix key should be considered earlier"); Assert.AreEqual(true, key3.CompareTo(key2) > 0, "but parts are considered individually before using length ord('_') > ord('<')"); IComparable<RecordKey> prefixend = RecordKey.AfterPrefix(key1); Assert.AreEqual(true, prefixend.CompareTo(key1) > 0, "prefix1"); Assert.AreEqual(true, prefixend.CompareTo(key2) > 0, "prefix2"); Assert.AreEqual(true, prefixend.CompareTo(key3) > 0, "prefix3"); Assert.AreEqual(true, prefixend.CompareTo(key4) > 0, "prefix4"); } }