public void T001_FullScanWithOnlyWorkingSegment()
{
LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\31");
db.setValueParsed("test/1", "a");
Assert.Fail("test not implemented");
}
public void T05_TestBackgroundFlushAndMergeConsistency()
{
// (1) one key to each of NUM_SEGMENTS separate segments
// (2) setup a separate thread that just repeatedly checks each key
// (3) perform a merge
// (4) shutdown the threads and see if any detected a readback failure
LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\7");
List<ValueCheckerThread> checkers = new List<ValueCheckerThread>();
int NUM_SEGMENTS = 7;
int performed_iterations = 0;
int TARGET_ITERATIONS = 10;
try {
for (int iter = 0; iter < TARGET_ITERATIONS; iter++) {
// setup the initial keys and checker threads
for (int x = 0; x < NUM_SEGMENTS; x++) {
string key = "test-" + x;
string value = "test-value-" + x;
db.setValueParsed(key, value);
// put it in it's own segment
db.flushWorkingSegment();
// start a checking thread
ValueCheckerThread checker = new ValueCheckerThread(db, key, value);
Thread newthread = new Thread(checker.doValidate);
newthread.Start();
checkers.Add(checker);
}
Thread.Sleep(5);
// verify there are no errors
foreach (ValueCheckerThread checker in checkers) {
Console.WriteLine("Thread key:{0} checks:{1} errors:{2}", checker.key_to_check, checker.num_checks, checker.num_errors);
}
foreach (ValueCheckerThread checker in checkers) {
Assert.AreEqual(0, checker.num_errors, "checker thread error, key(" + checker.key_to_check + ") error count != 0");
}
// trigger a merge
for (int x = 0; x < 20; x++) {
db.mergeIfNeeded();
}
Thread.Sleep(5);
// verify there are no errors
foreach (ValueCheckerThread checker in checkers) {
Console.WriteLine("Thread key:{0} checks:{1} errors:{2}", checker.key_to_check, checker.num_checks, checker.num_errors);
}
foreach (ValueCheckerThread checker in checkers) {
Assert.AreEqual(0, checker.num_errors, "checker thread error, key:" + checker.key_to_check);
}
// end the threads..
foreach (ValueCheckerThread checker in checkers) {
checker.end();
}
Thread.Sleep(10);
foreach (ValueCheckerThread checker in checkers) {
checker.waitForEnd();
}
checkers.Clear();
Thread.Sleep(5);
// delete the keys
System.Console.WriteLine("======= Clearing for next run...");
for (int x = 0; x < NUM_SEGMENTS; x++) {
string key = "test-" + x;
string value = "test-value-" + x;
db.setValue(new RecordKey().appendParsedKey(key), RecordUpdate.DeletionTombstone());
}
db.flushWorkingSegment();
Thread.Sleep(5);
db.mergeIfNeeded();
Thread.Sleep(5);
System.Console.WriteLine("======= Cleared for next run...");
performed_iterations = iter;
}
} finally {
System.Console.WriteLine("performed iterations = " + performed_iterations + " target iterations = " + TARGET_ITERATIONS);
}
}
public void T002_ScanDirections()
{
LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\3");
var rk_a = new RecordKey().appendParsedKey(".a");
var rk_b = new RecordKey().appendParsedKey(".b");
string[] keys = { ".a", ".b" };
foreach (var key in keys) {
db.setValueParsed(key, "valueof:" + key);
}
{
var rec = db.FindNext(rk_a, false);
Assert.AreEqual(rk_b, rec.Key, "simple FindNext");
}
{
var rec = db.FindPrev(rk_b, false);
Assert.AreEqual(rk_a, rec.Key, "simple FindPrev");
}
var scan_range = new ScanRange<RecordKey>(rk_a, rk_b, null);
// scan forward
int count = 0;
foreach (var row in db.scanForward(scan_range)) {
Console.WriteLine("forward scan: " + row);
if (count == keys.Length) {
Assert.Fail("too many keys returned in scan");
}
Assert.AreEqual(new RecordKeyType_String(keys[count]), row.Key.key_parts[0], "forward scan mistake");
count++;
}
if (count != keys.Length) {
Assert.Fail("not enough keys returned in scan");
}
// scan backward
count = keys.Length;
foreach (var row in db.scanBackward(scan_range)) {
Console.WriteLine("backward scan: " + row);
if (count == 0) {
Assert.Fail("too many keys returned in scan backward");
}
count--;
Assert.AreEqual(new RecordKeyType_String(keys[count]), row.Key.key_parts[0], "backward scan mistake");
}
if (count != 0) {
Assert.Fail("not enough keys returned in scan");
}
}
public void T001_WorkingSegmentReadWrite()
{
LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\3");
var rk = new RecordKey().appendParsedKey(".a");
db.setValueParsed(".a", "1");
KeyValuePair<RecordKey, RecordData> record;
try {
record = db.FindNext(rk, true);
Assert.AreEqual(rk, record.Key, "fetched key does not match");
} catch (KeyNotFoundException) {
Assert.Fail("couldn't find 'a' record");
}
int found_recs = 0;
var scan_range = new ScanRange<RecordKey>(rk, RecordKey.AfterPrefix(rk), null);
foreach (var row in db.scanForward(scan_range)) {
found_recs++;
}
Assert.AreEqual(1, found_recs, "found the wrong number of records in working segment scan!");
db.flushWorkingSegment();
try {
record = db.FindNext(rk, true);
Assert.AreEqual(rk, record.Key, "fetched key does not match (after flush)");
} catch (KeyNotFoundException) {
Assert.Fail("couldn't find 'a' record");
}
found_recs = 0;
foreach (var row in db.scanForward(
new ScanRange<RecordKey>(rk, RecordKey.AfterPrefix(rk), null))) {
found_recs++;
}
Assert.AreEqual(1, found_recs, "found the wrong number of records after flush !");
}
public void T001_MultiWorkingSegmentReadWrite()
{
LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\3");
var rk = new RecordKey().appendParsedKey(".data/a");
db.setValueParsed(".data/a", "1");
KeyValuePair<RecordKey, RecordData> record;
try {
record = db.FindNext(rk, true);
Assert.AreEqual(rk, record.Key, "fetched key does not match");
} catch (KeyNotFoundException) {
Assert.Fail("couldn't find 'a' record");
}
int found_recs = 0;
var scan_range = new ScanRange<RecordKey>(rk, RecordKey.AfterPrefix(rk), null);
foreach (var row in db.scanForward(scan_range)) {
found_recs++;
}
Assert.AreEqual(1, found_recs, "found the wrong number of records in working segment scan!");
db.DEBUG_addNewWorkingSegmentWithoutFlush();
db.setValueParsed(".data/b", "2");
Console.WriteLine("");
Console.WriteLine("--- contents --");
db.debugDump();
Console.WriteLine("");
// ------------------------------
try {
var rkb = new RecordKey().appendParsedKey(".data/b");
record = db.FindNext(rkb, true);
Assert.AreEqual(rkb, record.Key, "fetched key does not match (after flush)");
} catch (KeyNotFoundException) {
Assert.Fail("couldn't find 'b' record");
}
found_recs = 0;
var rk_prefix = new RecordKey().appendParsedKey(".data");
foreach (var row in db.scanForward(
new ScanRange<RecordKey>(rk_prefix, RecordKey.AfterPrefix(rk_prefix), null))) {
found_recs++;
}
Assert.AreEqual(2, found_recs, "found the wrong number of records after working segment addition !");
}
public static void do_bringup_test(DbgGUI win)
{
LayerManager db = new LayerManager(InitMode.NEW_REGION,"c:\\BENDtst\\main");
db.setValueParsed("test/3","a");
db.setValueParsed("test/2","b");
db.setValueParsed("test/1","c");
db.debugDump();
db.flushWorkingSegment(); // this will flush and read the current segment
Console.WriteLine("--- after flush");
db.debugDump();
dumpMergeCandidates(db);
Console.WriteLine("--- check record read");
RecordData data;
GetStatus status = db.getRecord(new RecordKey().appendParsedKey("test/3"), out data);
System.Console.WriteLine("getRecord({0}) returned {1}", "test/3", data.ToString());
Console.WriteLine("--- make lots of segments");
db.setValueParsed("test/4", "d");
db.flushWorkingSegment();
db.setValueParsed("test/5", "e");
db.flushWorkingSegment();
db.setValueParsed("test/6", "f");
db.flushWorkingSegment();
db.debugDump();
System.Console.WriteLine("-------- dump keys ---------------------");
dumpAllDbRecords(db);
dumpMergeCandidates(db);
System.Console.WriteLine("-------- draw graphical debug ---------------------");
win.debugDump(db);
System.Console.WriteLine("-------- PERFORMING A SINGLE MERGE ---------------------");
MergeCandidate mc;
mc = db.rangemapmgr.mergeManager.getBestCandidate();
System.Console.WriteLine("MERGE :" + mc);
db.performMerge(mc);
dumpMergeCandidates(db);
db.flushWorkingSegment();
db.debugDump();
dumpSegmentList(db);
win.debugDump(db);
System.Console.WriteLine("-------- SINGLE MERGE DONE, close/dispose ---------------------");
dumpSegmentList(db);
dumpMergeCandidates(db);
db.debugDump();
db.Dispose();
System.Console.WriteLine("-------- NOW RESUME ---------------------------------");
db = new LayerManager(InitMode.RESUME, "c:\\BENDtst\\main");
dumpSegmentList(db);
dumpMergeCandidates(db);
win.debugDump(db);
db.debugDump();
System.Console.WriteLine("-------- NOW FINDNEXT ---------------------------------");
dumpAllDbRecords(db);
win.debugDump(db);
System.Console.WriteLine("-------- NOW MERGE ALL SEGMENTS ---------------------------------");
dumpSegmentList(db);
db.mergeAllSegments();
db.debugDump();
win.debugDump(db);
// stop(); // ------------------------- (( S T O P )) ---------------------------------
System.Console.WriteLine("-------- NOW FINDNEXT (after merge) ---------------------------------");
dumpAllDbRecords(db);
//System.Console.WriteLine("-------- Now run Readthreads Test ---------------------------------");
//A03_LayerManagerTests test = new A03_LayerManagerTests();
//test.T10_LayerManager_ReadThreads();
dumpMergeCandidates(db);
win.debugDump(db);
db.Dispose();
System.Console.WriteLine("-------- Write ALOT of data ---------------------------------");
int keysize = 20
;
int keycount = 1000000;
int flush_period = 40000;
int commit_period = 1000;
bool random_order = true;
double elapsed;
DateTime start = DateTime.Now;
int record_count = 0;
db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\bigtest");
String value = "";
String keybase = "TestValueDataABC";
for (int x = 0; x < keysize / keybase.Length; x++) { value = value + keybase; }
int seed = (int)DateTime.Now.Ticks;
Random rnd = new Random(seed);
System.Console.WriteLine("*** RANDOM SEED: " + seed);
var write_group = db.newWriteGroup();
for (int x = 10000001; x < 10000001 + keycount; x++) {
if (random_order) {
write_group.setValueParsed("test/rnd/" + rnd.Next(), value);
} else {
write_group.setValueParsed("test/ordered/" + x, value);
}
record_count++;
if (x % commit_period == 0) { write_group.finish(); write_group = db.newWriteGroup(); }
if (x % flush_period == 0) {
System.Console.WriteLine("*** RANDOM SEED: " + seed);
write_group.finish(); write_group = db.newWriteGroup();
System.Console.WriteLine("start % 1000 cycle..");
db.flushWorkingSegment();
db.freespacemgr.debugDumbCurrentFreespace();
win.debugDump(db);
dumpMergeCandidates(db);
for (int mx = 0; mx < 30; mx++) {
mc = db.rangemapmgr.mergeManager.getBestCandidate();
if (mc == null) { break; }
if (mc.score() > (1.6 + (float)db.rangemapmgr.mergeManager.getMaxGeneration()/12.0f)) {
System.Console.WriteLine("** best merge score too high: " + mc);
break;
}
System.Console.WriteLine("merge " + mx + " : " + mc);
win.debugDump(db, mc);
db.performMerge(mc);
System.Console.WriteLine("mergedone " + mx + " : " + mc);
dumpSegmentList(db);
dumpMergeCandidates(db);
win.debugDump(db);
}
elapsed = (DateTime.Now - start).TotalMilliseconds / 1000.0;
System.GC.Collect();
Console.WriteLine("*************************************************");
Console.WriteLine("*************************************************");
Console.WriteLine("*** merge cycle done {0} records so far, in {1} total time, {2} records/second",
record_count,elapsed, (double)record_count/elapsed);
Console.WriteLine("*************************************************");
Console.WriteLine("*************************************************");
}
}
System.Console.WriteLine("-------- Merge a bunch more ------------------");
for (int x = 0; x < 30; x++) {
mc = db.rangemapmgr.mergeManager.getBestCandidate();
System.Console.WriteLine("merge : " + mc);
if (mc == null) break;
win.debugDump(db, mc);
db.performMerge(mc);
dumpSegmentList(db);
dumpMergeCandidates(db);
win.debugDump(db,null);
}
dumpSegmentList(db);
elapsed = (DateTime.Now - start).TotalMilliseconds / 1000.0;
Console.WriteLine("*************************************************");
Console.WriteLine("*************************************************");
Console.WriteLine("*** merge cycle done {0} records so far, in {1} total time, {2} records/second",
record_count, elapsed, (double)record_count / elapsed);
Console.WriteLine("*************************************************");
Console.WriteLine("*************************************************");
System.Console.WriteLine("** done.");
Environment.Exit(0);
}
