public void setValue(RecordKey skey, RecordUpdate supdate) { checkActive(); // (1) write our repl log entry DateTime now = DateTime.Now; long logstamp = id_gen.nextTimestamp(); RecordKey logkey = new RecordKey() .appendKeyPart("_logs") .appendKeyPart(ctx.server_guid) .appendKeyPart(new RecordKeyType_Long(logstamp)); // (1.1) pack the key/value together into the log entry byte[] packed_update; { MemoryStream writer = new MemoryStream(); // TODO: this seems like a really inefficient way to write out a key ISegmentBlockEncoder encoder = new SegmentBlockBasicEncoder(); encoder.setStream(writer); encoder.add(skey, supdate); encoder.flush(); packed_update = writer.ToArray(); } RecordUpdate logupdate = RecordUpdate.WithPayload(packed_update); Console.WriteLine("writing log entry: {0} -> [ {1} = {2} ]", logkey, skey, supdate); next_stage.setValue(logkey, logupdate); // (2) trigger the repl notifier that there is a new entry to push pusher.wakeUpLogSleepers(); // (2) write the record key Console.WriteLine("writing data entry: {0} = {1}", skey, supdate); RecordKey private_record_key = new RecordKey() .appendKeyPart("_data"); foreach (var part in skey.key_parts) { private_record_key.appendKeyPart(part); } next_stage.setValue(private_record_key, supdate); }
internal void applyLogEntry(string from_server_guid, long logstamp, RecordUpdate logdata) { // (0) unpack the data BlockAccessor ba = new BlockAccessor(logdata.data); ISegmentBlockDecoder decoder = new SegmentBlockBasicDecoder(ba); // (1) add it to our copy of that server's log this._recordLogEntry(from_server_guid, logstamp, logdata); // (2) add it to the database foreach (var kvp in decoder.sortedWalk()) { RecordKey local_data_key = new RecordKey() .appendKeyPart("_data"); foreach (var part in kvp.Key.key_parts) { local_data_key.appendKeyPart(part); } next_stage.setValue(local_data_key, kvp.Value); } }
public void T03_SegmentLayerGetRecordApplicationOrder() { // Assure that when records are written more than once, the updates are applied in the correct // order so we see the proper current data value LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\6"); { String[] keys = { "test-1", "test-2", "test-3" }; String[] values = { "a-first", "b-first", "c-first" }; LayerWriteGroup txn = db.newWriteGroup(); for (int i = 0; i < keys.Length; i++) { txn.setValueParsed(keys[i], values[i]); } txn.finish(); db.flushWorkingSegment(); // assure the records we wrote are NOT in the working segment, but ARE in the next layer for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should be **MISSING* { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should be clear"); } // assure the global query interface finds it { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "records should be found in layers"); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord()"); } } } // now write the same keys again with different values into the working segment { String[] keys = { "test-1", "test-2", "test-3" }; String[] values = { "a-second", "b-second", "c-second" }; System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); LayerWriteGroup txn = db.newWriteGroup(); for (int i = 0; i < keys.Length; i++) { txn.setValueParsed(keys[i], values[i]); } txn.finish(); // assure that both the workingsegment and layermanager see the NEW VALUES for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should be the NEW VALUES { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.PRESENT, status, "working segment should have NEW VALUES"); Assert.AreEqual(values[i], enc.GetString(update.data), "SegmentBuilder.getRecordUpdate should see NEW VALUES"); } // assure the global query interface finds the NEW VALUES { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager should see NEW VALUES"); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord() should see NEW VALUES"); } } // now flush the working segment (so we have two on-disk layers) db.flushWorkingSegment(); // assure we still see the NEW VALUES, but that the working segment is empty for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should MISSING { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should have NO values"); } // assure the global query interface finds the NEW VALUES { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager should see NEW VALUES"); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord() should see NEW VALUES"); } } } db.Dispose(); }
public void T04_SingleSegmentRootMetadataLogRecovery() { // TEST: test multiple segments flushed, and "log resumed" (walk .ROOT range map) // perform the previous test T03_SegmentLayerGetRecordApplicationOrder(); // ... and then perform a resume LayerManager db = new LayerManager(InitMode.RESUME, "c:\\BENDtst\\6"); String[] keys = { "test-1", "test-2", "test-3" }; String[] values = { "a-second", "b-second", "c-second" }; // verify that it has the same data as before the RESUME { // working segment should be empty for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should be MISSING // This is testing the checkpoint as well. If log resume didn't // CHECKPOINT_DROP, then the values will be duplicated in the working segment. { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should be MISSING"); } // assure the global query interface finds the NEW VALUES { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager should see NEW VALUES"); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord() should see NEW VALUES"); } } // now generate a BUNCH of new segments... { String[] secondkeys = { "second-test-1", "second-test-2", "second-test-3" }; String[] secondvalues = { "a-second", "b-second", "c-second" }; // put each new record in its OWN segment for (int i = 0; i < secondkeys.Length; i++) { LayerWriteGroup txn = db.newWriteGroup(); txn.setValueParsed(secondkeys[i], secondvalues[i]); txn.finish(); db.flushWorkingSegment(); } db.Dispose(); db.debugDump(); // RESUME db = new LayerManager(InitMode.RESUME, "c:\\BENDtst\\6"); // first test records should still be visible for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should be MISSING // This is testing the checkpoint as well. If log resume didn't // CHECKPOINT_DROP, then the values will be duplicated in the working segment. { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should be MISSING {0}", key); } // assure the global query interface finds the NEW VALUES { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager should see NEW VALUES : {0}", key); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord() should see NEW VALUES : {0}", key); } } db.debugDump(); // verify that the secondkeys/values are still in there for (int i = 0; i < secondkeys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(secondkeys[i]); // look directly in the working segment, they should be MISSING // This is testing the checkpoint as well. If log resume didn't // CHECKPOINT_DROP, then the values will be duplicated in the working segment. { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should be MISSING"); } // assure the global query interface finds the NEW VALUES { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager should see NEW VALUES, where is: " + key); Assert.AreEqual(secondvalues[i], data.ReadDataAsString(), "LayerManager.getRecord() should see NEW VALUES"); } } } db.Dispose(); } }
public void T01_LayerTxnLogResume() { String[] keys = { "test-1", "test-2", "test-3" }; String[] values = {"a","b","c" }; { LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\4"); LayerWriteGroup txn = db.newWriteGroup(); for (int i=0;i<keys.Length;i++) { txn.setValueParsed(keys[i],values[i]); } txn.finish(); // TODO: assure the freespace hasn't been affected // assure we have not committed any segments Assert.AreEqual(1, db.segmentlayers.Count, "shold be one memory segment"); Assert.AreEqual(db.segmentlayers[0], db.workingSegment, "memory segment should be the working segment"); // assure the working segment contains the right data // 3 test records, the NUMGENERATIONS record, and FREELIST/HEAD // TODO: make a more robust way to do this test (i.e. count non .ROOT records) Assert.AreEqual(5, db.workingSegment.RowCount, "record count match 1"); db.Dispose(); } { LayerManager db = new LayerManager(InitMode.RESUME, "c:\\BENDtst\\4"); System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); // assure we still have not committed any segments Assert.AreEqual(1, db.segmentlayers.Count, "after resume: one memory segment"); Assert.AreEqual(db.segmentlayers[0], db.workingSegment, "after resume: working segment setup"); // assure the working segment contains the right data Assert.AreEqual(5, db.workingSegment.RowCount, "after resume: record count == 5"); for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(new RecordKeyType_String(keys[i])); // look directly in the working segment { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.PRESENT, status, "SegmentBuilder.getRecordUpdate({0})", key); Assert.AreEqual(values[i], enc.GetString(update.data), "SegmentBuilder.getRecordUpdate({0})",key); } // assure the global query interface finds it { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "LayerManager.getRecord({0})", key); Assert.AreEqual(values[i], enc.GetString(data.data), "LayerManager.getRecord({0})",key); } } // cleanup db.Dispose(); } }
public void T02_LayerSegmentFlushAndFreespaceModification() { String[] keys = { "test-1", "test-2", "test-3" }; String[] values = { "a", "b", "c" }; System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); LayerManager db = new LayerManager(InitMode.NEW_REGION, "c:\\BENDtst\\5"); db.debugDump(); LayerWriteGroup txn = db.newWriteGroup(); for (int i = 0; i < keys.Length; i++) { txn.setValueParsed(keys[i], values[i]); } txn.finish(); db.flushWorkingSegment(); db.debugDump(); // assure that we checkpointed down to a single working segment Assert.AreEqual(1, db.segmentlayers.Count, "segment layer count"); // assure we allocated a new generation and gen0 range record (walk .ROOT range map) // TODO: move this test to RangemapManager, to remove this cross-dependency { RecordData data; RecordKey key = new RecordKey().appendParsedKey(".ROOT/VARS/NUMGENERATIONS"); Assert.AreEqual(GetStatus.PRESENT,db.getRecord(key, out data),"missing numgenerations record"); Assert.AreEqual("1", data.ReadDataAsString(),"generation count 1"); RecordUpdate update; Assert.AreEqual(GetStatus.PRESENT, db.workingSegment.getRecordUpdate(key, out update), "missing workingsegment numgenerations record"); Assert.AreEqual("1", enc.GetString(update.data), "generation count 2"); } #if false if (false) { RecordData data; Assert.AreEqual( GetStatus.PRESENT, db.getRecord(new RecordKey() .appendParsedKey(".ROOT/GEN") .appendKeyPart(new RecordKeyType_Long(0)) .appendKeyPart("</>"), out data), ".ROOT/GEN/0/</> key is missing"); } #endif // TODO: assure we subtracted the new range record from the freespace // assure the records we wrote are NOT in the working segment, but ARE in the next layer for (int i = 0; i < keys.Length; i++) { RecordKey key = new RecordKey(); key.appendKeyPart(keys[i]); // look directly in the working segment, they should be **MISSING* { RecordUpdate update; GetStatus status = db.workingSegment.getRecordUpdate(key, out update); Assert.AreEqual(GetStatus.MISSING, status, "working segment should be clear"); } // assure the global query interface finds it { RecordData data; GetStatus status = db.getRecord(key, out data); Assert.AreEqual(GetStatus.PRESENT, status, "records should be found in layers, {0} missing", key); Assert.AreEqual(values[i], data.ReadDataAsString(), "LayerManager.getRecord()"); } } db.Dispose(); }
public void mapSegment(LayerWriteGroup tx, int use_gen, RecordKey start_key, RecordKey end_key, IRegion reader) { if (! (tx.type == LayerWriteGroup.WriteGroupType.DISK_ATOMIC_NOFLUSH || tx.type == LayerWriteGroup.WriteGroupType.DISK_ATOMIC_FLUSH)) { throw new Exception("NewUnusedSegment.mapSegment() must be provided an ATOMIC write group"); } // remove the pending entry RecordKey key = new RecordKey().appendParsedKey(".ROOT/FREELIST/PENDING"); key.appendKeyPart(new RecordKeyType_Long(reader.getStartAddress())); tx.setValue(key, RecordUpdate.DeletionTombstone()); // add the new map tx.mylayer.rangemapmgr.mapGenerationToRegion(tx, use_gen, start_key, end_key, reader); }
public void T10_RecordKeyEncodeDecodeBugTest() { // test encode/decode with byte[] parts // 92 43 0 byte[] chars = { 92, 43, 0 }; { System.Text.ASCIIEncoding enc = new System.Text.ASCIIEncoding(); String keystring = enc.GetString(chars); enc.GetBytes(keystring); Assert.AreEqual(chars, enc.GetBytes(keystring), "string encoding not reversible"); RecordKey key = new RecordKey(); key.appendKeyPart(chars); byte[] data = key.encode(); Assert.AreEqual(key, new RecordKey(data), "check encode/decode with binary data"); // check nested key var wrap_key = new RecordKey().appendKeyPart(key.encode()); byte[] wrap_encoded = wrap_key.encode(); RecordKey wrap_decoded = new RecordKey(wrap_encoded); } }
// move the pending address into the freelist private void handleRegionSafeToFree(long start_addr, FreespaceExtent extent, LayerWriteGroup wg) { System.Console.WriteLine("*\n*\n*\n* handleRegionSafeToFree {0} \n*\n*\n*", start_addr); // (1) remove pending entry wg.setValue(pendingKeyForAddr(start_addr), RecordUpdate.DeletionTombstone()); // (2) write real freelist entry (TODO: merge with neighboring entries) { RecordKey key = new RecordKey().appendParsedKey(".ROOT/FREELIST/EXTENTS"); key.appendKeyPart(new RecordKeyType_Long(extent.end_addr)); wg.setValue(key, RecordUpdate.WithPayload(extent.pack())); } wg.finish(); }
private RecordKey pendingKeyForAddr(long start_addr) { RecordKey key = new RecordKey().appendParsedKey(".ROOT/FREELIST/PENDING"); key.appendKeyPart(new RecordKeyType_Long(start_addr)); return key; }
public void setValue(RecordKey key, RecordUpdate update) { // RecordKey key = key.clone(); if (this.is_frozen) { throw new Exception("snapshot not writable! " + this.frozen_at_timestamp); } // (1) get our timestamp long timestamp = id_gen.nextTimestamp(); // (2) add our timestamp attribute to the end of the keyspace key.appendKeyPart(new RecordKeyType_AttributeTimestamp(timestamp)); next_stage.setValue(key, update); }
public RecordKey genLowestKey() { RecordKey key = new RecordKey(); foreach (QualifierBase part in key_part_qualifiers) { key.appendKeyPart(part.genLowestKeyTest().ToString()); } return key; }
public void T06_RecordKeyDelimiterEscape() { string DELIM = new String(RecordKey.PRINTED_DELIMITER, 1); RecordKey key1 = new RecordKey(); key1.appendKeyParts("1", "2", "3"); Assert.AreEqual(3, key1.numParts()); RecordKey dkey1 = new RecordKey(key1.encode()); Assert.AreEqual(3, dkey1.numParts(), "dkey1 delimiter decode"); RecordKey key2 = new RecordKey(); key2.appendKeyPart("1" + DELIM + "2" + DELIM + "3"); Assert.AreEqual(1, key2.numParts()); RecordKey dkey2 = new RecordKey(key2.encode()); Assert.AreEqual(1, dkey2.numParts(), "dkey2 delimiter decode"); // key2 > key1 Assert.AreEqual(1, key2.CompareTo(key1)); Assert.AreEqual(-1, key1.CompareTo(key2)); }
internal IEnumerable<KeyValuePair<RecordKey, RecordData>> fetchLogEntries( string log_server_guid, RecordKeyType log_start_key, int limit = -1, bool block = false) { var rk_start = new RecordKey() .appendKeyPart("_logs") .appendKeyPart(log_server_guid); if (!log_start_key.Equals("")) { rk_start.appendKeyPart(log_start_key); } var rk_end = new RecordKey() .appendKeyPart("_logs") .appendKeyPart(log_server_guid); var scanrange = new ScanRange<RecordKey>(rk_start, RecordKey.AfterPrefix(rk_end), null); Console.WriteLine(" fetchLogEntries (block:{3}) for ({0}): start {1} end {2}", log_server_guid, rk_start, rk_end, block); bool matched_first = false; int count = 0; retry_log_fetch: foreach (var logrow in next_stage.scanForward(scanrange)) { if (!matched_first) { // the first logrow needs to match the log_start_key, or there was a gap in the log!! var logstamp = logrow.Key.key_parts[2]; if (logstamp.CompareTo(log_start_key) != 0) { throw new LogException( String.Format("log start gap! guid:{0} log_start_key:{1} logstamp:{2}", log_server_guid,log_start_key,logstamp)); } matched_first = true; continue; } yield return logrow; count++; // if we're limiting the number of return rows... if (limit != -1) { if (count > limit) { yield break; } } } if (!matched_first) { throw new LogException("no log entries!"); } // if we only matched one log row, then it should be the matching first row. if ((count == 0) && block) { Console.WriteLine("++++++++ block on log tail"); lock (this.logWaiters) { Monitor.Wait(this.logWaiters); } Console.WriteLine("++++++++ wakeup from log tail"); goto retry_log_fetch; } }
public NewUnusedSegment allocateNewSegment(LayerWriteGroup tx, int length) { // use one big nasty lock to prevent race conditions lock (this) { // try to find an extent with enough space to carve off a chunk foreach (var rec in store.scanForward(new ScanRange<RecordKey>(freelist_prefix, RecordKey.AfterPrefix(freelist_prefix), null))) { FreespaceExtent extent = FreespaceExtent.unpack(rec.Value.data); if (extent.length() == length) { // the extent is exactly the right size... make it pending LayerWriteGroup makepending_wg = tx.mylayer.newWriteGroup(type: LayerWriteGroup.WriteGroupType.DISK_ATOMIC_NOFLUSH); // add a pending entry for this block { RecordKey key = new RecordKey().appendParsedKey(".ROOT/FREELIST/PENDING"); key.appendKeyPart(new RecordKeyType_Long(extent.start_addr)); makepending_wg.setValue(key, RecordUpdate.WithPayload(extent.pack())); } // remove the freelist entry { RecordKey key = new RecordKey().appendParsedKey(".ROOT/FREELIST/EXTENTS"); key.appendKeyPart(new RecordKeyType_Long(extent.end_addr)); makepending_wg.setValue(key, RecordUpdate.DeletionTombstone()); } makepending_wg.finish(); return new NewUnusedSegment(store, extent); } else if (extent.length() > length) { // TODO: carve a piece off the extent and return the pending piece } } // if we can't find a free segment, grow the heap return growHeap(tx, length); // TODO: then carve a segment out of the new grown heap } }
public void setValue(RecordKey key, RecordUpdate update) { if (this.is_frozen) { throw new Exception("snapshot not writable! " + this.frozen_at_snapshotnumber); } // add our snapshot_number to the end of the keyspace key.appendKeyPart(new RecordKeyType_AttributeTimestamp(this.current_snapshot)); // wrap the update into a sub-update, mostly because tombstones need to be "real" records // to us var sub_update = RecordUpdate.WithPayload(update.encode()); next_stage.setValue(key, sub_update); }
public void T02b_RecordKeyNesting() { String[] parts1 = { "aaaa", "bbbb" }; String[] parts2 = { "xxxx", "yyyy", "zzzz" }; RecordKey nestedkey = new RecordKey(); nestedkey.appendKeyParts(parts2); RecordKey parentkey = new RecordKey(); parentkey.appendKeyPart(parts1[0]); // "aaaa" parentkey.appendKeyPart(nestedkey.encode()); // (xxxx,yyyy,zzzz) parentkey.appendKeyPart(parts1[1]); // "bbbb" RecordKey decodedkey = new RecordKey(parentkey.encode()); Assert.AreEqual(decodedkey.ToString(), parentkey.ToString(), "tostring versions of keys don't match"); Assert.AreEqual(decodedkey.numParts(), parentkey.numParts(), "nested delimiters are changing number of keyparts"); Assert.AreEqual(decodedkey, parentkey, "nested key encode/decode error"); }