BlockAccessor datastream; // keep in mind, we don't want to share this, since it has only one seek pointer
#endregion Fields
#region Constructors
// prepare to read from a stream
public SegmentBlockBasicDecoder(BlockAccessor datastream)
{
this.datastream = datastream;
if (this.datastream.Length == 0) {
throw new Exception("SegmentBlockBasicDecoder: handed empty stream");
}
}
public void handleCommand(LogCommands cmd, byte[] cmddata)
{
if (cmd == LogCommands.UPDATE) {
// decode basic block key/value writes
BlockAccessor ba = new BlockAccessor(cmddata);
ISegmentBlockDecoder decoder = new SegmentBlockBasicDecoder(ba);
foreach (KeyValuePair<RecordKey, RecordUpdate> kvp in decoder.sortedWalk()) {
// add the updates to our working segment...
lock (mylayer.segmentlayers) {
#if false
// some status debug code...
if (RangemapManager.RangeKey.isRangeKey(kvp.Key)) {
System.Console.WriteLine("LayerManager.handleCommand : setValue() {0} => {1}",
kvp.Key.ToString(), kvp.Value.ToString());
if (kvp.Value.type != RecordUpdateTypes.DELETION_TOMBSTONE && kvp.Value.data.Length != 16) {
throw new Exception("!!! corrupted rangekey appeared in handleCommand");
}
}
#endif
mylayer.workingSegment.setRecord(kvp.Key, kvp.Value);
}
}
} else if (cmd == LogCommands.CHECKPOINT_START) {
// here we move aside the checkpoint segment...
// - if this is during live operation, the checkpoint is running as soon as we return.
// - if this is during recovery, there is no checkpoint running and we'll need to
// TODO: we need some kind of key/checksum to be sure that we CHECKPOINT and DROP the right dataF
checkpointSegment = mylayer.workingSegment;
SegmentMemoryBuilder newsegment = new SegmentMemoryBuilder();
lock (mylayer.segmentlayers) {
mylayer.workingSegment = newsegment;
mylayer.segmentlayers.Insert(0, mylayer.workingSegment);
}
} else if (cmd == LogCommands.CHECKPOINT_DROP) {
// TODO: we need some kind of key/checksum to be sure that we CHECKPOINT and DROP the right data
if (checkpointSegment != null) {
lock (mylayer.segmentlayers) {
mylayer.segmentlayers.Remove(checkpointSegment);
checkpointSegment = null;
}
} else {
throw new Exception("can't drop, no segment to drop");
}
} else {
throw new Exception("unimplemented command");
}
}
// prepare to read from a stream
public SegmentBlockDecoderRecordOffsetList(BlockAccessor datastream)
{
this.datastream = datastream;
if (this.datastream.Length == 0) {
throw new Exception("SegmentBlockDecoderRecordOffsetList: handed empty stream");
}
// read the footer index size
// FIXME: BUG BUG BUG!! using SeekOrigin.End is only valid here because our current RegionManager
// is handing us a file. We need to decide if future Regionmanagers are going to explicitly
// make "subregion" Streams, or whether we need to handle this differently.
datastream.Seek(-4, SeekOrigin.End); // last 4 bytes of block
BinaryReader r = new BinaryReader(datastream);
number_of_records = r.ReadInt32();
Console.WriteLine("numrecords: " + number_of_records);
}
public static BlockAccessor decode(BlockAccessor block)
{
byte[] data = new byte[block.Length];
if (block.Read(data,0,(int)block.Length) != block.Length) {
throw new Exception("BlockAccessor partial read");
}
MemoryStream ms = new MemoryStream(data);
BinaryReader reader = new BinaryReader(ms);
UInt32 uncompressed_length = reader.ReadUInt32();
byte[] uncompressed_data = new byte[uncompressed_length];
GZipInputStream uncompressed_stream = new GZipInputStream(ms);
if (uncompressed_stream.Read(uncompressed_data, 0, (int)uncompressed_length)
!= uncompressed_length) {
throw new Exception("GZipInputStream partial read");
}
return new BlockAccessor(uncompressed_data);
}
public void T01_RecordOffsetList_sortedWalk()
{
string[] testvalues = { "test/1", "test/2", "test/3" };
byte[] databuffer;
// encode a buffer
{
MemoryStream ms = new MemoryStream();
// add some values to the block encoder
SegmentBlockEncoderRecordOffsetList enc = new SegmentBlockEncoderRecordOffsetList();
enc.setStream(ms);
for (int i = 0; i < testvalues.Length; i++) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[i]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[i]);
enc.add(tkey, tupdate);
}
enc.flush();
databuffer = ms.ToArray();
}
Console.WriteLine("databuffer len : " + databuffer.Length);
Console.WriteLine("Hex: " + Lsd.ToHexString(databuffer));
// test sortedWalk
{
BlockAccessor rs = new BlockAccessor(databuffer);
var decoder = new SegmentBlockDecoderRecordOffsetList(rs);
int count = 0;
foreach (var row in decoder.sortedWalk()) {
Console.WriteLine(row);
count++;
}
Assert.AreEqual(testvalues.Length, count, "wrong number of elements in sorted walk");
}
}
internal void applyLogEntry(string from_server_guid, long logstamp, RecordUpdate logdata)
{
// (0) unpack the data
BlockAccessor ba = new BlockAccessor(logdata.data);
ISegmentBlockDecoder decoder = new SegmentBlockBasicDecoder(ba);
// (1) add it to our copy of that server's log
this._recordLogEntry(from_server_guid, logstamp, logdata);
// (2) add it to the database
foreach (var kvp in decoder.sortedWalk()) {
RecordKey local_data_key = new RecordKey()
.appendKeyPart("_data");
foreach (var part in kvp.Key.key_parts) {
local_data_key.appendKeyPart(part);
}
next_stage.setValue(local_data_key, kvp.Value);
}
}
public void T01_BasicBlock_Find()
{
string[] testvalues = { "test/1", "test/2", "test/3" };
byte[] databuffer;
// encode a buffer
{
MemoryStream ms = new MemoryStream();
// add some values to the block encoder
SegmentBlockBasicEncoder enc = new SegmentBlockBasicEncoder();
enc.setStream(ms);
for (int i = 0; i < testvalues.Length; i++) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[i]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[i]);
enc.add(tkey, tupdate);
}
enc.flush();
databuffer = ms.ToArray();
}
// test FindNext(key,equal_ok=true)
{
BlockAccessor rs = new BlockAccessor(databuffer);
SegmentBlockBasicDecoder decoder = new SegmentBlockBasicDecoder(rs);
for (int i = testvalues.Length -1; i >= 0; i--) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[i]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[i]);
KeyValuePair<RecordKey,RecordUpdate> row = decoder.FindNext(tkey, true);
Assert.AreEqual(tkey, row.Key, "record keys should match");
Assert.AreEqual(tupdate, row.Value, "record values should match:");
}
}
// test FindNext(key,equal_ok=false)
{
BlockAccessor rs = new BlockAccessor(databuffer);
SegmentBlockBasicDecoder decoder = new SegmentBlockBasicDecoder(rs);
for (int i = testvalues.Length - 2; i >= 0; i--) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[i]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[i]);
RecordKey fkey = new RecordKey().appendParsedKey(testvalues[i + 1]);
RecordUpdate fupdate = RecordUpdate.WithPayload("data: " + testvalues[i+1]);
KeyValuePair<RecordKey, RecordUpdate> row = decoder.FindNext(tkey, false);
Assert.AreEqual(fkey, row.Key, "findnext(,false) should find next key");
Assert.AreEqual(fupdate, row.Value, "findnext(,false) should finx next key (value was botched)");
}
// test for "next" after end of buffer, and we should get an exception
// test for "next" at beginning and we should get first
// test some random values in the middle
}
}
public void T02_BasicBlock_Enumerators()
{
// exercize the iscannable interface
string[] testvalues = { "test/1", "test/2", "test/3" };
byte[] databuffer;
// encode a buffer
{
MemoryStream ms = new MemoryStream();
// add some values to the block encoder
SegmentBlockBasicEncoder enc = new SegmentBlockBasicEncoder();
enc.setStream(ms);
for (int i = 0; i < testvalues.Length; i++) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[i]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[i]);
enc.add(tkey, tupdate);
}
enc.flush();
databuffer = ms.ToArray();
}
// decode and test the buffer, scan enumerators
{
BlockAccessor rs = new BlockAccessor(databuffer);
SegmentBlockBasicDecoder decoder = new SegmentBlockBasicDecoder(rs);
int pos = 0;
IScanner<RecordKey> scanner = ScanRange<RecordKey>.All();
foreach (KeyValuePair<RecordKey, RecordUpdate> row in decoder.scanForward(scanner)) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[pos]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[pos]);
Assert.AreEqual(tkey, row.Key, "forward, record keys should match");
Assert.AreEqual(tupdate, row.Value, "forward, record values should match:");
pos++;
}
Assert.AreEqual(testvalues.Length, pos, "forward, should return all values we put in");
pos = testvalues.Length - 1;
foreach (KeyValuePair<RecordKey, RecordUpdate> row in decoder.scanBackward(scanner)) {
RecordKey tkey = new RecordKey().appendParsedKey(testvalues[pos]);
RecordUpdate tupdate = RecordUpdate.WithPayload("data: " + testvalues[pos]);
Assert.AreEqual(tkey, row.Key, "backward, record keys should match, pos=" + pos);
Assert.AreEqual(tupdate, row.Value, "backward, record values should match:");
pos--;
}
Assert.AreEqual(-1, pos, "backward, should return all values we put in");
}
}
private static KeyValuePair<RecordKey, RecordUpdate> _decodeRecordFromBlock(BlockAccessor rs)
{
// ..we are ASSUMING that the records starts right here in the stream, it better be true!
// ..TODO: considering adding two separate record start/end markers
// Accumulate the key.
List<byte> keydata = new List<byte>();
bool keydone = false;
while (rs.Position < rs.Length && !keydone) {
byte c = (byte)rs.ReadByte();
switch (c) {
case SegmentBlockBasicEncoder.END_OF_LINE: // end of line
throw new Exception("reached end of line before keyvalue delimiter");
case SegmentBlockBasicEncoder.KEY_VAL_SEP: // key value delimiter
keydone = true;
break;
case SegmentBlockBasicEncoder.ESCAPE_CHAR:
byte nc = (byte)rs.ReadByte();
byte unescaped = (byte)(nc + (byte)SegmentBlockBasicEncoder.ESCAPE_OFFSET);
if (unescaped < SegmentBlockBasicEncoder.FIRST_SPECIAL || unescaped > SegmentBlockBasicEncoder.LAST_SPECIAL) {
throw new Exception("unhandled escape payload 1: " + ((int)nc).ToString());
}
keydata.Add(unescaped);
break;
default:
keydata.Add(c);
break;
}
}
if (!keydone) { throw new Exception("reached end of buffer before keydone!"); }
// accumulate the value
List<byte> valuedata = new List<byte>();
bool valuedone = false;
while (rs.Position < rs.Length && !valuedone) {
byte c = (byte)rs.ReadByte();
switch (c) {
case SegmentBlockBasicEncoder.END_OF_LINE: // end of line
valuedone = true;
break;
case SegmentBlockBasicEncoder.KEY_VAL_SEP: // key value delimiter
throw new Exception("found keyvalue delimiter in value");
case SegmentBlockBasicEncoder.ESCAPE_CHAR:
byte nc = (byte)rs.ReadByte();
byte unescaped = (byte)(nc + SegmentBlockBasicEncoder.ESCAPE_OFFSET);
if (unescaped < SegmentBlockBasicEncoder.FIRST_SPECIAL || unescaped > SegmentBlockBasicEncoder.LAST_SPECIAL) {
throw new Exception("unhandled escape sequence 2: " + unescaped.ToString());
}
valuedata.Add(unescaped);
break;
default:
valuedata.Add(c);
break;
}
}
if (valuedone != true) {
throw new Exception("SegmentBlockBasicDecoder: finished record without reaching END_OF_LINE");
}
RecordKey key = new RecordKey(keydata.ToArray());
RecordUpdate value = RecordUpdate.FromEncodedData(valuedata.ToArray());
// Debug.WriteLine("scanning " + key.ToString() + " : " + value.ToString());
return new KeyValuePair<RecordKey, RecordUpdate>(key, value);
}
public ISegmentBlockDecoder makeDecoder(BlockAccessor block)
{
return new SegmentBlockBasicDecoder(
SegmentBlockCompressedDecodeStage.decode(block));
}
