// Add the specified key and value.
public void Add(Key key, Value value)
{
lock (_tableLock) {
_totalKeySize += key.Length;
_totalValueSize += value.Length;
_internalTable [key] = value; // Set value in the hashtable
}
}
public void LevelMergeDuplicateValuesTest()
{
string path = Path.GetFullPath ("TestData\\LevelMergeDuplicateValuesTest");
if (!Directory.Exists (path)) Directory.CreateDirectory (path);
foreach (string file in Directory.GetFiles(path)) File.Delete (file);
int num_tables_to_merge = 4;
int items_per_table = 2500;
int totalData = 0;
for (int i = 0; i < num_tables_to_merge; i++) {
var mt = new MemTable ();
for (int j = 0; j < items_per_table; j++) {
int numToStore = j % 100;
var key = new Key (new ByteArray (BitConverter.GetBytes (numToStore)));
var value = new Value (BitConverter.GetBytes (j));
mt.Add (key, value);
}
mt.WriteToSortedBlockTable ("TestData\\LevelMergeDuplicateValuesTest", 0, i);
totalData += mt.Size;
}
var cache = new RazorCache ();
var timer = new Stopwatch ();
timer.Start ();
Manifest mf = new Manifest ("TestData\\LevelMergeDuplicateValuesTest");
SortedBlockTable.MergeTables (cache, mf, 1, new List<PageRef> {
new PageRef { Level = 0, Version = 0},
new PageRef { Level = 0, Version = 1},
new PageRef { Level = 0, Version = 2},
new PageRef { Level = 0, Version = 3}
}, ExceptionHandling.ThrowAll, null);
timer.Stop ();
// Open the block table and scan it to check the stored values
var sbt = new SortedBlockTable (cache, mf.BaseFileName, 1, 1);
try {
var pairs = sbt.Enumerate ().ToList ();
Assert.AreEqual (100, pairs.Count ());
Assert.AreEqual (2400, BitConverter.ToInt32 (pairs.First ().Value.ValueBytes, 0));
} finally {
sbt.Close ();
}
Console.WriteLine ("Wrote a multilevel merge with duplicates at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
}
public static Value From(byte[] bytes, int offset, int length)
{
if (length <= 0)
throw new ArgumentOutOfRangeException ("Length of the Value must be at least 1 byte.");
var v = new Value ();
v._bytes = ByteArray.From (bytes, offset, length);
return v;
}
public bool Add(Key key, Value value)
{
if (_journal == null || _memTable == null)
return false;
if (_journal.Add (key, value)) {
_memTable.Add (key, value);
return true;
} else
return false;
}
public bool Lookup(Key key, out Value value)
{
return _memTable.Lookup (key, out value);
}
// Add an item to the journal. It's possible that a thread is still Adding while another thread is Closing the journal.
// in that case, we return false and expect the caller to do the operation over again on another journal instance.
public bool Add(Key key, Value value)
{
lock (_writeLock) {
if (_writer == null)
return false;
else {
_writer.Write7BitEncodedInt (key.Length);
_writer.Write (key.InternalBytes);
_writer.Write7BitEncodedInt (value.Length);
_writer.Write (value.InternalBytes);
_writer.Flush ();
return true;
}
}
}
// Lookup the specified key and value.
public bool Lookup(Key key, out Value value)
{
lock (_tableLock)
return _internalTable.Find (ref key, out value);
}
static bool SearchBlockForKey(byte[] block, Key key, out Value value)
{
int offset = BitConverter.ToUInt16 (block, 0); // grab the tree root
value = Value.Empty;
while (offset >= 2 && offset < Config.SortedBlockSize && block[offset] == (byte)RecordHeaderFlag.Record) {
int startingOffset = offset;
offset += 1; // skip header
offset += 4; // skip tree pointers
int keySize = Helper.Decode7BitInt (block, ref offset);
int cmp = key.CompareTo (block, offset, keySize);
if (cmp == 0) {
// Found it
var pair = ReadPair (block, ref startingOffset);
value = pair.Value;
return true;
} else if (cmp < 0)
offset = BitConverter.ToUInt16 (block, startingOffset + 1); // key < node => explore left side
else if (cmp > 0)
offset = BitConverter.ToUInt16 (block, startingOffset + 3); // key > node => explore right side
}
return false;
}
// Write a new key value pair to the output file. This method assumes that the data is being fed in key-sorted order.
public void WritePair(Key key, Value value)
{
byte[] keySize = new byte[8];
int keySizeLen = Helper.Encode7BitInt (keySize, key.Length);
byte[] valueSize = new byte[8];
int valueSizeLen = Helper.Encode7BitInt (valueSize, value.Length);
int bytesNeeded = keySizeLen + key.Length + valueSizeLen + value.Length + 4 + 1;
if ((_bufferPos + bytesNeeded) > Config.SortedBlockSize)
WriteDataBlock (); // Do we need to write out a block before adding this key value pair?
if (_bufferPos == 0) { // If we are at the beginning of the buffer, then add this key to the index.
_pageIndex.Add (key);
_bufferPos += 2; // Add a place for the root node offset.
}
_keyOffsets.Add ((ushort)_bufferPos); // Store the pair in preparation for writing.
_buffer [_bufferPos++] = (byte)RecordHeaderFlag.Record; // A record header.
_bufferPos += 4; // Adds space for left and right node pointers.
// Writes the data out to the buffer.
Array.Copy (keySize, 0, _buffer, _bufferPos, keySizeLen);
_bufferPos += keySizeLen;
Array.Copy (key.InternalBytes, 0, _buffer, _bufferPos, key.Length);
_bufferPos += key.Length;
Array.Copy (valueSize, 0, _buffer, _bufferPos, valueSizeLen);
_bufferPos += valueSizeLen;
Array.Copy (value.InternalBytes, 0, _buffer, _bufferPos, value.Length);
_bufferPos += value.Length;
WrittenSize += bytesNeeded;
}
static bool ScanBlockForKey(byte[] block, Key key, out Value value)
{
int offset = 2; // skip over the tree root pointer
value = Value.Empty;
while (offset >= 2 && offset < Config.SortedBlockSize && block[offset] == (byte)RecordHeaderFlag.Record) {
int startingOffset = offset;
offset++; // skip past the header flag
offset += 4; // skip past the tree pointers
int keySize = Helper.Decode7BitInt (block, ref offset);
int cmp = key.CompareTo (block, offset, keySize);
if (cmp == 0) {
// Found it
var pair = ReadPair (block, ref startingOffset);
value = pair.Value;
return true;
} else if (cmp < 0)
return false;
offset += keySize;
// Skip past the value
int valueSize = Helper.Decode7BitInt (block, ref offset);
offset += valueSize;
}
return false;
}
public static bool Lookup(string baseFileName, int level, int version, RazorCache cache, Key key, out Value value, ExceptionHandling exceptionHandling, Action<string> logger)
{
SortedBlockTable sbt = new SortedBlockTable (cache, baseFileName, level, version);
try {
int dataBlockNum = FindBlockForKey (baseFileName, level, version, cache, key);
if (dataBlockNum >= 0 && dataBlockNum < sbt._dataBlocks) {
byte[] block = sbt.ReadBlock (LocalThreadAllocatedBlock (), dataBlockNum);
return SearchBlockForKey (block, key, out value);
}
} finally {
sbt.Close ();
}
value = Value.Empty;
return false;
}
void InternalSet(Key k, Value v, IDictionary<string, byte[]> indexedValues)
{
int adds = 10;
while (!_currentJournaledMemTable.Add(k, v)) {
adds--;
if (adds <= 0)
throw new InvalidOperationException ("Failed too many times trying to add an item to the JournaledMemTable");
}
// Add secondary index values if they were provided
if (indexedValues != null)
AddToIndex (k.KeyBytes, indexedValues);
if (_currentJournaledMemTable.Full)
RotateMemTable ();
TableManager.Default.MarkKeyValueStoreAsModified (this);
}
byte[] AssembleGetResult(Key lookupKey, Value result)
{
switch (result.Type) {
case ValueFlag.Null:
case ValueFlag.Deleted:
return null;
case ValueFlag.SmallValue:
return result.ValueBytes;
case ValueFlag.LargeValueDescriptor:
{
lock (multiPageLock) {
result = InternalGet (lookupKey); // read the descriptor again in case it changed
if (result.Type == ValueFlag.LargeValueDescriptor) { // make sure type is still large value descriptor and continue
int valueSize = BitConverter.ToInt32 (result.ValueBytes, 0);
byte[] bytes = new byte[valueSize];
int offset = 0;
byte seqNum = 1;
while (offset < valueSize) {
var blockKey = lookupKey.WithSequence (seqNum);
var block = InternalGet (blockKey);
if (block.Type != ValueFlag.LargeValueChunk)
throw new InvalidDataException (string.Format ("Corrupted data: block is missing. Block Type: {0} SeqNum: {1}, Block Key: {2}", block.Type, seqNum, blockKey));
offset += block.CopyValueBytesTo (bytes, offset);
seqNum++;
}
return bytes;
} else
return AssembleGetResult (lookupKey, result);
}
}
default:
throw new InvalidOperationException ("Unexpected value flag for result.");
}
}
// Set the specified key, value and indexedValues.
// The multi page lock.
public void Set(byte[] key, byte[] value, IDictionary<string, byte[]> indexedValues)
{
int valueSize = value.Length;
if (valueSize <= Config.MaxSmallValueSize) {
var k = new Key (key, 0);
var v = new Value (value, ValueFlag.SmallValue);
InternalSet (k, v, indexedValues);
} else {
lock (multiPageLock) {
if (value.Length >= Config.MaxLargeValueSize)
throw new InvalidDataException (string.Format ("Value is larger than the maximum size. ({0} bytes)", Config.MaxLargeValueSize));
int offset = 0;
byte seqNum = 1;
while (offset < valueSize) {
var k = new Key (key, seqNum);
int length = Math.Min (valueSize - offset, Config.MaxSmallValueSize);
var v = new Value (ByteArray.From (value, offset, length).InternalBytes, ValueFlag.LargeValueChunk);
InternalSet (k, v, null);
offset += length;
seqNum++;
}
var dk = new Key (key, 0);
var dv = new Value (BitConverter.GetBytes (valueSize), ValueFlag.LargeValueDescriptor);
InternalSet (dk, dv, indexedValues);
}
}
}