public void Add(Key key, Value value)
{
lock (_tableLock) {
_totalKeySize += key.Length;
_totalValueSize += value.Length;
// Set value in the hashtable
_internalTable[key] = value;
}
}
public void LevelMergeDuplicateValuesTest()
{
string path = Path.GetFullPath("TestData\\LevelMergeDuplicateValuesTest");
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
foreach (string file in Directory.GetFiles(path)) {
File.Delete(file);
}
int num_tables_to_merge = 4;
int items_per_table = 2500;
int totalData = 0;
for (int i = 0; i < num_tables_to_merge; i++) {
var mt = new MemTable();
for (int j = 0; j < items_per_table; j++) {
int numToStore = j % 100;
var key = new Key(new ByteArray(BitConverter.GetBytes(numToStore)));
var value = new Value(BitConverter.GetBytes(j));
mt.Add(key, value);
}
mt.WriteToSortedBlockTable("TestData\\LevelMergeDuplicateValuesTest", 0, i);
totalData += mt.Size;
}
var cache = new RazorCache();
var timer = new Stopwatch();
timer.Start();
Manifest mf = new Manifest("TestData\\LevelMergeDuplicateValuesTest");
SortedBlockTable.MergeTables(cache, mf, 1, new List<PageRef>{
new PageRef { Level = 0, Version = 0},
new PageRef { Level = 0, Version = 1},
new PageRef { Level = 0, Version = 2},
new PageRef { Level = 0, Version = 3}
}, ExceptionHandling.ThrowAll, null);
timer.Stop();
// Open the block table and scan it to check the stored values
var sbt = new SortedBlockTable(cache, mf.BaseFileName, 1, 1);
try {
var pairs = sbt.Enumerate().ToList();
Assert.AreEqual(100, pairs.Count());
Assert.AreEqual(2400, BitConverter.ToInt32(pairs.First().Value.ValueBytes, 0));
} finally {
sbt.Close();
}
Console.WriteLine("Wrote a multilevel merge with duplicates at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
}
private byte[] AssembleGetResult(Key lookupKey, Value result) {
switch (result.Type) {
case ValueFlag.Null:
case ValueFlag.Deleted:
return null;
case ValueFlag.SmallValue:
return result.ValueBytes;
case ValueFlag.LargeValueDescriptor: {
lock (multiPageLock) {
// read the descriptor again in case it changed
result = InternalGet(lookupKey);
// make sure type is still large value descriptor and continue
if (result.Type == ValueFlag.LargeValueDescriptor) {
int valueSize = BitConverter.ToInt32(result.ValueBytes, 0);
byte[] bytes = new byte[valueSize];
int offset = 0;
byte seqNum = 1;
while (offset < valueSize) {
var blockKey = lookupKey.WithSequence(seqNum);
var block = InternalGet(blockKey);
if (block.Type != ValueFlag.LargeValueChunk)
throw new InvalidDataException(string.Format("Corrupted data: block is missing. Block Type: {0} SeqNum: {1}, Block Key: {2}", block.Type, seqNum, blockKey));
offset += block.CopyValueBytesTo(bytes, offset);
seqNum++;
}
return bytes;
} else {
return AssembleGetResult(lookupKey, result);
}
}
}
default:
throw new InvalidOperationException("Unexpected value flag for result.");
}
}
private void InternalSet(Key k, Value v, IEnumerable<KeyValuePair<string, byte[]>> indexedValues) {
int adds = 10;
while (!_currentJournaledMemTable.Add(k, v)) {
adds--;
if (adds <= 0)
throw new InvalidOperationException("Failed too many times trying to add an item to the JournaledMemTable");
}
// Add secondary index values if they were provided
if (indexedValues != null)
AddToIndex(k.KeyBytes, indexedValues);
if (_currentJournaledMemTable.Full) {
RotateMemTable();
}
TableManager.Default.MarkKeyValueStoreAsModified(this);
}
public void Set(byte[] key, byte[] value, IEnumerable<KeyValuePair<string, byte[]>> indexedValues) {
int valueSize = value.Length;
if (valueSize <= Config.MaxSmallValueSize) {
var k = new Key(key, 0);
var v = new Value(value, ValueFlag.SmallValue);
InternalSet(k, v, indexedValues);
} else {
lock (multiPageLock) {
if (value.Length >= Config.MaxLargeValueSize)
throw new InvalidDataException(string.Format("Value is larger than the maximum size. ({0} bytes)", Config.MaxLargeValueSize));
int offset = 0;
byte seqNum = 1;
while (offset < valueSize) {
var k = new Key(key, seqNum);
int length = Math.Min(valueSize - offset, Config.MaxSmallValueSize);
var v = new Value(ByteArray.From(value, offset, length).InternalBytes, ValueFlag.LargeValueChunk);
InternalSet(k, v, null);
offset += length;
seqNum++;
}
var dk = new Key(key, 0);
var dv = new Value(BitConverter.GetBytes(valueSize), ValueFlag.LargeValueDescriptor);
InternalSet(dk, dv, indexedValues);
}
}
}
public bool Lookup(Key key, out Value value)
{
lock (_tableLock) {
return _internalTable.Find(ref key, out value);
}
}
// Add an item to the journal. It's possible that a thread is still Adding while another thread is Closing the journal.
// in that case, we return false and expect the caller to do the operation over again on another journal instance.
public bool Add(Key key, Value value)
{
lock (_writeLock) {
if (_writer == null)
return false;
else {
_writer.Write7BitEncodedInt(key.Length);
_writer.Write(key.InternalBytes);
_writer.Write7BitEncodedInt(value.Length);
_writer.Write(value.InternalBytes);
_writer.Flush();
return true;
}
}
}
public bool Add(Key key, Value value)
{
if (_journal == null || _memTable == null)
return false;
if (_journal.Add(key, value)) {
_memTable.Add(key, value);
return true;
} else {
return false;
}
}
public bool Lookup(Key key, out Value value)
{
return _memTable.Lookup(key, out value);
}
public static Value From(byte[] bytes, int offset, int length)
{
if (length <= 0)
throw new ArgumentOutOfRangeException("Length of the Value must be at least 1 byte.");
var v = new Value();
v._bytes = ByteArray.From(bytes, offset, length);
return v;
}