Example #1
0
        public void Add(Key key, Value value)
        {
            lock (_tableLock) {
                _totalKeySize += key.Length;
                _totalValueSize += value.Length;

                // Set value in the hashtable
                _internalTable[key] = value;
            }
        }
Example #2
0
        public void LevelMergeDuplicateValuesTest()
        {
            string path = Path.GetFullPath("TestData\\LevelMergeDuplicateValuesTest");
            if (!Directory.Exists(path))
                Directory.CreateDirectory(path);
            foreach (string file in Directory.GetFiles(path)) {
                File.Delete(file);
            }

            int num_tables_to_merge = 4;
            int items_per_table = 2500;
            int totalData = 0;
            for (int i = 0; i < num_tables_to_merge; i++) {
                var mt = new MemTable();
                for (int j = 0; j < items_per_table; j++) {
                    int numToStore = j % 100;
                    var key = new Key(new ByteArray(BitConverter.GetBytes(numToStore)));
                    var value = new Value(BitConverter.GetBytes(j));
                    mt.Add(key, value);
                }
                mt.WriteToSortedBlockTable("TestData\\LevelMergeDuplicateValuesTest", 0, i);
                totalData += mt.Size;
            }

            var cache = new RazorCache();
            var timer = new Stopwatch();
            timer.Start();

            Manifest mf = new Manifest("TestData\\LevelMergeDuplicateValuesTest");
            SortedBlockTable.MergeTables(cache, mf, 1, new List<PageRef>{
                                                                            new PageRef { Level = 0, Version = 0},
                                                                            new PageRef { Level = 0, Version = 1},
                                                                            new PageRef { Level = 0, Version = 2},
                                                                            new PageRef { Level = 0, Version = 3}
                                                                        }, ExceptionHandling.ThrowAll, null);
            timer.Stop();

            // Open the block table and scan it to check the stored values
            var sbt = new SortedBlockTable(cache, mf.BaseFileName, 1, 1);
            try {
                var pairs = sbt.Enumerate().ToList();
                Assert.AreEqual(100, pairs.Count());
                Assert.AreEqual(2400, BitConverter.ToInt32(pairs.First().Value.ValueBytes, 0));
            } finally {
                sbt.Close();
            }

            Console.WriteLine("Wrote a multilevel merge with duplicates at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
        }
Example #3
0
        private byte[] AssembleGetResult(Key lookupKey, Value result) {
            switch (result.Type) {
                case ValueFlag.Null:
                case ValueFlag.Deleted:
                    return null;
                case ValueFlag.SmallValue:
                    return result.ValueBytes;
                case ValueFlag.LargeValueDescriptor: {
                        lock (multiPageLock) {
                            // read the descriptor again in case it changed
                            result = InternalGet(lookupKey);

                            // make sure type is still large value descriptor and continue
                            if (result.Type == ValueFlag.LargeValueDescriptor) {
                                int valueSize = BitConverter.ToInt32(result.ValueBytes, 0);
                                byte[] bytes = new byte[valueSize];
                                int offset = 0;
                                byte seqNum = 1;
                                while (offset < valueSize) {
                                    var blockKey = lookupKey.WithSequence(seqNum);
                                    var block = InternalGet(blockKey);
                                    if (block.Type != ValueFlag.LargeValueChunk)
                                        throw new InvalidDataException(string.Format("Corrupted data: block is missing. Block Type: {0} SeqNum: {1}, Block Key: {2}", block.Type, seqNum, blockKey));
                                    offset += block.CopyValueBytesTo(bytes, offset);
                                    seqNum++;
                                }
                                return bytes;
                            } else {
                                return AssembleGetResult(lookupKey, result);
                            }
                        }
                    }
                default:
                    throw new InvalidOperationException("Unexpected value flag for result.");
            }
        }
Example #4
0
        private void InternalSet(Key k, Value v, IEnumerable<KeyValuePair<string, byte[]>> indexedValues) {
            int adds = 10;
            while (!_currentJournaledMemTable.Add(k, v)) {
                adds--;
                if (adds <= 0)
                    throw new InvalidOperationException("Failed too many times trying to add an item to the JournaledMemTable");
            }
            // Add secondary index values if they were provided
            if (indexedValues != null)
                AddToIndex(k.KeyBytes, indexedValues);

            if (_currentJournaledMemTable.Full) {
                RotateMemTable();
            }

            TableManager.Default.MarkKeyValueStoreAsModified(this);
        }
Example #5
0
        public void Set(byte[] key, byte[] value, IEnumerable<KeyValuePair<string, byte[]>> indexedValues) {

            int valueSize = value.Length;
            if (valueSize <= Config.MaxSmallValueSize) {
                var k = new Key(key, 0);
                var v = new Value(value, ValueFlag.SmallValue);
                InternalSet(k, v, indexedValues);
            } else {
                lock (multiPageLock) {
                    if (value.Length >= Config.MaxLargeValueSize)
                        throw new InvalidDataException(string.Format("Value is larger than the maximum size. ({0} bytes)", Config.MaxLargeValueSize));

                    int offset = 0;
                    byte seqNum = 1;
                    while (offset < valueSize) {
                        var k = new Key(key, seqNum);
                        int length = Math.Min(valueSize - offset, Config.MaxSmallValueSize);
                        var v = new Value(ByteArray.From(value, offset, length).InternalBytes, ValueFlag.LargeValueChunk);
                        InternalSet(k, v, null);
                        offset += length;
                        seqNum++;
                    }
                    var dk = new Key(key, 0);
                    var dv = new Value(BitConverter.GetBytes(valueSize), ValueFlag.LargeValueDescriptor);
                    InternalSet(dk, dv, indexedValues);
                }
            }
        }
Example #6
0
 public bool Lookup(Key key, out Value value)
 {
     lock (_tableLock) {
         return _internalTable.Find(ref key, out value);
     }
 }
Example #7
0
 // Add an item to the journal. It's possible that a thread is still Adding while another thread is Closing the journal.
 // in that case, we return false and expect the caller to do the operation over again on another journal instance.
 public bool Add(Key key, Value value)
 {
     lock (_writeLock) {
         if (_writer == null)
             return false;
         else {
             _writer.Write7BitEncodedInt(key.Length);
             _writer.Write(key.InternalBytes);
             _writer.Write7BitEncodedInt(value.Length);
             _writer.Write(value.InternalBytes);
             _writer.Flush();
             return true;
         }
     }
 }
Example #8
0
        public bool Add(Key key, Value value)
        {
            if (_journal == null || _memTable == null)
                return false;

            if (_journal.Add(key, value)) {
                _memTable.Add(key, value);
                return true;
            } else {
                return false;
            }
        }
Example #9
0
 public bool Lookup(Key key, out Value value)
 {
     return _memTable.Lookup(key, out value);
 }
Example #10
0
        public static Value From(byte[] bytes, int offset, int length)
        {
            if (length <= 0)
                throw new ArgumentOutOfRangeException("Length of the Value must be at least 1 byte.");

            var v = new Value();
            v._bytes = ByteArray.From(bytes, offset, length);
            return v;
        }