Beispiel #1
0
 // Add the specified key and value.
 public void Add(Key key, Value value)
 {
     lock (_tableLock) {
         _totalKeySize += key.Length;
         _totalValueSize += value.Length;
         _internalTable [key] = value; // Set value in the hashtable
     }
 }
Beispiel #2
0
        public void LevelMergeDuplicateValuesTest()
        {
            string path = Path.GetFullPath ("TestData\\LevelMergeDuplicateValuesTest");
            if (!Directory.Exists (path)) Directory.CreateDirectory (path);
            foreach (string file in Directory.GetFiles(path)) File.Delete (file);

            int num_tables_to_merge = 4;
            int items_per_table = 2500;
            int totalData = 0;
            for (int i = 0; i < num_tables_to_merge; i++) {
                var mt = new MemTable ();
                for (int j = 0; j < items_per_table; j++) {
                    int numToStore = j % 100;
                    var key = new Key (new ByteArray (BitConverter.GetBytes (numToStore)));
                    var value = new Value (BitConverter.GetBytes (j));
                    mt.Add (key, value);
                }
                mt.WriteToSortedBlockTable ("TestData\\LevelMergeDuplicateValuesTest", 0, i);
                totalData += mt.Size;
            }

            var cache = new RazorCache ();
            var timer = new Stopwatch ();
            timer.Start ();

            Manifest mf = new Manifest ("TestData\\LevelMergeDuplicateValuesTest");
            SortedBlockTable.MergeTables (cache, mf, 1, new List<PageRef> {
                new PageRef { Level = 0, Version = 0},
                new PageRef { Level = 0, Version = 1},
                new PageRef { Level = 0, Version = 2},
                new PageRef { Level = 0, Version = 3}
            }, ExceptionHandling.ThrowAll, null);
            timer.Stop ();

            // Open the block table and scan it to check the stored values
            var sbt = new SortedBlockTable (cache, mf.BaseFileName, 1, 1);
            try {
                var pairs = sbt.Enumerate ().ToList ();
                Assert.AreEqual (100, pairs.Count ());
                Assert.AreEqual (2400, BitConverter.ToInt32 (pairs.First ().Value.ValueBytes, 0));
            } finally {
                sbt.Close ();
            }

            Console.WriteLine ("Wrote a multilevel merge with duplicates at a throughput of {0} MB/s", (double)totalData / timer.Elapsed.TotalSeconds / (1024.0 * 1024.0));
        }
Beispiel #3
0
        public static Value From(byte[] bytes, int offset, int length)
        {
            if (length <= 0)
                throw new ArgumentOutOfRangeException ("Length of the Value must be at least 1 byte.");

            var v = new Value ();
            v._bytes = ByteArray.From (bytes, offset, length);
            return v;
        }
Beispiel #4
0
        public bool Add(Key key, Value value)
        {
            if (_journal == null || _memTable == null)
                return false;

            if (_journal.Add (key, value)) {
                _memTable.Add (key, value);
                return true;
            } else
                return false;
        }
Beispiel #5
0
 public bool Lookup(Key key, out Value value)
 {
     return _memTable.Lookup (key, out value);
 }
Beispiel #6
0
 // Add an item to the journal. It's possible that a thread is still Adding while another thread is Closing the journal.
 // in that case, we return false and expect the caller to do the operation over again on another journal instance.
 public bool Add(Key key, Value value)
 {
     lock (_writeLock) {
         if (_writer == null)
             return false;
         else {
             _writer.Write7BitEncodedInt (key.Length);
             _writer.Write (key.InternalBytes);
             _writer.Write7BitEncodedInt (value.Length);
             _writer.Write (value.InternalBytes);
             _writer.Flush ();
             return true;
         }
     }
 }
Beispiel #7
0
 // Lookup the specified key and value.
 public bool Lookup(Key key, out Value value)
 {
     lock (_tableLock)
         return _internalTable.Find (ref key, out value);
 }
Beispiel #8
0
        static bool SearchBlockForKey(byte[] block, Key key, out Value value)
        {
            int offset = BitConverter.ToUInt16 (block, 0); // grab the tree root
            value = Value.Empty;

            while (offset >= 2 && offset < Config.SortedBlockSize && block[offset] == (byte)RecordHeaderFlag.Record) {
                int startingOffset = offset;
                offset += 1; // skip header
                offset += 4; // skip tree pointers
                int keySize = Helper.Decode7BitInt (block, ref offset);
                int cmp = key.CompareTo (block, offset, keySize);
                if (cmp == 0) {
                    // Found it
                    var pair = ReadPair (block, ref startingOffset);
                    value = pair.Value;
                    return true;
                } else if (cmp < 0)
                    offset = BitConverter.ToUInt16 (block, startingOffset + 1); // key < node => explore left side
                else if (cmp > 0)
                    offset = BitConverter.ToUInt16 (block, startingOffset + 3); // key > node => explore right side
            }
            return false;
        }
Beispiel #9
0
        // Write a new key value pair to the output file. This method assumes that the data is being fed in key-sorted order.
        public void WritePair(Key key, Value value)
        {
            byte[] keySize = new byte[8];
            int keySizeLen = Helper.Encode7BitInt (keySize, key.Length);
            byte[] valueSize = new byte[8];
            int valueSizeLen = Helper.Encode7BitInt (valueSize, value.Length);

            int bytesNeeded = keySizeLen + key.Length + valueSizeLen + value.Length + 4 + 1;

            if ((_bufferPos + bytesNeeded) > Config.SortedBlockSize)
                WriteDataBlock (); // Do we need to write out a block before adding this key value pair?

            if (_bufferPos == 0) { // If we are at the beginning of the buffer, then add this key to the index.
                _pageIndex.Add (key);
                _bufferPos += 2; // Add a place for the root node offset.
            }

            _keyOffsets.Add ((ushort)_bufferPos); // Store the pair in preparation for writing.
            _buffer [_bufferPos++] = (byte)RecordHeaderFlag.Record; // A record header.
            _bufferPos += 4; // Adds space for left and right node pointers.

            // Writes the data out to the buffer.
            Array.Copy (keySize, 0, _buffer, _bufferPos, keySizeLen);
            _bufferPos += keySizeLen;
            Array.Copy (key.InternalBytes, 0, _buffer, _bufferPos, key.Length);
            _bufferPos += key.Length;
            Array.Copy (valueSize, 0, _buffer, _bufferPos, valueSizeLen);
            _bufferPos += valueSizeLen;
            Array.Copy (value.InternalBytes, 0, _buffer, _bufferPos, value.Length);
            _bufferPos += value.Length;

            WrittenSize += bytesNeeded;
        }
Beispiel #10
0
        static bool ScanBlockForKey(byte[] block, Key key, out Value value)
        {
            int offset = 2; // skip over the tree root pointer
            value = Value.Empty;

            while (offset >= 2 && offset < Config.SortedBlockSize && block[offset] == (byte)RecordHeaderFlag.Record) {
                int startingOffset = offset;
                offset++; // skip past the header flag
                offset += 4; // skip past the tree pointers
                int keySize = Helper.Decode7BitInt (block, ref offset);
                int cmp = key.CompareTo (block, offset, keySize);
                if (cmp == 0) {
                    // Found it
                    var pair = ReadPair (block, ref startingOffset);
                    value = pair.Value;
                    return true;
                } else if (cmp < 0)
                    return false;
                offset += keySize;
                // Skip past the value
                int valueSize = Helper.Decode7BitInt (block, ref offset);
                offset += valueSize;
            }
            return false;
        }
Beispiel #11
0
 public static bool Lookup(string baseFileName, int level, int version, RazorCache cache, Key key, out Value value, ExceptionHandling exceptionHandling, Action<string> logger)
 {
     SortedBlockTable sbt = new SortedBlockTable (cache, baseFileName, level, version);
     try {
         int dataBlockNum = FindBlockForKey (baseFileName, level, version, cache, key);
         if (dataBlockNum >= 0 && dataBlockNum < sbt._dataBlocks) {
             byte[] block = sbt.ReadBlock (LocalThreadAllocatedBlock (), dataBlockNum);
             return SearchBlockForKey (block, key, out value);
         }
     } finally {
         sbt.Close ();
     }
     value = Value.Empty;
     return false;
 }
Beispiel #12
0
        void InternalSet(Key k, Value v, IDictionary<string, byte[]> indexedValues)
        {
            int adds = 10;
            while (!_currentJournaledMemTable.Add(k, v)) {
                adds--;
                if (adds <= 0)
                    throw new InvalidOperationException ("Failed too many times trying to add an item to the JournaledMemTable");
            }
            // Add secondary index values if they were provided
            if (indexedValues != null)
                AddToIndex (k.KeyBytes, indexedValues);

            if (_currentJournaledMemTable.Full)
                RotateMemTable ();

            TableManager.Default.MarkKeyValueStoreAsModified (this);
        }
Beispiel #13
0
 byte[] AssembleGetResult(Key lookupKey, Value result)
 {
     switch (result.Type) {
     case ValueFlag.Null:
     case ValueFlag.Deleted:
         return null;
     case ValueFlag.SmallValue:
         return result.ValueBytes;
     case ValueFlag.LargeValueDescriptor:
         {
             lock (multiPageLock) {
                 result = InternalGet (lookupKey); // read the descriptor again in case it changed
                 if (result.Type == ValueFlag.LargeValueDescriptor) { // make sure type is still large value descriptor and continue
                     int valueSize = BitConverter.ToInt32 (result.ValueBytes, 0);
                     byte[] bytes = new byte[valueSize];
                     int offset = 0;
                     byte seqNum = 1;
                     while (offset < valueSize) {
                         var blockKey = lookupKey.WithSequence (seqNum);
                         var block = InternalGet (blockKey);
                         if (block.Type != ValueFlag.LargeValueChunk)
                             throw new InvalidDataException (string.Format ("Corrupted data: block is missing. Block Type: {0} SeqNum: {1}, Block Key: {2}", block.Type, seqNum, blockKey));
                         offset += block.CopyValueBytesTo (bytes, offset);
                         seqNum++;
                     }
                     return bytes;
                 } else
                     return AssembleGetResult (lookupKey, result);
             }
         }
     default:
         throw new InvalidOperationException ("Unexpected value flag for result.");
     }
 }
Beispiel #14
0
        // Set the specified key, value and indexedValues.
        // The multi page lock.
        public void Set(byte[] key, byte[] value, IDictionary<string, byte[]> indexedValues)
        {
            int valueSize = value.Length;
            if (valueSize <= Config.MaxSmallValueSize) {
                var k = new Key (key, 0);
                var v = new Value (value, ValueFlag.SmallValue);
                InternalSet (k, v, indexedValues);
            } else {
                lock (multiPageLock) {
                    if (value.Length >= Config.MaxLargeValueSize)
                        throw new InvalidDataException (string.Format ("Value is larger than the maximum size. ({0} bytes)", Config.MaxLargeValueSize));

                    int offset = 0;
                    byte seqNum = 1;

                    while (offset < valueSize) {
                        var k = new Key (key, seqNum);
                        int length = Math.Min (valueSize - offset, Config.MaxSmallValueSize);
                        var v = new Value (ByteArray.From (value, offset, length).InternalBytes, ValueFlag.LargeValueChunk);
                        InternalSet (k, v, null);
                        offset += length;
                        seqNum++;
                    }

                    var dk = new Key (key, 0);
                    var dv = new Value (BitConverter.GetBytes (valueSize), ValueFlag.LargeValueDescriptor);
                    InternalSet (dk, dv, indexedValues);
                }
            }
        }