private bool TryParseCurrentEntry()
        {
            if (_reader.Position >= _reader.Length - 4)
            {
                Key = null;
                return(false);
            }

            // An entry for a particular key-value pair has the form:
            //     shared_bytes: varint32
            ulong sharedBytes = _reader.ReadVarLong();

            if (Key == null && sharedBytes != 0)
            {
                throw new Exception("Shared bytes, but no key");
            }
            //     unshared_bytes: varint32
            ulong nonSharedBytes = _reader.ReadVarLong();
            //     value_length: varint32
            ulong valueLength = _reader.ReadVarLong();
            //     key_delta: char[unshared_bytes]
            ReadOnlySpan <byte> keyDelta = _reader.Read(nonSharedBytes);

            Span <byte> combinedKey = new byte[sharedBytes + nonSharedBytes];

            Key.Slice(0, (int)sharedBytes).CopyTo(combinedKey.Slice(0, (int)sharedBytes));
            keyDelta.Slice(0, (int)nonSharedBytes).CopyTo(combinedKey.Slice((int)sharedBytes, (int)nonSharedBytes));
            Key = combinedKey;

            // Save handle and skip entry data
            _lastValue = new BlockHandle((ulong)_reader.Position, valueLength);
            _reader.Seek((int)valueLength, SeekOrigin.Current);

            return(true);
        }
示例#2
0
        public ResultStatus Get(Span <byte> key)
        {
            if (Log.IsDebugEnabled)
            {
                Log.Debug($"\nSearch Key={key.ToHexString()}");
            }

            // To find a key in the table you:
            // 1) Read the block index. This index have one entry for each block in the file. For each entry it holds the
            //    last index and a block handle for the block. Binary search this and find the correct block.
            // 2) Search either each entry in the block (brute force) OR use the restart index at the end of the block to find an entry
            //    closer to the index you looking for. The restart index contain a subset of the keys with an offset of where it is located.
            //    Use this offset to start closer to the key (entry) you looking for.
            // 3) Match the key and return the data

            // Search block index
            if (_blockIndex == null || _metaIndex == null)
            {
                Footer footer;
                using (MemoryMappedViewStream stream = _memFile.CreateViewStream(_file.Length - Footer.FooterLength, Footer.FooterLength, MemoryMappedFileAccess.Read))
                {
                    footer = Footer.Read(stream);
                }
                _blockIndex = footer.BlockIndexBlockHandle.ReadBlock(_memFile);
                _metaIndex  = footer.MetaindexBlockHandle.ReadBlock(_memFile);
            }

            BlockHandle handle = FindBlockHandleInBlockIndex(key);

            if (handle == null)
            {
                Log.Error($"Expected to find block, but did not");
                return(ResultStatus.NotFound);
            }

            if (_bloomFilterPolicy == null)
            {
                var filters = GetFilters();
                if (filters.TryGetValue("filter.leveldb.BuiltinBloomFilter2", out BlockHandle filterHandle))
                {
                    //var filterBlock = filterHandle.ReadBlock(_memViewStream);
                    var filterBlock = filterHandle.ReadBlock(_memFile);
                    if (Log.IsDebugEnabled)
                    {
                        Log.Debug("\n" + filterBlock.HexDump(cutAfterFive: true));
                    }

                    _bloomFilterPolicy = new BloomFilterPolicy();
                    _bloomFilterPolicy.Parse(filterBlock);
                }
            }
            if (_bloomFilterPolicy?.KeyMayMatch(key, handle.Offset) ?? true)
            {
                var targetBlock = GetBlock(handle);

                return(SeekKeyInBlockData(key, targetBlock));
            }

            return(ResultStatus.NotFound);
        }
示例#3
0
        private bool TryParseCurrentEntry()
        {
            if (_reader.Eof)
            {
                return(false);
            }

            // An entry for a particular key-value pair has the form:
            //     shared_bytes: varint32
            var sharedBytes = _reader.ReadVarLong();

            Debug.Assert(Key != null || sharedBytes == 0);
            //     unshared_bytes: varint32
            var nonSharedBytes = _reader.ReadVarLong();
            //     value_length: varint32
            var valueLength = _reader.ReadVarLong();
            //     key_delta: char[unshared_bytes]
            ReadOnlySpan <byte> keyDelta = _reader.Read(nonSharedBytes);

            Span <byte> combinedKey = new byte[sharedBytes + nonSharedBytes];

            Key.Slice(0, (int)sharedBytes).CopyTo(combinedKey.Slice(0, (int)sharedBytes));
            keyDelta.Slice(0, (int)nonSharedBytes).CopyTo(combinedKey.Slice((int)sharedBytes, (int)nonSharedBytes));
            Key = combinedKey;

            // Save handle and skip entry data
            _lastValue = new BlockHandle((ulong)_reader.Position, valueLength);
            _reader.Seek((int)valueLength, SeekOrigin.Current);

            return(true);
        }
 private void Flush()
 {
     byte[] lastKey = _blockCreator.LastKey;
     if (lastKey?.Length > 0)
     {
         BlockHandle handle = WriteBlock(_stream, _blockCreator);
         _blockIndexCreator.Add(lastKey, handle.Encode());
     }
 }
        internal BlockSeeker(ReadOnlySpan <byte> blockData)
        {
            _blockData     = blockData;
            _reader        = new SpanReader(blockData);
            Key            = null;
            _restartOffset = 0;
            _restartCount  = 0;
            _comparator    = new BytewiseComparator();
            _lastValue     = null;

            Initialize();
        }
        public void Finish()
        {
            Flush();

            // writer filters block
            // writer meta index block
            //BlockHandle metaIndexHandle = WriteBlock(_stream, null); //TODO
            BlockHandle metaIndexHandle = WriteBlock(_stream, _filterIndexCreator, true);
            // write block index
            BlockHandle blockIndexHandle = WriteBlock(_stream, _blockIndexCreator, true);

            // write footer
            var footer = new Footer(metaIndexHandle, blockIndexHandle);

            footer.Write(_stream);
            _stream.Flush();
        }
示例#7
0
        /// <summary>
        ///     The second layer of the search tree is a table file's block index of keys. The block index is part of the table
        ///     file's metadata which is located at the end of its physical data file. The index contains one entry for each
        ///     logical data block within the table file. The entry contains the last key in the block and the offset of the block
        ///     within the table file. leveldb performs a binary search of the block index to locate a candidate data block. It
        ///     reads the candidate data block from the table file.
        /// </summary>
        public static Footer Read(Stream stream)
        {
            stream.Seek(-FooterLength, SeekOrigin.End);
            Span <byte> footer = new byte[FooterLength];

            stream.Read(footer);

            var reader = new SpanReader(footer);

            var metaIndexHandle = BlockHandle.ReadBlockHandle(ref reader);
            var indexHandle     = BlockHandle.ReadBlockHandle(ref reader);

            reader.Seek(-sizeof(ulong), SeekOrigin.End);
            ReadOnlySpan <byte> magic = reader.Read(8);

            if (Magic.AsSpan().SequenceCompareTo(magic) != 0)
            {
                throw new Exception("Invalid footer. Magic end missing. This is not a proper table file");
            }

            return(new Footer(metaIndexHandle, indexHandle));
        }
示例#8
0
        /// <summary>
        ///     The second layer of the search tree is a table file's block index of keys. The block index is part of the table
        ///     file's metadata which is located at the end of its physical data file. The index contains one entry for each
        ///     logical data block within the table file. The entry contains the last key in the block and the offset of the block
        ///     within the table file. leveldb performs a binary search of the block index to locate a candidate data block. It
        ///     reads the candidate data block from the table file.
        /// </summary>
        public static Footer Read(MemoryMappedViewStream stream)
        {
            stream.Seek(-FooterLength, SeekOrigin.End);
            Span <byte> footer = new byte[FooterLength];

            stream.Read(footer);

            SpanReader reader = new SpanReader(footer);

            BlockHandle metaIndexHandle = BlockHandle.ReadBlockHandle(ref reader);
            BlockHandle indexHandle     = BlockHandle.ReadBlockHandle(ref reader);

            reader.Seek(-sizeof(ulong), SeekOrigin.End);
            var magic = reader.ReadUInt64();

            if (Magic != magic)
            {
                throw new Exception("Invalid footer. Magic end missing. This is not a proper table file");
            }

            return(new Footer(metaIndexHandle, indexHandle));
        }
示例#9
0
        public ResultStatus Get(Span <byte> key)
        {
            if (Log.IsDebugEnabled)
            {
                Log.Debug($"Get Key from table: {key.ToHexString()}");
            }

            // To find a key in the table you:
            // 1) Read the block index. This index have one entry for each block in the file. For each entry it holds the
            //    last index and a block handle for the block. Binary search this and find the correct block.
            // 2) Search either each entry in the block (brute force) OR use the restart index at the end of the block to find an entry
            //    closer to the index you looking for. The restart index contain a subset of the keys with an offset of where it is located.
            //    Use this offset to start closer to the key (entry) you looking for.
            // 3) Match the key and return the data

            // Search block index
            Initialize();

            BlockHandle handle = FindBlockHandleInBlockIndex(key);

            if (handle == null)
            {
                if (Log.IsDebugEnabled)
                {
                    Log.Warn($"Key was in range for table, but found no exact match in file {_file.Name}");
                }
                return(ResultStatus.NotFound);
            }

            if (_bloomFilterPolicy?.KeyMayMatch(key, handle.Offset) ?? true)
            {
                byte[] targetBlock = GetBlock(handle);

                return(SeekKeyInBlockData(key, targetBlock));
            }

            return(ResultStatus.NotFound);
        }
示例#10
0
 public Footer(BlockHandle metaIndexBlockHandle, BlockHandle blockIndexBlockHandle)
 {
     MetaIndexBlockHandle  = metaIndexBlockHandle;
     BlockIndexBlockHandle = blockIndexBlockHandle;
 }
示例#11
0
 private byte[] GetBlock(BlockHandle handle)
 {
     if (!_blockCache.TryGetValue(handle, out byte[] targetBlock))
示例#12
0
 internal byte[] GetBlock(BlockHandle handle)
 {
     lock (_blockCache)
     {
         if (!_blockCache.TryGetValue(handle, out byte[] targetBlock))
示例#13
0
 protected bool Equals(BlockHandle other)
 {
     return(Offset == other.Offset && Length == other.Length);
 }