private bool TryParseCurrentEntry()
{
if (_reader.Position >= _reader.Length - 4)
{
Key = null;
return(false);
}
// An entry for a particular key-value pair has the form:
// shared_bytes: varint32
ulong sharedBytes = _reader.ReadVarLong();
if (Key == null && sharedBytes != 0)
{
throw new Exception("Shared bytes, but no key");
}
// unshared_bytes: varint32
ulong nonSharedBytes = _reader.ReadVarLong();
// value_length: varint32
ulong valueLength = _reader.ReadVarLong();
// key_delta: char[unshared_bytes]
ReadOnlySpan <byte> keyDelta = _reader.Read(nonSharedBytes);
Span <byte> combinedKey = new byte[sharedBytes + nonSharedBytes];
Key.Slice(0, (int)sharedBytes).CopyTo(combinedKey.Slice(0, (int)sharedBytes));
keyDelta.Slice(0, (int)nonSharedBytes).CopyTo(combinedKey.Slice((int)sharedBytes, (int)nonSharedBytes));
Key = combinedKey;
// Save handle and skip entry data
_lastValue = new BlockHandle((ulong)_reader.Position, valueLength);
_reader.Seek((int)valueLength, SeekOrigin.Current);
return(true);
}
public ResultStatus Get(Span <byte> key)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"\nSearch Key={key.ToHexString()}");
}
// To find a key in the table you:
// 1) Read the block index. This index have one entry for each block in the file. For each entry it holds the
// last index and a block handle for the block. Binary search this and find the correct block.
// 2) Search either each entry in the block (brute force) OR use the restart index at the end of the block to find an entry
// closer to the index you looking for. The restart index contain a subset of the keys with an offset of where it is located.
// Use this offset to start closer to the key (entry) you looking for.
// 3) Match the key and return the data
// Search block index
if (_blockIndex == null || _metaIndex == null)
{
Footer footer;
using (MemoryMappedViewStream stream = _memFile.CreateViewStream(_file.Length - Footer.FooterLength, Footer.FooterLength, MemoryMappedFileAccess.Read))
{
footer = Footer.Read(stream);
}
_blockIndex = footer.BlockIndexBlockHandle.ReadBlock(_memFile);
_metaIndex = footer.MetaindexBlockHandle.ReadBlock(_memFile);
}
BlockHandle handle = FindBlockHandleInBlockIndex(key);
if (handle == null)
{
Log.Error($"Expected to find block, but did not");
return(ResultStatus.NotFound);
}
if (_bloomFilterPolicy == null)
{
var filters = GetFilters();
if (filters.TryGetValue("filter.leveldb.BuiltinBloomFilter2", out BlockHandle filterHandle))
{
//var filterBlock = filterHandle.ReadBlock(_memViewStream);
var filterBlock = filterHandle.ReadBlock(_memFile);
if (Log.IsDebugEnabled)
{
Log.Debug("\n" + filterBlock.HexDump(cutAfterFive: true));
}
_bloomFilterPolicy = new BloomFilterPolicy();
_bloomFilterPolicy.Parse(filterBlock);
}
}
if (_bloomFilterPolicy?.KeyMayMatch(key, handle.Offset) ?? true)
{
var targetBlock = GetBlock(handle);
return(SeekKeyInBlockData(key, targetBlock));
}
return(ResultStatus.NotFound);
}
private bool TryParseCurrentEntry()
{
if (_reader.Eof)
{
return(false);
}
// An entry for a particular key-value pair has the form:
// shared_bytes: varint32
var sharedBytes = _reader.ReadVarLong();
Debug.Assert(Key != null || sharedBytes == 0);
// unshared_bytes: varint32
var nonSharedBytes = _reader.ReadVarLong();
// value_length: varint32
var valueLength = _reader.ReadVarLong();
// key_delta: char[unshared_bytes]
ReadOnlySpan <byte> keyDelta = _reader.Read(nonSharedBytes);
Span <byte> combinedKey = new byte[sharedBytes + nonSharedBytes];
Key.Slice(0, (int)sharedBytes).CopyTo(combinedKey.Slice(0, (int)sharedBytes));
keyDelta.Slice(0, (int)nonSharedBytes).CopyTo(combinedKey.Slice((int)sharedBytes, (int)nonSharedBytes));
Key = combinedKey;
// Save handle and skip entry data
_lastValue = new BlockHandle((ulong)_reader.Position, valueLength);
_reader.Seek((int)valueLength, SeekOrigin.Current);
return(true);
}
private void Flush()
{
byte[] lastKey = _blockCreator.LastKey;
if (lastKey?.Length > 0)
{
BlockHandle handle = WriteBlock(_stream, _blockCreator);
_blockIndexCreator.Add(lastKey, handle.Encode());
}
}
internal BlockSeeker(ReadOnlySpan <byte> blockData)
{
_blockData = blockData;
_reader = new SpanReader(blockData);
Key = null;
_restartOffset = 0;
_restartCount = 0;
_comparator = new BytewiseComparator();
_lastValue = null;
Initialize();
}
public void Finish()
{
Flush();
// writer filters block
// writer meta index block
//BlockHandle metaIndexHandle = WriteBlock(_stream, null); //TODO
BlockHandle metaIndexHandle = WriteBlock(_stream, _filterIndexCreator, true);
// write block index
BlockHandle blockIndexHandle = WriteBlock(_stream, _blockIndexCreator, true);
// write footer
var footer = new Footer(metaIndexHandle, blockIndexHandle);
footer.Write(_stream);
_stream.Flush();
}
/// <summary>
/// The second layer of the search tree is a table file's block index of keys. The block index is part of the table
/// file's metadata which is located at the end of its physical data file. The index contains one entry for each
/// logical data block within the table file. The entry contains the last key in the block and the offset of the block
/// within the table file. leveldb performs a binary search of the block index to locate a candidate data block. It
/// reads the candidate data block from the table file.
/// </summary>
public static Footer Read(Stream stream)
{
stream.Seek(-FooterLength, SeekOrigin.End);
Span <byte> footer = new byte[FooterLength];
stream.Read(footer);
var reader = new SpanReader(footer);
var metaIndexHandle = BlockHandle.ReadBlockHandle(ref reader);
var indexHandle = BlockHandle.ReadBlockHandle(ref reader);
reader.Seek(-sizeof(ulong), SeekOrigin.End);
ReadOnlySpan <byte> magic = reader.Read(8);
if (Magic.AsSpan().SequenceCompareTo(magic) != 0)
{
throw new Exception("Invalid footer. Magic end missing. This is not a proper table file");
}
return(new Footer(metaIndexHandle, indexHandle));
}
/// <summary>
/// The second layer of the search tree is a table file's block index of keys. The block index is part of the table
/// file's metadata which is located at the end of its physical data file. The index contains one entry for each
/// logical data block within the table file. The entry contains the last key in the block and the offset of the block
/// within the table file. leveldb performs a binary search of the block index to locate a candidate data block. It
/// reads the candidate data block from the table file.
/// </summary>
public static Footer Read(MemoryMappedViewStream stream)
{
stream.Seek(-FooterLength, SeekOrigin.End);
Span <byte> footer = new byte[FooterLength];
stream.Read(footer);
SpanReader reader = new SpanReader(footer);
BlockHandle metaIndexHandle = BlockHandle.ReadBlockHandle(ref reader);
BlockHandle indexHandle = BlockHandle.ReadBlockHandle(ref reader);
reader.Seek(-sizeof(ulong), SeekOrigin.End);
var magic = reader.ReadUInt64();
if (Magic != magic)
{
throw new Exception("Invalid footer. Magic end missing. This is not a proper table file");
}
return(new Footer(metaIndexHandle, indexHandle));
}
public ResultStatus Get(Span <byte> key)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"Get Key from table: {key.ToHexString()}");
}
// To find a key in the table you:
// 1) Read the block index. This index have one entry for each block in the file. For each entry it holds the
// last index and a block handle for the block. Binary search this and find the correct block.
// 2) Search either each entry in the block (brute force) OR use the restart index at the end of the block to find an entry
// closer to the index you looking for. The restart index contain a subset of the keys with an offset of where it is located.
// Use this offset to start closer to the key (entry) you looking for.
// 3) Match the key and return the data
// Search block index
Initialize();
BlockHandle handle = FindBlockHandleInBlockIndex(key);
if (handle == null)
{
if (Log.IsDebugEnabled)
{
Log.Warn($"Key was in range for table, but found no exact match in file {_file.Name}");
}
return(ResultStatus.NotFound);
}
if (_bloomFilterPolicy?.KeyMayMatch(key, handle.Offset) ?? true)
{
byte[] targetBlock = GetBlock(handle);
return(SeekKeyInBlockData(key, targetBlock));
}
return(ResultStatus.NotFound);
}
public Footer(BlockHandle metaIndexBlockHandle, BlockHandle blockIndexBlockHandle)
{
MetaIndexBlockHandle = metaIndexBlockHandle;
BlockIndexBlockHandle = blockIndexBlockHandle;
}
private byte[] GetBlock(BlockHandle handle)
{
if (!_blockCache.TryGetValue(handle, out byte[] targetBlock))
internal byte[] GetBlock(BlockHandle handle)
{
lock (_blockCache)
{
if (!_blockCache.TryGetValue(handle, out byte[] targetBlock))
protected bool Equals(BlockHandle other)
{
return(Offset == other.Offset && Length == other.Length);
}