public Block(StorageOptions storageOptions, ReadOptions readOptions, BlockHandle handle, FileData fileData)
{
try
{
_handle = handle;
_storageOptions = storageOptions;
_fileData = fileData;
if (handle.Position > fileData.Size || (handle.Position + handle.Count + BlockTrailerSize) > fileData.Size)
throw new CorruptedDataException("The specified accessor is beyond the bounds of the provided mappedFile");
_accessor = _fileData.File.CreateAccessor(handle.Position, handle.Count + BlockTrailerSize);
if (readOptions.VerifyChecksums)
{
var crc = Crc.Unmask(_accessor.ReadInt32(handle.Count + 1));
var actualCrc = CalculateActualCrc(handle.Count + 1); // data + tag
if (crc != actualCrc)
throw new CorruptedDataException("block checksum mismatch");
}
RestartsCount = _accessor.ReadInt32(handle.Count - sizeof(int));
RestartsOffset = handle.Count - (RestartsCount * sizeof(int)) - sizeof(int);
if (RestartsOffset > handle.Count)
throw new CorruptedDataException("restart offset wrapped around");
}
catch (Exception)
{
Dispose();
throw;
}
}
public void DecodeFrom(IArrayAccessor accessor)
{
for (int i = 0; i < _tableMagicBytes.Length; i++)
{
var b = accessor[EncodedLength - _tableMagicBytes.Length + i];
if( b != _tableMagicBytes[i])
throw new InvalidOperationException("Not a sstable (bad magic number)");
}
MetaIndexHandle = new BlockHandle();
IndexHandle = new BlockHandle();
var size = MetaIndexHandle.DecodeFrom(accessor, 0);
IndexHandle.DecodeFrom(accessor, size);
}
internal IIterator GetFileIterator(ReadOptions readOptions, BlockHandle handle)
{
var fileNumber = (ulong)handle.Position;
var fileSize = handle.Count;
return storageContext.TableCache.NewIterator(readOptions, fileNumber, fileSize);
}
private BlockHandle WriteBlock(BlockBuilder block)
{
// File format contains a sequence of blocks where each block has:
// block_data: uint8[n]
// type: uint8 - right now always uncompressed
// crc: uint32
var size = block.Finish();
var originalPosition = block.OriginalPosition;
var handle = new BlockHandle
{
Count = size,
Position = originalPosition
};
// write trailer
block.Stream.WriteByte(0); // type - uncompressed
_dataStream.WriteInt32(Crc.Mask(block.Stream.WriteCrc));
block.Dispose();
return handle;
}
public void Finish()
{
Flush();
_closed = true;
BlockHandle filterBlockHandle = null;
//write filter block
if (_filterBuilder != null)
filterBlockHandle = _filterBuilder.Finish(_dataStream);
// write metadata block
var metaIndexBlock = new BlockBuilder(_dataStream, _storageState, _storageState.InternalKeyComparator, _storageState.Options.BlockRestartInterval);
if (filterBlockHandle != null)
{
metaIndexBlock.Add(("filter." + _storageState.Options.FilterPolicy.Name), filterBlockHandle.AsStream());
}
var metadIndexBlockHandle = WriteBlock(metaIndexBlock);
// write index block
if (_pendingIndexEntry)
{
var newKey = _storageState.InternalKeyComparator.FindShortestSuccessor(_lastKey, ref _scratchBuffer);
_indexBlock.Add(newKey, _pendingHandle.AsStream());
_pendingIndexEntry = false;
}
var indexBlockSize = _indexBlock.Finish();
_indexBlock.Stream.WriteByte(0);//write type, uncompressed
_indexBlock.Stream.WriteInt32(Crc.Mask(_indexBlock.Stream.WriteCrc));
_indexBlock.Stream.Position = _originalIndexStreamPosition;
var indexBlockHandler = new BlockHandle
{
Position = _dataStream.Position,
Count = indexBlockSize
};
_indexBlock.Stream.Stream.CopyTo(_dataStream);
// write footer
var footer = new Footer
{
IndexHandle = indexBlockHandler,
MetaIndexHandle = metadIndexBlockHandle
};
footer.EncodeTo(_dataStream);
}
/// <summary>
/// Advance: Forces the creation of a new block.
/// Client code should probably not call this method
/// </summary>
public void Flush()
{
if (_dataBlock.IsEmpty)
return;
if (_closed)
throw new InvalidOperationException("Cannot add after the table builder was closed");
if (_pendingIndexEntry)
throw new InvalidOperationException("Cannot call Flush when pending for an index entry");
_pendingHandle = WriteBlock(_dataBlock);
_dataBlock = new BlockBuilder(_dataStream, _storageState, _storageState.InternalKeyComparator, _storageState.Options.BlockRestartInterval);
_pendingIndexEntry = true;
_dataStream.Flush();
if (_filterBuilder != null)
_filterBuilder.StartBlock(_dataStream.Position);
}
private void InitDataIterator()
{
if (_indexIterator.IsValid == false)
{
SetDataIterator(null);
return;
}
var handle = new BlockHandle();
using (var stream = _indexIterator.CreateValueStream())
{
handle.DecodeFrom(stream);
}
if (handle.Equals(_currentDataHandle) && // nothing to change
_dataIterator != null) // but we have no iterator?
return;
IIterator blockIterator = null;
try
{
blockIterator = getIterator(_readOptions, handle);
SetDataIterator(blockIterator);
}
catch (Exception)
{
if (blockIterator != null)
blockIterator.Dispose();
throw;
}
_currentDataHandle = handle;
}
protected bool Equals(BlockHandle other)
{
return Position == other.Position && Count == other.Count;
}
public Table(StorageState storageState, FileData fileData)
{
_storageState = storageState;
try
{
_fileData = fileData;
if (_storageState.Options.MaxBlockCacheSizePerTableFile > 0)
{
_blockCache = new LruCache<BlockHandle, Block>(_storageState.Options.MaxBlockCacheSizePerTableFile);
}
if (fileData.Size < Footer.EncodedLength)
throw new CorruptedDataException("File is too short to be an sstable");
var footer = new Footer();
using (var accessor = fileData.File.CreateAccessor(fileData.Size - Footer.EncodedLength, Footer.EncodedLength))
{
footer.DecodeFrom(accessor);
}
var readOptions = new ReadOptions
{
VerifyChecksums = _storageState.Options.ParanoidChecks
};
_indexBlock = new Block(_storageState.Options, readOptions, footer.IndexHandle, fileData);
_indexBlock.IncrementUsage();
if (_storageState.Options.FilterPolicy == null)
return; // we don't need any metadata
using (var metaBlock = new Block(_storageState.Options, readOptions, footer.MetaIndexHandle, fileData))
using (var iterator = metaBlock.CreateIterator(CaseInsensitiveComparator.Default))
{
var filterName = ("filter." + _storageState.Options.FilterPolicy.Name);
iterator.Seek(filterName);
if (iterator.IsValid && CaseInsensitiveComparator.Default.Compare(filterName, iterator.Key) == 0)
{
var handle = new BlockHandle();
using (var stream = iterator.CreateValueStream())
{
handle.DecodeFrom(stream);
}
var filterAccessor = _fileData.File.CreateAccessor(handle.Position, handle.Count);
try
{
_filter = _storageState.Options.FilterPolicy.CreateFilter(filterAccessor);
}
catch (Exception)
{
if (_filter == null)
filterAccessor.Dispose();
else
_filter.Dispose();
throw;
}
}
}
}
catch (Exception)
{
Dispose();
throw;
}
}
internal IIterator CreateBlockIterator(ReadOptions readOptions, BlockHandle handle)
{
if (_blockCache == null)
{
Block uncachedBlock = null;
IIterator blockIterator = null;
try
{
uncachedBlock = new Block(_storageState.Options, readOptions, handle, _fileData);
// uncachedBlock.IncrementUsage(); - intentionally not calling this, will be disposed when the iterator is disposed
blockIterator = uncachedBlock.CreateIterator(_storageState.InternalKeyComparator);
return blockIterator;
}
catch (Exception)
{
if (uncachedBlock != null)
uncachedBlock.Dispose();
if (blockIterator != null)
blockIterator.Dispose();
throw;
}
}
Block value;
if (_blockCache.TryGet(handle, out value))
{
return value.CreateIterator(_storageState.InternalKeyComparator);
}
var block = new Block(_storageState.Options, readOptions, handle, _fileData);
block.IncrementUsage(); // the cache is using this, so avoid having it disposed by the cache while in use
_blockCache.Set(handle, block);
return block.CreateIterator(_storageState.InternalKeyComparator);
}
internal Tuple<Slice, Stream> InternalGet(ReadOptions readOptions, InternalKey key)
{
using (var iterator = _indexBlock.CreateIterator(_storageState.InternalKeyComparator))
{
iterator.Seek(key.TheInternalKey);
if (iterator.IsValid == false)
return null;
var handle = new BlockHandle();
using (var stream = iterator.CreateValueStream())
{
handle.DecodeFrom(stream);
}
if (_filter != null && _filter.KeyMayMatch(handle.Position, key.UserKey) == false)
{
return null; // opptimized not found by filter, no need to read the actual block
}
using (var blockIterator = CreateBlockIterator(readOptions, handle))
{
blockIterator.Seek(key.TheInternalKey);
if (blockIterator.IsValid == false)
return null;
return Tuple.Create(blockIterator.Key, blockIterator.CreateValueStream());
}
}
}
