public Block(StorageOptions storageOptions, ReadOptions readOptions, BlockHandle handle, FileData fileData) { try { _handle = handle; _storageOptions = storageOptions; _fileData = fileData; if (handle.Position > fileData.Size || (handle.Position + handle.Count + BlockTrailerSize) > fileData.Size) throw new CorruptedDataException("The specified accessor is beyond the bounds of the provided mappedFile"); _accessor = _fileData.File.CreateAccessor(handle.Position, handle.Count + BlockTrailerSize); if (readOptions.VerifyChecksums) { var crc = Crc.Unmask(_accessor.ReadInt32(handle.Count + 1)); var actualCrc = CalculateActualCrc(handle.Count + 1); // data + tag if (crc != actualCrc) throw new CorruptedDataException("block checksum mismatch"); } RestartsCount = _accessor.ReadInt32(handle.Count - sizeof(int)); RestartsOffset = handle.Count - (RestartsCount * sizeof(int)) - sizeof(int); if (RestartsOffset > handle.Count) throw new CorruptedDataException("restart offset wrapped around"); } catch (Exception) { Dispose(); throw; } }
public void DecodeFrom(IArrayAccessor accessor) { for (int i = 0; i < _tableMagicBytes.Length; i++) { var b = accessor[EncodedLength - _tableMagicBytes.Length + i]; if( b != _tableMagicBytes[i]) throw new InvalidOperationException("Not a sstable (bad magic number)"); } MetaIndexHandle = new BlockHandle(); IndexHandle = new BlockHandle(); var size = MetaIndexHandle.DecodeFrom(accessor, 0); IndexHandle.DecodeFrom(accessor, size); }
internal IIterator GetFileIterator(ReadOptions readOptions, BlockHandle handle) { var fileNumber = (ulong)handle.Position; var fileSize = handle.Count; return storageContext.TableCache.NewIterator(readOptions, fileNumber, fileSize); }
private BlockHandle WriteBlock(BlockBuilder block) { // File format contains a sequence of blocks where each block has: // block_data: uint8[n] // type: uint8 - right now always uncompressed // crc: uint32 var size = block.Finish(); var originalPosition = block.OriginalPosition; var handle = new BlockHandle { Count = size, Position = originalPosition }; // write trailer block.Stream.WriteByte(0); // type - uncompressed _dataStream.WriteInt32(Crc.Mask(block.Stream.WriteCrc)); block.Dispose(); return handle; }
public void Finish() { Flush(); _closed = true; BlockHandle filterBlockHandle = null; //write filter block if (_filterBuilder != null) filterBlockHandle = _filterBuilder.Finish(_dataStream); // write metadata block var metaIndexBlock = new BlockBuilder(_dataStream, _storageState, _storageState.InternalKeyComparator, _storageState.Options.BlockRestartInterval); if (filterBlockHandle != null) { metaIndexBlock.Add(("filter." + _storageState.Options.FilterPolicy.Name), filterBlockHandle.AsStream()); } var metadIndexBlockHandle = WriteBlock(metaIndexBlock); // write index block if (_pendingIndexEntry) { var newKey = _storageState.InternalKeyComparator.FindShortestSuccessor(_lastKey, ref _scratchBuffer); _indexBlock.Add(newKey, _pendingHandle.AsStream()); _pendingIndexEntry = false; } var indexBlockSize = _indexBlock.Finish(); _indexBlock.Stream.WriteByte(0);//write type, uncompressed _indexBlock.Stream.WriteInt32(Crc.Mask(_indexBlock.Stream.WriteCrc)); _indexBlock.Stream.Position = _originalIndexStreamPosition; var indexBlockHandler = new BlockHandle { Position = _dataStream.Position, Count = indexBlockSize }; _indexBlock.Stream.Stream.CopyTo(_dataStream); // write footer var footer = new Footer { IndexHandle = indexBlockHandler, MetaIndexHandle = metadIndexBlockHandle }; footer.EncodeTo(_dataStream); }
/// <summary> /// Advance: Forces the creation of a new block. /// Client code should probably not call this method /// </summary> public void Flush() { if (_dataBlock.IsEmpty) return; if (_closed) throw new InvalidOperationException("Cannot add after the table builder was closed"); if (_pendingIndexEntry) throw new InvalidOperationException("Cannot call Flush when pending for an index entry"); _pendingHandle = WriteBlock(_dataBlock); _dataBlock = new BlockBuilder(_dataStream, _storageState, _storageState.InternalKeyComparator, _storageState.Options.BlockRestartInterval); _pendingIndexEntry = true; _dataStream.Flush(); if (_filterBuilder != null) _filterBuilder.StartBlock(_dataStream.Position); }
private void InitDataIterator() { if (_indexIterator.IsValid == false) { SetDataIterator(null); return; } var handle = new BlockHandle(); using (var stream = _indexIterator.CreateValueStream()) { handle.DecodeFrom(stream); } if (handle.Equals(_currentDataHandle) && // nothing to change _dataIterator != null) // but we have no iterator? return; IIterator blockIterator = null; try { blockIterator = getIterator(_readOptions, handle); SetDataIterator(blockIterator); } catch (Exception) { if (blockIterator != null) blockIterator.Dispose(); throw; } _currentDataHandle = handle; }
protected bool Equals(BlockHandle other) { return Position == other.Position && Count == other.Count; }
public Table(StorageState storageState, FileData fileData) { _storageState = storageState; try { _fileData = fileData; if (_storageState.Options.MaxBlockCacheSizePerTableFile > 0) { _blockCache = new LruCache<BlockHandle, Block>(_storageState.Options.MaxBlockCacheSizePerTableFile); } if (fileData.Size < Footer.EncodedLength) throw new CorruptedDataException("File is too short to be an sstable"); var footer = new Footer(); using (var accessor = fileData.File.CreateAccessor(fileData.Size - Footer.EncodedLength, Footer.EncodedLength)) { footer.DecodeFrom(accessor); } var readOptions = new ReadOptions { VerifyChecksums = _storageState.Options.ParanoidChecks }; _indexBlock = new Block(_storageState.Options, readOptions, footer.IndexHandle, fileData); _indexBlock.IncrementUsage(); if (_storageState.Options.FilterPolicy == null) return; // we don't need any metadata using (var metaBlock = new Block(_storageState.Options, readOptions, footer.MetaIndexHandle, fileData)) using (var iterator = metaBlock.CreateIterator(CaseInsensitiveComparator.Default)) { var filterName = ("filter." + _storageState.Options.FilterPolicy.Name); iterator.Seek(filterName); if (iterator.IsValid && CaseInsensitiveComparator.Default.Compare(filterName, iterator.Key) == 0) { var handle = new BlockHandle(); using (var stream = iterator.CreateValueStream()) { handle.DecodeFrom(stream); } var filterAccessor = _fileData.File.CreateAccessor(handle.Position, handle.Count); try { _filter = _storageState.Options.FilterPolicy.CreateFilter(filterAccessor); } catch (Exception) { if (_filter == null) filterAccessor.Dispose(); else _filter.Dispose(); throw; } } } } catch (Exception) { Dispose(); throw; } }
internal IIterator CreateBlockIterator(ReadOptions readOptions, BlockHandle handle) { if (_blockCache == null) { Block uncachedBlock = null; IIterator blockIterator = null; try { uncachedBlock = new Block(_storageState.Options, readOptions, handle, _fileData); // uncachedBlock.IncrementUsage(); - intentionally not calling this, will be disposed when the iterator is disposed blockIterator = uncachedBlock.CreateIterator(_storageState.InternalKeyComparator); return blockIterator; } catch (Exception) { if (uncachedBlock != null) uncachedBlock.Dispose(); if (blockIterator != null) blockIterator.Dispose(); throw; } } Block value; if (_blockCache.TryGet(handle, out value)) { return value.CreateIterator(_storageState.InternalKeyComparator); } var block = new Block(_storageState.Options, readOptions, handle, _fileData); block.IncrementUsage(); // the cache is using this, so avoid having it disposed by the cache while in use _blockCache.Set(handle, block); return block.CreateIterator(_storageState.InternalKeyComparator); }
internal Tuple<Slice, Stream> InternalGet(ReadOptions readOptions, InternalKey key) { using (var iterator = _indexBlock.CreateIterator(_storageState.InternalKeyComparator)) { iterator.Seek(key.TheInternalKey); if (iterator.IsValid == false) return null; var handle = new BlockHandle(); using (var stream = iterator.CreateValueStream()) { handle.DecodeFrom(stream); } if (_filter != null && _filter.KeyMayMatch(handle.Position, key.UserKey) == false) { return null; // opptimized not found by filter, no need to read the actual block } using (var blockIterator = CreateBlockIterator(readOptions, handle)) { blockIterator.Seek(key.TheInternalKey); if (blockIterator.IsValid == false) return null; return Tuple.Create(blockIterator.Key, blockIterator.CreateValueStream()); } } }