/// <summary> /// This function not only writes the end of the block (checksum) but also /// writes the metadata about the block (size and reserved fields in the beginning. /// </summary> /// <remarks> /// The data is written is 8 bytes aligned. /// /// Name Type Size /// /// Metadata /// BlockSize int 4 /// RESERVED 4 /// /// Block byte[] N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// Checksum ulong 8 /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// Note: Larges Block supported is 2GB /// </remarks> /// <remarks> /// MCoskun: Note that KeyBlockMetadata.BlockSize includes KeyBlockMetadata and KeyBlock. /// It does not include the checksum /// </remarks> /// <remarks> /// MCoskun: Note that checksum includes the KeyBlockMetadata and KeyBlock. /// </remarks> private void WriteEndOfBlock() { Diagnostics.Assert(this.currentBlockPosition > 0, this.traceType, "Current block position should be higher than zero when end of block has reached"); Diagnostics.Assert(this.currentBlockPosition < this.blockAlignmentSize, this.traceType, "It should be strictly lesser than the blockalignment size"); this.keyBuffer.AssertIfNotAligned(); // Write blocksize and checksum. var blockEndPosition = (int)this.keyBuffer.BaseStream.Position; var blockStartPosition = (int)(blockEndPosition - this.currentBlockPosition); // Move buffer to start position. this.keyBuffer.BaseStream.Position = blockStartPosition; // Write block size - current block position + checksum will account for block size. var blockSize = this.currentBlockPosition + ChecksumSize; Diagnostics.Assert(blockSize > 0, this.traceType, "Block size should be greater than zero."); var keyBlockMetadata = new KeyChunkMetadata(blockSize); keyBlockMetadata.Write(this.keyBuffer); this.keyBuffer.AssertIfNotAligned(); // Move to end of stream to write checksum. this.keyBuffer.BaseStream.Position = blockEndPosition; var checksum = this.keyBuffer.GetChecksum(blockStartPosition, this.currentBlockPosition); this.keyBuffer.Write(checksum); this.keyBuffer.AssertIfNotAligned(); this.currentBlockPosition += ChecksumSize; }
private async Task <bool> ReadChunkAsync() { const int BlockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize; this.itemsBuffer.Clear(); this.index = 0; // Pick a chunk size that is a multiple of 4k lesser than the end offset. var chunkSize = this.GetChunkSize(); if (chunkSize == 0) { return(false); } // Read the entire chunk (plus the checksum and next chunk size) into memory. this.memoryStream.Position = 0; this.memoryStream.SetLength(chunkSize); await this.fileStream.ReadAsync(this.memoryStream.GetBuffer(), 0, chunkSize).ConfigureAwait(false); while (true) { var alignedStartBlockOffset = checked ((int)this.reader.BaseStream.Position); var blockMetadata = KeyChunkMetadata.Read(this.reader); var currentBlockSize = blockMetadata.BlockSize; var alignedBlockSize = GetBlockSize(currentBlockSize); // Check if the next block was only partially read. If so, read the remainder of it into memory. if ((alignedStartBlockOffset + alignedBlockSize) > chunkSize) { var remainingBlockSize = (alignedStartBlockOffset + alignedBlockSize) - chunkSize; Diagnostics.Assert(remainingBlockSize % BlockAlignmentSize == 0, this.traceType, "Remaining block size should be 4K aligned."); this.memoryStream.SetLength(chunkSize + remainingBlockSize); await this.fileStream.ReadAsync(this.memoryStream.GetBuffer(), chunkSize, remainingBlockSize).ConfigureAwait(false); chunkSize = chunkSize + remainingBlockSize; } var keysFromBlock = this.ReadBlock(currentBlockSize); foreach (var keyData in keysFromBlock) { this.itemsBuffer.Add(keyData); } // Move the reader ahead to the next block, if possible, else reset and break. this.reader.BaseStream.Position = alignedStartBlockOffset + alignedBlockSize; if (alignedStartBlockOffset + alignedBlockSize >= chunkSize) { Diagnostics.Assert( alignedStartBlockOffset + alignedBlockSize == chunkSize, this.traceType, "Failed to read block of keys due to chunk size not aligned to blocks."); break; } } // Track the number of keys returned. this.keyCount += this.itemsBuffer.Count; Diagnostics.Assert(this.itemsBuffer.Count > 0, this.traceType, "Unexpectedly read a chunk that has zero keys."); return(true); }