/// <summary>
/// This function not only writes the end of the block (checksum) but also
/// writes the metadata about the block (size and reserved fields in the beginning.
/// </summary>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// Metadata
/// BlockSize int 4
/// RESERVED 4
///
/// Block byte[] N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// Checksum ulong 8
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Block supported is 2GB
/// </remarks>
/// <remarks>
/// MCoskun: Note that KeyBlockMetadata.BlockSize includes KeyBlockMetadata and KeyBlock.
/// It does not include the checksum
/// </remarks>
/// <remarks>
/// MCoskun: Note that checksum includes the KeyBlockMetadata and KeyBlock.
/// </remarks>
private void WriteEndOfBlock()
{
Diagnostics.Assert(this.currentBlockPosition > 0, this.traceType, "Current block position should be higher than zero when end of block has reached");
Diagnostics.Assert(this.currentBlockPosition < this.blockAlignmentSize, this.traceType, "It should be strictly lesser than the blockalignment size");
this.keyBuffer.AssertIfNotAligned();
// Write blocksize and checksum.
var blockEndPosition = (int)this.keyBuffer.BaseStream.Position;
var blockStartPosition = (int)(blockEndPosition - this.currentBlockPosition);
// Move buffer to start position.
this.keyBuffer.BaseStream.Position = blockStartPosition;
// Write block size - current block position + checksum will account for block size.
var blockSize = this.currentBlockPosition + ChecksumSize;
Diagnostics.Assert(blockSize > 0, this.traceType, "Block size should be greater than zero.");
var keyBlockMetadata = new KeyChunkMetadata(blockSize);
keyBlockMetadata.Write(this.keyBuffer);
this.keyBuffer.AssertIfNotAligned();
// Move to end of stream to write checksum.
this.keyBuffer.BaseStream.Position = blockEndPosition;
var checksum = this.keyBuffer.GetChecksum(blockStartPosition, this.currentBlockPosition);
this.keyBuffer.Write(checksum);
this.keyBuffer.AssertIfNotAligned();
this.currentBlockPosition += ChecksumSize;
}
private async Task <bool> ReadChunkAsync()
{
const int BlockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
this.itemsBuffer.Clear();
this.index = 0;
// Pick a chunk size that is a multiple of 4k lesser than the end offset.
var chunkSize = this.GetChunkSize();
if (chunkSize == 0)
{
return(false);
}
// Read the entire chunk (plus the checksum and next chunk size) into memory.
this.memoryStream.Position = 0;
this.memoryStream.SetLength(chunkSize);
await this.fileStream.ReadAsync(this.memoryStream.GetBuffer(), 0, chunkSize).ConfigureAwait(false);
while (true)
{
var alignedStartBlockOffset = checked ((int)this.reader.BaseStream.Position);
var blockMetadata = KeyChunkMetadata.Read(this.reader);
var currentBlockSize = blockMetadata.BlockSize;
var alignedBlockSize = GetBlockSize(currentBlockSize);
// Check if the next block was only partially read. If so, read the remainder of it into memory.
if ((alignedStartBlockOffset + alignedBlockSize) > chunkSize)
{
var remainingBlockSize = (alignedStartBlockOffset + alignedBlockSize) - chunkSize;
Diagnostics.Assert(remainingBlockSize % BlockAlignmentSize == 0, this.traceType, "Remaining block size should be 4K aligned.");
this.memoryStream.SetLength(chunkSize + remainingBlockSize);
await this.fileStream.ReadAsync(this.memoryStream.GetBuffer(), chunkSize, remainingBlockSize).ConfigureAwait(false);
chunkSize = chunkSize + remainingBlockSize;
}
var keysFromBlock = this.ReadBlock(currentBlockSize);
foreach (var keyData in keysFromBlock)
{
this.itemsBuffer.Add(keyData);
}
// Move the reader ahead to the next block, if possible, else reset and break.
this.reader.BaseStream.Position = alignedStartBlockOffset + alignedBlockSize;
if (alignedStartBlockOffset + alignedBlockSize >= chunkSize)
{
Diagnostics.Assert(
alignedStartBlockOffset + alignedBlockSize == chunkSize,
this.traceType,
"Failed to read block of keys due to chunk size not aligned to blocks.");
break;
}
}
// Track the number of keys returned.
this.keyCount += this.itemsBuffer.Count;
Diagnostics.Assert(this.itemsBuffer.Count > 0, this.traceType, "Unexpectedly read a chunk that has zero keys.");
return(true);
}