/// <summary>
/// Write a block of log records to the output stream, including the size of the block and a checksum.
/// </summary>
/// <param name="outputStream">Backup log stream.</param>
/// <param name="blockStream">Stream for the block of log records.</param>
/// <param name="blockWriter">Writer for the block of log records.</param>
/// <param name="cancellationToken">Token used to signal cancellation.</param>
/// <returns>Task that represents the asynchronous operation.</returns>
private async Task WriteLogRecordBlockAsync(
Stream outputStream,
MemoryStream blockStream,
BinaryWriter blockWriter,
CancellationToken cancellationToken)
{
// Get the size of the block.
var blockSize = checked ((int)blockStream.Position);
Utility.Assert(blockSize > sizeof(int), "BackupLogFile tried to write a zero-sized block of log records.");
// Write the block size at the start of the block.
blockStream.Position = 0;
blockWriter.Write(blockSize);
// Checksum this block of data (with the block size included).
var blockChecksum = CRC64.ToCRC64(blockStream.GetBuffer(), 0, blockSize);
// Add the checksum at the end of the memory stream.
blockStream.Position = blockSize;
blockWriter.Write(blockChecksum);
// Flush the records block to the output stream.
await outputStream.WriteAsync(
blockStream.GetBuffer(),
0,
checked ((int)blockStream.Position),
cancellationToken).ConfigureAwait(false);
// Reset back to the start (leaving space for the 'int' block size).
blockStream.Position = sizeof(int);
}
/// <summary>
/// Serialize the block into the given stream.
/// </summary>
/// <param name="stream">Stream to write to.</param>
/// <param name="serializer"></param>
public static async Task <BlockHandle> WriteBlockAsync(Stream stream, Action <BinaryWriter> serializer)
{
using (var memoryStream = new MemoryStream(DefaultMemoryStreamSize))
using (var memoryWriter = new BinaryWriter(memoryStream))
{
// Serialize the table section.
serializer.Invoke(memoryWriter);
var blockSize = checked ((int)memoryStream.Position);
// Calculate the checksum.
var checksum = CRC64.ToCRC64(memoryStream.GetBuffer(), 0, blockSize);
// Get the position and size of this section within the stream.
var sectionHandle = new BlockHandle(stream.Position, blockSize);
// Write the checksum.
memoryWriter.Write(checksum);
var blockSizeWithChecksum = checked ((int)memoryStream.Position);
Utility.Assert((blockSizeWithChecksum - blockSize) == 8, "unexpected block size.");
// Write the buffer and checksum to the stream.
await stream.WriteAsync(memoryStream.GetBuffer(), 0, blockSizeWithChecksum).ConfigureAwait(false);
return(sectionHandle);
}
}
/// <summary>
/// Write the state provider metadata to the output stream.
/// </summary>
/// <param name="outputstream">Output stream.</param>
/// <param name="metadataList">The state providers' metadata.</param>
/// <param name="cancellationToken">Token used to signal cancellation.</param>
/// <returns>Task that represents the asynchronous operation.</returns>
private async Task <StateManagerFileBlocks> WriteMetadataAsync(
Stream outputstream,
IEnumerable <SerializableMetadata> metadataList,
CancellationToken cancellationToken)
{
// Track blocks of records, to improve sequential reads.
var recordBlockSizes = new List <int>();
var recordBlockSize = 0;
using (var itemStream = new MemoryStream(DesiredBlockSize * 2))
using (var itemWriter = new InMemoryBinaryWriter(itemStream))
{
foreach (var metadata in metadataList)
{
// Write each state provider metadata into the memory stream.
var recordPosition = checked ((int)itemStream.Position);
var recordSize = WriteMetadata(itemWriter, metadata);
// Checksum this metadata record (with the record size included).
var recordChecksum = CRC64.ToCRC64(itemStream.GetBuffer(), recordPosition, recordSize);
// Add the checksum at the end of the memory stream.
itemWriter.Write(recordChecksum);
// Track record block sizes. After enough bytes are buffered in memory, flush to disk.
recordBlockSize = checked ((int)itemStream.Position);
if (recordBlockSize >= DesiredBlockSize)
{
// Flush the metadata records to the output stream.
await outputstream.WriteAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);
recordBlockSizes.Add(recordBlockSize);
itemStream.Position = 0;
}
// Update properties.
this.Properties.StateProviderCount++;
if (metadata.ParentStateProviderId == DynamicStateManager.EmptyStateProviderId)
{
this.Properties.RootStateProviderCount++;
}
}
// Flush any remaining bytes to disk, and add the last record block size (if any).
recordBlockSize = checked ((int)itemStream.Position);
if (recordBlockSize > 0)
{
await outputstream.WriteAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);
recordBlockSizes.Add(recordBlockSize);
}
}
// Return the record block sizes.
var blocks = new StateManagerFileBlocks(recordBlockSizes.ToArray());
return(blocks);
}
/// <summary>
/// Deserialize a block from the given stream.
/// </summary>
/// <param name="stream">Stream to read from.</param>
/// <param name="blockHandle"></param>
/// <param name="deserializer"></param>
/// <returns>The block read from the stream.</returns>
public static async Task <TBlock> ReadBlockAsync <TBlock>(
Stream stream,
BlockHandle blockHandle,
Func <BinaryReader, BlockHandle, TBlock> deserializer)
{
// Data validation.
if (blockHandle.Offset < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_StreamOffset));
}
if (blockHandle.Size < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_BlockSize));
}
if (blockHandle.EndOffset + sizeof(ulong) > stream.Length)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Block_Beyond_Stream));
}
// Move the stream to the block's position within the stream.
stream.Position = blockHandle.Offset;
var blockSize = checked ((int)blockHandle.Size);
var blockSizeWithChecksum = blockSize + sizeof(ulong);
// Read the bytes for the block, to calculate the checksum.
var blockBytes = new byte[blockSizeWithChecksum];
await stream.ReadAsync(blockBytes, 0, blockBytes.Length).ConfigureAwait(false);
// Calculate the checksum.
var blockChecksum = CRC64.ToCRC64(blockBytes, 0, blockSize);
// Read the actual checksum (checksum is after the block bytes).
var checksum = BitConverter.ToUInt64(blockBytes, blockSize);
// Verify the checksum.
if (checksum != blockChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Checksum_Mismatch));
}
// Deserialize the actual file block.
using (var memoryStream = new MemoryStream(blockBytes, 0, blockSize, writable: false))
using (var memoryReader = new BinaryReader(memoryStream))
{
return(deserializer.Invoke(memoryReader, new BlockHandle(0, blockHandle.Size)));
}
}
/// <summary>
/// Read the list of state providers' metadata from the <see cref="StateManagerFile"/>.
/// </summary>
/// <param name="stream">Stream to read from.</param>
/// <param name="traceType">Tracing type information.</param>
/// <param name="deserializeTypes">Should the types be deserialized.</param>
/// <param name="cancellationToken">Token used to signal cancellation.</param>
/// <returns>The list of state providers' metadata read.</returns>
private async Task <List <SerializableMetadata> > ReadMetadataAsync(
Stream stream,
string traceType,
bool deserializeTypes,
CancellationToken cancellationToken)
{
var metadataList = new List <SerializableMetadata>((int)this.properties.StateProviderCount);
// Read the blocks, to improve sequential reads.
var blocks = await FileBlock.ReadBlockAsync(
stream,
this.properties.BlocksHandle,
(sectionReader, sectionHandle) => StateManagerFileBlocks.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Currently, we expect the block sizes to always be present.
if (blocks == null || blocks.RecordBlockSizes == null)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_BlockSizesMissing));
}
var recordBlockSizes = blocks.RecordBlockSizes;
// Read blocks from the file. Each state provider metadata is checksummed individually.
using (var itemStream = new MemoryStream(DesiredBlockSize * 2))
using (var itemReader = new InMemoryBinaryReader(itemStream))
{
stream.Position = this.properties.MetadataHandle.Offset;
var endOffset = this.properties.MetadataHandle.EndOffset;
// Each block has one or more state provider metadata records.
foreach (var recordBlockSize in recordBlockSizes)
{
if (stream.Position + recordBlockSize > endOffset)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_BlockExtendPastFile));
}
// Read the block into memory.
itemStream.Position = 0;
itemStream.SetLength(recordBlockSize);
await stream.ReadAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);
// Read to the end of the metadata section.
var endBlockOffset = itemStream.Length;
while (itemStream.Position < endBlockOffset)
{
var position = itemStream.Position;
// Read the record size and validate it is not obviously corrupted.
if (position + sizeof(int) > endBlockOffset)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MissingRecordSize));
}
var recordSize = itemReader.ReadInt32();
var recordSizeWithChecksum = recordSize + sizeof(ulong);
// We need to do extra validation on the recordSize, because we haven't validated the bits
// against the checksum and we need the recordSize to locate the checksum.
if (recordSize < 0)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_NegativeRecordSize));
}
if (position + recordSize > endBlockOffset)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_RecordExtendsPastFile));
}
if (position + recordSizeWithChecksum > endBlockOffset)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MissingChecksum));
}
// Compute the checksum.
var computedChecksum = CRC64.ToCRC64(
itemStream.GetBuffer(),
checked ((int)position),
recordSize);
// Read the checksum (checksum is after the record bytes).
itemStream.Position = position + recordSize;
var checksum = itemReader.ReadUInt64();
// Verify the checksum.
if (checksum != computedChecksum)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MismatchedChecksum));
}
// Read and re-create the state provider metadata, now that the checksum is validated.
itemStream.Position = position;
var metadata = ReadMetadata(itemReader, traceType, deserializeTypes);
metadataList.Add(metadata);
itemStream.Position = position + recordSizeWithChecksum;
}
}
}
// Post-validation.
if (metadataList.Count != (int)this.Properties.StateProviderCount)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MetadataCountMismatch));
}
return(metadataList);
}
public async Task <bool> MoveNextAsync(CancellationToken cancellationToken)
{
var position = this.backupLogStream.Position;
var endOffset = this.logRecordHandle.EndOffset;
if (position < endOffset)
{
// Read the size of the next block.
if (position + sizeof(int) > endOffset)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_MissingBlockSize));
}
await this.backupLogStream.ReadAsync(this.buffer, 0, sizeof(int)).ConfigureAwait(false);
var blockSize = BitConverter.ToInt32(this.buffer, 0);
var blockSizeWithChecksum = blockSize + sizeof(ulong);
// We need to do extra validation on the blockSize, because we haven't validated the bits
// against the checksum and we need the blockSize to locate the checksum.
if (blockSize < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_NegativeBlockSize));
}
if (position + blockSize > endOffset)
{
throw new InvalidDataException(
string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_ExtendsPastFileEnd));
}
if (position + blockSizeWithChecksum > endOffset)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_MissingChecksum));
}
// Read the next block into memory (including the size and checksum).
this.backupLogStream.Position = position;
this.blockStream.Position = 0;
this.blockStream.SetLength(blockSizeWithChecksum);
await this.backupLogStream.ReadAsync(this.blockStream.GetBuffer(), 0, blockSizeWithChecksum).ConfigureAwait(false);
// Compute the checksum.
var computedChecksum = CRC64.ToCRC64(this.blockStream.GetBuffer(), 0, blockSize);
// Read the checksum (checksum is after the block bytes).
var checksum = BitConverter.ToUInt64(this.blockStream.GetBuffer(), blockSize);
// Verify the checksum.
if (checksum != computedChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_Checksum_Mismatch));
}
// Return the prepared block reader, skipping over the int32 'size' field at the start and dropping the checksum.
this.blockStream.SetLength(blockSize);
this.Current = this.blockStream;
this.Current.Position = sizeof(int);
return(true);
}
this.Current = null;
return(false);
}
public ulong GetChecksum()
{
return(CRC64.ToCRC64(this.BaseStream.GetBuffer(), 0, checked ((int)this.BaseStream.Position)));
}
/// <summary>
/// Checksums the entire buffer.
/// </summary>
public ulong GetChecksum(int startOffset, int count)
{
return(CRC64.ToCRC64(this.BaseStream.GetBuffer(), startOffset, count));
}
