/// <summary>
/// Serialize the block into the given stream.
/// </summary>
/// <param name="stream">Stream to write to.</param>
/// <param name="serializer"></param>
public static async Task <BlockHandle> WriteBlockAsync(Stream stream, Action <BinaryWriter> serializer)
{
using (var memoryStream = new MemoryStream(DefaultMemoryStreamSize))
using (var memoryWriter = new BinaryWriter(memoryStream))
{
// Serialize the table section.
serializer.Invoke(memoryWriter);
var blockSize = checked ((int)memoryStream.Position);
// Calculate the checksum.
var checksum = CRC64.ToCRC64(memoryStream.GetBuffer(), 0, blockSize);
// Get the position and size of this section within the stream.
var sectionHandle = new BlockHandle(stream.Position, blockSize);
// Write the checksum.
memoryWriter.Write(checksum);
var blockSizeWithChecksum = checked ((int)memoryStream.Position);
Utility.Assert((blockSizeWithChecksum - blockSize) == 8, "unexpected block size.");
// Write the buffer and checksum to the stream.
await stream.WriteAsync(memoryStream.GetBuffer(), 0, blockSizeWithChecksum).ConfigureAwait(false);
return(sectionHandle);
}
}
/// <summary>
/// Create a new <see cref="LogRecordBlockAsyncEnumerator"/> class.
/// </summary>
/// <param name="stream">Backup log stream.</param>
/// <param name="logRecordHandle">Location in the backup log stream of the log record chunks.</param>
public LogRecordBlockAsyncEnumerator(Stream stream, BlockHandle logRecordHandle)
{
stream.Position = logRecordHandle.Offset;
this.backupLogStream = stream;
this.logRecordHandle = logRecordHandle;
this.blockStream = new MemoryStream();
}
/// <summary>
/// Deserialize a <see cref="FileFooter"/> from the given stream.
/// </summary>
/// <param name="reader">Stream to read from.</param>
/// <param name="handle">Starting offset and size within the stream for the file footer.</param>
/// <returns>The <see cref="FileFooter"/> read from the stream.</returns>
public static FileFooter Read(BinaryReader reader, BlockHandle handle)
{
var propertiesHandle = BlockHandle.Read(reader);
var version = reader.ReadInt32();
return(new FileFooter(propertiesHandle, version));
}
/// <summary>
/// Deserialize a block from the given stream.
/// </summary>
/// <param name="stream">Stream to read from.</param>
/// <param name="blockHandle"></param>
/// <param name="deserializer"></param>
/// <returns>The block read from the stream.</returns>
public static async Task <TBlock> ReadBlockAsync <TBlock>(
Stream stream,
BlockHandle blockHandle,
Func <BinaryReader, BlockHandle, TBlock> deserializer)
{
// Data validation.
if (blockHandle.Offset < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_StreamOffset));
}
if (blockHandle.Size < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_BlockSize));
}
if (blockHandle.EndOffset + sizeof(ulong) > stream.Length)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Block_Beyond_Stream));
}
// Move the stream to the block's position within the stream.
stream.Position = blockHandle.Offset;
var blockSize = checked ((int)blockHandle.Size);
var blockSizeWithChecksum = blockSize + sizeof(ulong);
// Read the bytes for the block, to calculate the checksum.
var blockBytes = new byte[blockSizeWithChecksum];
await stream.ReadAsync(blockBytes, 0, blockBytes.Length).ConfigureAwait(false);
// Calculate the checksum.
var blockChecksum = CRC64.ToCRC64(blockBytes, 0, blockSize);
// Read the actual checksum (checksum is after the block bytes).
var checksum = BitConverter.ToUInt64(blockBytes, blockSize);
// Verify the checksum.
if (checksum != blockChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Checksum_Mismatch));
}
// Deserialize the actual file block.
using (var memoryStream = new MemoryStream(blockBytes, 0, blockSize, writable: false))
using (var memoryReader = new BinaryReader(memoryStream))
{
return(deserializer.Invoke(memoryReader, new BlockHandle(0, blockHandle.Size)));
}
}
/// <summary>
/// Read the given property value.
/// </summary>
/// <param name="reader">Stream to read from.</param>
/// <param name="property">Property to read.</param>
/// <param name="valueSize">The size in bytes of the value.</param>
protected override void ReadProperty(BinaryReader reader, string property, int valueSize)
{
switch (property)
{
case MetadataHandlePropertyName:
this.MetadataHandle = BlockHandle.Read(reader);
break;
case BlocksHandlePropertyName:
this.BlocksHandle = BlockHandle.Read(reader);
break;
case StateProviderCountPropertyName:
this.StateProviderCount = reader.ReadUInt32();
break;
case RootStateProviderCountPropertyName:
this.RootStateProviderCount = reader.ReadUInt32();
break;
case PartitionIdPropertyName:
this.PartitionId = new Guid(reader.ReadBytes(valueSize));
break;
case ReplicaIdPropertyName:
this.ReplicaId = reader.ReadInt64();
break;
case PrepareCheckpointLSNPropertyName:
if (this.test_Ignore)
{
// If test ignore set to true, we need to jump over the value block.
base.ReadProperty(reader, property, valueSize);
break;
}
this.PrepareCheckpointLSN = reader.ReadInt64();
Utility.Assert(
PrepareCheckpointLSN >= StateManagerConstants.ZeroLSN,
"StateManagerFileProperties::ReadProperty If the file has PrepareCheckpointLSN property, it must larger than or equal to 0, PrepareCheckpointLSN: {0}.",
prepareCheckpointLSN);
break;
default:
base.ReadProperty(reader, property, valueSize);
break;
}
}
/// <summary>
/// Deserialize the <see cref="FileProperties"/> from the given stream.
/// </summary>
/// <param name="reader">The stream to read from.</param>
/// <param name="handle">Starting offset and size within the stream for the file properties.</param>
/// <returns>The deserialized <see cref="FileProperties"/> section.</returns>
public static TProperties Read <TProperties>(BinaryReader reader, BlockHandle handle)
where TProperties : FileProperties, new()
{
var properties = new TProperties();
if (handle.Size == 0)
{
return(properties);
}
// The FileProperties start at the given 'offset' within the stream, and contains properties up to 'offset' + 'size'.
reader.BaseStream.Position = handle.Offset;
while (reader.BaseStream.Position < handle.EndOffset)
{
// Read the property name.
var property = reader.ReadString();
// Read the size in bytes of the property's value.
var valueSize = VarInt.ReadInt32(reader);
if (valueSize < 0)
{
throw new InvalidDataException(string.Format(SR.Error_FileProperties_Corrupt, SR.Error_FileProperties_NegativeSize));
}
if (reader.BaseStream.Position + valueSize > handle.EndOffset)
{
throw new InvalidDataException(
string.Format(SR.Error_FileProperties_Corrupt, SR.Error_FileProperties_LargeSize));
}
// Read the property's value.
properties.ReadProperty(reader, property, valueSize);
}
// Validate the properties section ended exactly where expected.
if (reader.BaseStream.Position != handle.EndOffset)
{
throw new InvalidDataException(string.Format(SR.Error_FileProperties_Corrupt, SR.Error_FileProperties_SectionSize));
}
return(properties);
}
/// <summary>
/// Initializes a new instance of the <see cref="BackupLogFileAsyncEnumerator"/> class.
/// </summary>
/// <param name="backupLogFile">Backup log file.</param>
/// <param name="logRecordHandle">Start and end offset for the log records.</param>
public BackupLogFileAsyncEnumerator(BackupLogFile backupLogFile, BlockHandle logRecordHandle)
{
Utility.Assert(backupLogFile != null, "backupLogFile != null");
this.backupLogFile = backupLogFile;
// Open the file with asynchronous flag and 4096 cache size (C# default).
// MCoskun: Uses default IOPriorityHint since this operation is called during Restore.
this.backupLogStream = FabricFile.Open(
this.backupLogFile.FileName,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
4096,
FileOptions.Asynchronous);
this.logBlockEnumerator = new LogRecordBlockAsyncEnumerator(this.backupLogStream, logRecordHandle);
this.index = 0;
}
/// <summary>
/// Deserialize a <see cref="StateManagerFileBlocks"/> from the given stream.
/// </summary>
/// <param name="reader">Stream to deserialize from.</param>
/// <param name="handle">Starting offset and size within the stream for the table index.</param>
/// <returns>The deserialized <see cref="StateManagerFileBlocks"/>.</returns>
public static StateManagerFileBlocks Read(BinaryReader reader, BlockHandle handle)
{
if (handle.Size == 0)
{
return(null);
}
var blocks = new StateManagerFileBlocks();
reader.BaseStream.Position = handle.Offset;
// Read the record block sizes.
blocks.RecordBlockSizes = ReadArray(reader);
// Validate the section was read correctly.
if (reader.BaseStream.Position != handle.EndOffset)
{
throw new InvalidDataException(SR.Error_SMBlocksCorrupt);
}
return(blocks);
}
/// <summary>
/// Read the given property value.
/// </summary>
/// <param name="reader">Stream to read from.</param>
/// <param name="property">Property to read.</param>
/// <param name="valueSize">The size in bytes of the value.</param>
protected override void ReadProperty(BinaryReader reader, string property, int valueSize)
{
switch (property)
{
case RecordsHandlePropertyName:
this.RecordsHandle = BlockHandle.Read(reader);
break;
case CountPropertyName:
this.Count = reader.ReadUInt32();
break;
case IndexingRecordEpochPropertyName:
var indexDataLossNumber = reader.ReadInt64();
var indexConfigurationNumber = reader.ReadInt64();
this.IndexingRecordEpoch = new Epoch(indexDataLossNumber, indexConfigurationNumber);
break;
case IndexingRecordLsnPropertyName:
this.IndexingRecordLsn = new LogicalSequenceNumber(reader.ReadInt64());
break;
case LastBackedUpEpochPropertyName:
var backupDataLossNumber = reader.ReadInt64();
var backupConfigurationNumber = reader.ReadInt64();
this.LastBackedUpEpoch = new Epoch(backupDataLossNumber, backupConfigurationNumber);
break;
case LastBackedUpLsnPropertyName:
this.LastBackedUpLsn = new LogicalSequenceNumber(reader.ReadInt64());
break;
default:
base.ReadProperty(reader, property, valueSize);
break;
}
}
/// <summary>
/// Open an existing <see cref="StateManagerFile"/> from the given file path.
/// </summary>
/// <param name="filePath">Path to the checkpoint file.</param>
/// <param name="traceType">Tracing information.</param>
/// <param name="deserializeTypes">Should the type information be deserialized.</param>
/// <param name="cancellationToken">Token used to signal cancellation.</param>
/// <returns>The opened <see cref="StateManagerFile"/>.</returns>
internal static async Task <StateManagerFile> OpenAsync(
string filePath,
string traceType,
bool deserializeTypes,
CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(filePath))
{
throw new ArgumentException(SR.Error_FilePath_Null_NoArgs);
}
if (!FabricFile.Exists(filePath))
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, SR.Error_FilePath_Null, filePath));
}
// Open the file with asynchronous flag and 4096 cache size (C# default).
// MCoskun: Default IoPriorityHint is used.
// Reason: Used during open, restore and complete checkpoint.
using (var filestream = FabricFile.Open(
filePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
4096,
FileOptions.Asynchronous))
{
var stateManagerFile = new StateManagerFile(filePath);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(
filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
FileFooter.SerializedSize);
stateManagerFile.footer = await FileBlock.ReadBlockAsync(
filestream,
footerHandle,
(reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Verify we know how to deserialize this version of the state manager checkpoint file.
if (stateManagerFile.footer.Version != Version)
{
throw new InvalidDataException(
string.Format(
CultureInfo.CurrentCulture,
"StateManager checkpoint file cannot be deserialized (unknown version number: {0}, expected version number: {1})",
stateManagerFile.footer.Version,
Version));
}
// Read and validate the properties section.
var propertiesHandle = stateManagerFile.footer.PropertiesHandle;
stateManagerFile.properties = await FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(reader, handle) => FileProperties.Read <StateManagerFileProperties>(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Read the state provider metadata.
stateManagerFile.StateProviderMetadata = await stateManagerFile.ReadMetadataAsync(filestream, traceType, deserializeTypes, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
stateManagerFile.FileSize = filestream.Length;
return(stateManagerFile);
}
}
/// <summary>
/// Open an existing <see cref="BackupLogFile"/> from the given file path.
/// </summary>
/// <param name="fileName">Path to the backup log file.</param>
/// <param name="cancellationToken">Token used to signal cancellation.</param>
/// <returns>The opened <see cref="BackupLogFile"/>.</returns>
public static async Task <BackupLogFile> OpenAsync(string fileName, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException(SR.Error_FilePath_Null_NoArgs);
}
if (!FabricFile.Exists(fileName))
{
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, SR.Error_FilePath_Null, fileName));
}
// Open the file with asynchronous flag and 4096 cache size (C# default).
// MCoskun: Uses default IOPriorityHint since this operation is called during Restore.
using (var filestream = FabricFile.Open(
fileName,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
4096,
FileOptions.Asynchronous))
{
var backupLogFile = new BackupLogFile(fileName);
backupLogFile.Size = filestream.Length;
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(
filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
FileFooter.SerializedSize);
backupLogFile.footer =
await
FileBlock.ReadBlockAsync(
filestream,
footerHandle,
(reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Verify we know how to deserialize this version of the backup log file.
if (backupLogFile.footer.Version != Version)
{
throw new InvalidDataException(
string.Format(
CultureInfo.CurrentCulture,
SR.Error_BackupLogFile_Deserialized,
backupLogFile.footer.Version,
Version));
}
// Read and validate the properties section.
var propertiesHandle = backupLogFile.footer.PropertiesHandle;
backupLogFile.properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(reader, handle) => FileProperties.Read <BackupLogFileProperties>(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(backupLogFile);
}
}
/// <summary>
/// Create a new <see cref="FileFooter"/>.
/// </summary>
/// <param name="propertiesHandle">Stream offset and size to the Properties section.</param>
/// <param name="version">Serialization version number for the file.</param>
public FileFooter(BlockHandle propertiesHandle, int version)
{
this.PropertiesHandle = propertiesHandle;
this.Version = version;
}
