/// <summary>
/// Deserializes the metadata (footer, properties, etc.) for this checkpoint file.
/// </summary>
private async Task ReadMetadataAsync()
{
Stream filestream = null;
try
{
filestream = this.ReaderPool.AcquireStream();
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(filestream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
this.Footer =
await FileBlock.ReadBlockAsync(filestream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the checkpoint file.
if (this.Footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_KeyCheckpointFile_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = this.Footer.PropertiesHandle;
this.Properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <KeyCheckpointFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
finally
{
this.ReaderPool.ReleaseStream(filestream, true);
}
}
public static async Task <RecoveryPointMetadataFile> OpenAsync(
Stream filestream,
string recoveryPointMetadataFilePath,
CancellationToken cancellationToken)
{
var recoveryPointMetadataFile = new RecoveryPointMetadataFile(recoveryPointMetadataFilePath);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(
filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
FileFooter.SerializedSize);
recoveryPointMetadataFile.footer =
await
FileBlock.ReadBlockAsync(
filestream,
footerHandle,
(reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Verify we know how to deserialize this version of the backup log file.
if (recoveryPointMetadataFile.footer.Version != Version)
{
throw new InvalidDataException(
string.Format(
CultureInfo.CurrentCulture,
SR.Error_BackupMetadata_Deserialized,
recoveryPointMetadataFile.footer.Version,
Version));
}
// Read and validate the properties section.
var propertiesHandle = recoveryPointMetadataFile.footer.PropertiesHandle;
recoveryPointMetadataFile.properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(reader, handle) => FileProperties.Read <RecoveryPointMetadataFileProperties>(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(recoveryPointMetadataFile);
}
public static async Task WriteAsync(MetadataTable metadataTable, Stream stream)
{
// Write all metadata to disk.
var properties = await WriteDiskMetadataAsync(metadataTable.Table, stream).ConfigureAwait(false);
properties.CheckpointLSN = metadataTable.CheckpointLSN;
// Write the Properties.
var propertiesHandle = await FileBlock.WriteBlockAsync(stream, (sectionWriter) => properties.Write(sectionWriter)).ConfigureAwait(false);
// Write the Footer.
var footer = new FileFooter(propertiesHandle, FileVersion);
await FileBlock.WriteBlockAsync(stream, (sectionWriter) => footer.Write(sectionWriter)).ConfigureAwait(false);
// Finally, flush to disk.
await stream.FlushAsync().ConfigureAwait(false);
}
private FileFooter ReadFileFooter(byte[] fileData, int fileHeaderFooterOffset)
{
const int FooterRecordByteLength = 36;
const int NumSetLists = 8;
const int NumOtherRecords = 4;
// note there will be other footer records if IRs are used.
const int MinimumFooterLength = (NumSetLists + NumOtherRecords) * FooterRecordByteLength;
if (fileHeaderFooterOffset > fileData.Length - MinimumFooterLength)
{
throw new FileFooterSmallException();
}
var recordCount = (fileData.Length - fileHeaderFooterOffset) / FooterRecordByteLength;
if (fileHeaderFooterOffset + (recordCount * FooterRecordByteLength) != fileData.Length)
{
throw new FileFooterBadException();
}
var result = new FileFooter();
var enc = new UTF8Encoding();
for (int n = 0; n < recordCount; ++n)
{
var startOffset = fileHeaderFooterOffset + (n * FooterRecordByteLength);
var title = enc.GetString(fileData, startOffset, 4);
var sectionType = GetSectionType(title);
result.Records.Add(new FileFooterRecord
{
Title = title,
SectionType = sectionType,
Description = GetFooterRecordDescription(sectionType),
StartSectionOffset = BitConverter.ToInt32(fileData, startOffset + 4),
SectionLength = BitConverter.ToInt32(fileData, startOffset + 12),
IsSectionDeflated = BitConverter.ToInt32(fileData, startOffset + 20) == 1,
InflatedLength = BitConverter.ToInt32(fileData, startOffset + 24),
});
}
return(result);
}
/// <summary>
/// A Flush indicates that all keys have been written to the checkpoint file (via AddItemAsync), so
/// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra
/// metadata (e.g. properties and footer), and flush to disk.
/// </summary>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <returns></returns>
public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// If there's any buffered keys in memory, flush them to disk first.
if (memoryBuffer.BaseStream.Position > 0)
{
await this.FlushMemoryBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false);
}
// Update the Properties.
this.Properties.KeysHandle = new BlockHandle(0, fileStream.Position);
// Write the Properties.
var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false);
// Write the Footer.
this.Footer = new FileFooter(propertiesHandle, FileVersion);
await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false);
// Finally, flush to disk.
await fileStream.FlushAsync().ConfigureAwait(false);
}
private void ReadHeaderAndFooter(byte[] fileData)
{
FileHeader = ReadFileHeader(fileData);
FileFooter = ReadFileFooter(fileData, FileHeader.FooterOffset);
var enc = new UTF8Encoding();
// backup description...
var backupDescriptionRecord = FileFooter.Records.SingleOrDefault(x => x.Title == DescriptionCode);
if (backupDescriptionRecord != null)
{
FileHeader.BackupDescription = enc.GetString(
fileData,
backupDescriptionRecord.StartSectionOffset,
backupDescriptionRecord.SectionLength);
}
// names of set lists...
var setListNamesRecord = FileFooter.Records.SingleOrDefault(x => x.Title == SetListNamesCode);
if (setListNamesRecord != null)
{
var setListNames = enc.GetString(
fileData,
setListNamesRecord.StartSectionOffset,
setListNamesRecord.SectionLength);
var tokens = setListNames.Split('\0');
foreach (var token in tokens)
{
if (!string.IsNullOrWhiteSpace(token))
{
FileHeader.SetListNames.Add(token);
}
}
}
}
/// <summary>
/// A Flush indicates that all values have been written to the checkpoint file (via AddItemAsync), so
/// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra
/// metadata (e.g. properties and footer), and flush to disk.
/// </summary>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <returns></returns>
public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// If there's any buffered values in memory, flush them to disk first.
if (memoryBuffer.BaseStream.Position > 0)
{
await this.WriteToFileBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false);
}
Diagnostics.Assert(fileStream.Position % DefaultBlockAlignmentSize == 0, this.traceType, "Value checkpoint file is incorrectly block aligned at the end.");
// Update the Properties.
this.Properties.ValuesHandle = new BlockHandle(0, fileStream.Position);
// Write the Properties.
var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false);
// Write the Footer.
this.Footer = new FileFooter(propertiesHandle, FileVersion);
await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false);
// Finally, flush to disk.
await fileStream.FlushAsync().ConfigureAwait(false);
}
/// <summary>
/// Deserializes the metadata (footer, properties, etc.) for this checkpoint file.
/// </summary>
private async Task ReadMetadataAsync()
{
Stream filestream = null;
try
{
filestream = this.ReaderPool.AcquireStream();
var snapFileStream = filestream as FileStream;
Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream");
Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(filestream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
this.Footer =
await FileBlock.ReadBlockAsync(filestream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the checkpoint file.
if (this.Footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_ValueCheckpoint_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = this.Footer.PropertiesHandle;
this.Properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <ValueCheckpointFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
finally
{
this.ReaderPool.ReleaseStream(filestream, true);
}
}
/// <summary>
///
/// </summary>
/// <remarks>Exposed for testability.</remarks>
internal static async Task PopulateMetadataAsync(MetadataTable metadataTable, Stream stream, string traceType)
{
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(stream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
var footer = await FileBlock.ReadBlockAsync(stream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the file.
if (footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_MetadataManager_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = footer.PropertiesHandle;
var properties =
await
FileBlock.ReadBlockAsync(
stream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <MetadataManagerFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
// Read disk metadata into memory.
await ReadDiskMetadataAsync(metadataTable.Table, stream, properties, traceType).ConfigureAwait(false);
metadataTable.CheckpointLSN = properties.CheckpointLSN;
}
internal static async Task ValidateAsync(string path, string traceType)
{
// MCoskun: Open of metadata file's IO priority is not set.
// Reason: USed during CompleteCheckpoint which can block copy and backup.
using (var stream = FabricFile.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous))
{
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(stream.Length - FileFooter.SerializedSize - sizeof(ulong), FileFooter.SerializedSize);
var footer = await FileBlock.ReadBlockAsync(stream, footerHandle, (sectionReader, sectionHandle) => FileFooter.Read(sectionReader, sectionHandle)).ConfigureAwait(false);
// Verify we know how to deserialize this version of the file.
if (footer.Version != FileVersion)
{
throw new InvalidDataException(SR.Error_MetadataManager_Deserialized);
}
// Read and validate the Properties section.
var propertiesHandle = footer.PropertiesHandle;
var properties = await FileBlock.ReadBlockAsync(
stream,
propertiesHandle,
(sectionReader, sectionHandle) => FilePropertySection.Read <MetadataManagerFileProperties>(sectionReader, sectionHandle)).ConfigureAwait(false);
}
}
