/// <summary>
///
/// </summary>
/// <remarks>On open, metadata table needs to be created.</remarks>
private static async Task <MetadataManagerFileProperties> WriteDiskMetadataAsync(Dictionary <uint, FileMetadata> metadataTable, Stream outputstream)
{
var properties = new MetadataManagerFileProperties();
var startOfChunk = true;
using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
using (var metadataWriter = new InMemoryBinaryWriter(metadataStream))
{
// Buffer metadata into memory, and periodically flush chunks to disk.
foreach (var metadata in metadataTable.Values)
{
if (startOfChunk)
{
// Leave space for an int 'size' at the front of the memory stream.
// Leave space for size and RESERVED
metadataStream.Position += PropertyChunkMetadata.Size;
startOfChunk = false;
}
metadata.Write(metadataWriter);
properties.FileCount++;
// Flush the memory buffer periodically to disk.
if (metadataStream.Position >= MemoryBufferFlushSize)
{
await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false);
startOfChunk = true;
}
}
// If there's any remaining buffered metadata in memory, flush them to disk.
if (metadataStream.Position > PropertyChunkMetadata.Size)
{
await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false);
}
}
// Update properties.
properties.MetadataHandle = new BlockHandle(offset: 0, size: outputstream.Position);
return(properties);
}
/// <summary>
/// Read all file metadata from the metadata file.
/// </summary>
/// <param name="metadataTable">The metadata table.</param>
/// <param name="filestream">The file stream to read from.</param>
/// <param name="properties">The metadata manager file properties.</param>
/// <param name="traceType">Tracing information.</param>
/// <returns></returns>
private static async Task <int> ReadDiskMetadataAsync(
Dictionary <uint, FileMetadata> metadataTable, Stream filestream, MetadataManagerFileProperties properties, string traceType)
{
var startOffset = properties.MetadataHandle.Offset;
var endOffset = properties.MetadataHandle.EndOffset;
var metadataCount = 0;
var fileId = 0;
// No metadata to read (there are no metadata chunks).
if (startOffset + sizeof(int) >= endOffset)
{
return(fileId);
}
filestream.Position = startOffset;
using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
using (var metadataReader = new InMemoryBinaryReader(metadataStream))
{
// Read the first key chunk size into memory.
metadataStream.SetLength(64 * 1024);
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, PropertyChunkMetadata.Size).ConfigureAwait(false);
var propertyChunkMetadata = PropertyChunkMetadata.Read(metadataReader);
var chunkSize = propertyChunkMetadata.BlockSize;
filestream.Position -= PropertyChunkMetadata.Size;
while (filestream.Position + chunkSize + sizeof(ulong) <= endOffset)
{
// Consistency checks.
if (chunkSize < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_NegativeSize_OneArgs, chunkSize));
}
// Read the entire chunk (plus the checksum and next chunk size) into memory.
metadataStream.SetLength(chunkSize + sizeof(ulong) + sizeof(int));
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, chunkSize + sizeof(ulong) + sizeof(int)).ConfigureAwait(false);
// Read the checksum.
metadataStream.Position = chunkSize;
var checksum = metadataReader.ReadUInt64();
// Re-compute the checksum.
var expectedChecksum = CRC64.ToCRC64(metadataStream.GetBuffer(), 0, chunkSize);
if (checksum != expectedChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
}
// Deserialize the value into memory.
metadataStream.Position = sizeof(int);
metadataReader.ReadPaddingUntilAligned(true);
while (metadataStream.Position < chunkSize)
{
var fileMetadata = FileMetadata.Read(metadataReader, traceType);
if (metadataTable.ContainsKey(fileMetadata.FileId))
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_DuplicateFileId_Found_OneArgs, fileMetadata.FileId));
}
metadataTable.Add(fileMetadata.FileId, fileMetadata);
metadataCount++;
}
// Read the next chunk size.
chunkSize = BitConverter.ToInt32(metadataStream.GetBuffer(), chunkSize + sizeof(ulong));
filestream.Position -= sizeof(int);
}
// Consistency checks.
if (filestream.Position != endOffset)
{
throw new InvalidDataException(SR.Error_Metadata_Corrupt_IncorrectSize);
}
if (metadataCount != properties.FileCount)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_FileCountMismatch_TwoArgs, metadataCount, properties.FileCount));
}
return(fileId);
}
}
