/// <summary>
/// Deserialzies the FileMetadata from stream.
/// </summary>
/// <param name="reader"></param>
/// <param name="tracer"></param>
/// <returns></returns>
/// <remarks>
/// Name || Size
///
/// TotalNumberOfEntries 8
/// NumberOfValidEntries 8
/// NumberOfDeletedEntries 8
/// TimeStamp 8
///
/// FileId 4
/// CanBeDeleted 1
/// RESERVED 3
///
/// FileName N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// </remarks>
public static FileMetadata Read(InMemoryBinaryReader reader, string tracer)
{
Utility.Assert(reader.IsAligned(), "must be aligned");
var totalNumberOfEntries = reader.ReadInt64();
var numberOfValidEntries = reader.ReadInt64();
var numberOfDeletedEntries = reader.ReadInt64();
var timeStamp = reader.ReadInt64();
var fileId = reader.ReadUInt32();
reader.ReadPaddingUntilAligned(true);
var fileName = reader.ReadString();
reader.ReadPaddingUntilAligned(false);
Utility.Assert(reader.IsAligned(), "must be aligned");
return(new FileMetadata(tracer, fileId,
fileName, totalNumberOfEntries,
numberOfValidEntries, timeStamp,
numberOfDeletedEntries, false,
FileMetadata.InvalidTimeStamp,
FileMetadata.InvalidTimeStamp));
}
/// <summary>
/// Read key from file for merge.
/// </summary>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// KeySize int 4
/// Kind byte 1
/// RESERVED 3
/// VersionSequenceNumber long 8
///
/// (DeletedVersion)
/// TimeStamp long 8
///
/// (Inserted || Updated)
/// Offset long 8
/// ValueChecksum ulong 8
/// ValueSize int 4
/// RESERVED 4
///
/// Key TKey N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Key size supported is int.MaxValue in bytes.
/// </remarks>
public KeyData <TKey, TValue> ReadKey <TKey, TValue>(InMemoryBinaryReader memoryBuffer, IStateSerializer <TKey> keySerializer)
{
memoryBuffer.ThrowIfNotAligned();
// This mirrors WriteKey().
var keySize = memoryBuffer.ReadInt32();
var kind = (RecordKind)memoryBuffer.ReadByte();
memoryBuffer.ReadPaddingUntilAligned(true);
var lsn = memoryBuffer.ReadInt64();
long valueOffset = 0;
var valueSize = 0;
ulong valueChecksum = 0;
long TimeStamp = 0;
if (kind == RecordKind.DeletedVersion)
{
TimeStamp = memoryBuffer.ReadInt64();
}
else
{
valueOffset = memoryBuffer.ReadInt64();
valueChecksum = memoryBuffer.ReadUInt64();
valueSize = memoryBuffer.ReadInt32();
memoryBuffer.ReadPaddingUntilAligned(true);
}
// Protection in case the user's key serializer doesn't leave the stream at the correct end point.
var keyPosition = memoryBuffer.BaseStream.Position;
var key = keySerializer.Read(memoryBuffer);
memoryBuffer.BaseStream.Position = keyPosition + keySize;
memoryBuffer.ReadPaddingUntilAligned(false);
TVersionedItem <TValue> value = null;
switch (kind)
{
case RecordKind.DeletedVersion:
value = new TDeletedItem <TValue>(lsn, this.FileId);
break;
case RecordKind.InsertedVersion:
value = new TInsertedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
case RecordKind.UpdatedVersion:
value = new TUpdatedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
default:
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpointFile_RecordKind, (byte)kind));
}
return(new KeyData <TKey, TValue>(key, value, TimeStamp));
}
public static PropertyChunkMetadata Read(InMemoryBinaryReader reader)
{
reader.ThrowIfNotAligned();
var currentBlockSize = reader.ReadInt32();
reader.ReadPaddingUntilAligned(true);
reader.ThrowIfNotAligned();
return(new PropertyChunkMetadata(currentBlockSize));
}
/// <summary>
/// Read all file metadata from the metadata file.
/// </summary>
/// <param name="metadataTable">The metadata table.</param>
/// <param name="filestream">The file stream to read from.</param>
/// <param name="properties">The metadata manager file properties.</param>
/// <param name="traceType">Tracing information.</param>
/// <returns></returns>
private static async Task <int> ReadDiskMetadataAsync(
Dictionary <uint, FileMetadata> metadataTable, Stream filestream, MetadataManagerFileProperties properties, string traceType)
{
var startOffset = properties.MetadataHandle.Offset;
var endOffset = properties.MetadataHandle.EndOffset;
var metadataCount = 0;
var fileId = 0;
// No metadata to read (there are no metadata chunks).
if (startOffset + sizeof(int) >= endOffset)
{
return(fileId);
}
filestream.Position = startOffset;
using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
using (var metadataReader = new InMemoryBinaryReader(metadataStream))
{
// Read the first key chunk size into memory.
metadataStream.SetLength(64 * 1024);
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, PropertyChunkMetadata.Size).ConfigureAwait(false);
var propertyChunkMetadata = PropertyChunkMetadata.Read(metadataReader);
var chunkSize = propertyChunkMetadata.BlockSize;
filestream.Position -= PropertyChunkMetadata.Size;
while (filestream.Position + chunkSize + sizeof(ulong) <= endOffset)
{
// Consistency checks.
if (chunkSize < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_NegativeSize_OneArgs, chunkSize));
}
// Read the entire chunk (plus the checksum and next chunk size) into memory.
metadataStream.SetLength(chunkSize + sizeof(ulong) + sizeof(int));
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, chunkSize + sizeof(ulong) + sizeof(int)).ConfigureAwait(false);
// Read the checksum.
metadataStream.Position = chunkSize;
var checksum = metadataReader.ReadUInt64();
// Re-compute the checksum.
var expectedChecksum = CRC64.ToCRC64(metadataStream.GetBuffer(), 0, chunkSize);
if (checksum != expectedChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
}
// Deserialize the value into memory.
metadataStream.Position = sizeof(int);
metadataReader.ReadPaddingUntilAligned(true);
while (metadataStream.Position < chunkSize)
{
var fileMetadata = FileMetadata.Read(metadataReader, traceType);
if (metadataTable.ContainsKey(fileMetadata.FileId))
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_DuplicateFileId_Found_OneArgs, fileMetadata.FileId));
}
metadataTable.Add(fileMetadata.FileId, fileMetadata);
metadataCount++;
}
// Read the next chunk size.
chunkSize = BitConverter.ToInt32(metadataStream.GetBuffer(), chunkSize + sizeof(ulong));
filestream.Position -= sizeof(int);
}
// Consistency checks.
if (filestream.Position != endOffset)
{
throw new InvalidDataException(SR.Error_Metadata_Corrupt_IncorrectSize);
}
if (metadataCount != properties.FileCount)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_FileCountMismatch_TwoArgs, metadataCount, properties.FileCount));
}
return(fileId);
}
}