/// <summary>
/// Read key from file for merge.
/// </summary>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// KeySize int 4
/// Kind byte 1
/// RESERVED 3
/// VersionSequenceNumber long 8
///
/// (DeletedVersion)
/// TimeStamp long 8
///
/// (Inserted || Updated)
/// Offset long 8
/// ValueChecksum ulong 8
/// ValueSize int 4
/// RESERVED 4
///
/// Key TKey N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Key size supported is int.MaxValue in bytes.
/// </remarks>
public KeyData <TKey, TValue> ReadKey <TKey, TValue>(InMemoryBinaryReader memoryBuffer, IStateSerializer <TKey> keySerializer)
{
memoryBuffer.ThrowIfNotAligned();
// This mirrors WriteKey().
var keySize = memoryBuffer.ReadInt32();
var kind = (RecordKind)memoryBuffer.ReadByte();
memoryBuffer.ReadPaddingUntilAligned(true);
var lsn = memoryBuffer.ReadInt64();
long valueOffset = 0;
var valueSize = 0;
ulong valueChecksum = 0;
long TimeStamp = 0;
if (kind == RecordKind.DeletedVersion)
{
TimeStamp = memoryBuffer.ReadInt64();
}
else
{
valueOffset = memoryBuffer.ReadInt64();
valueChecksum = memoryBuffer.ReadUInt64();
valueSize = memoryBuffer.ReadInt32();
memoryBuffer.ReadPaddingUntilAligned(true);
}
// Protection in case the user's key serializer doesn't leave the stream at the correct end point.
var keyPosition = memoryBuffer.BaseStream.Position;
var key = keySerializer.Read(memoryBuffer);
memoryBuffer.BaseStream.Position = keyPosition + keySize;
memoryBuffer.ReadPaddingUntilAligned(false);
TVersionedItem <TValue> value = null;
switch (kind)
{
case RecordKind.DeletedVersion:
value = new TDeletedItem <TValue>(lsn, this.FileId);
break;
case RecordKind.InsertedVersion:
value = new TInsertedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
case RecordKind.UpdatedVersion:
value = new TUpdatedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
default:
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpointFile_RecordKind, (byte)kind));
}
return(new KeyData <TKey, TValue>(key, value, TimeStamp));
}
public static async Task <IEnumerable <IListElement <T> > > ReadAsync(Stream stream, IStateSerializer <T> stateSerializer, string traceType)
{
var listElementsCountSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
var listElementsBytesSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
var listElementsCount = await SerializationHelper.ReadIntAsync(listElementsCountSegment, stream).ConfigureAwait(false);
if (listElementsCount < 0)
{
throw new InvalidDataException(string.Format("Unexpected listElementsCount: {0}", listElementsCount));
}
var listElementsBytes = await SerializationHelper.ReadIntAsync(listElementsBytesSegment, stream).ConfigureAwait(false);
if (listElementsBytes < 0)
{
throw new InvalidDataException(string.Format("Unexpected listElementsBytes: {0}", listElementsBytes));
}
using (var reader = new InMemoryBinaryReader(new MemoryStream()))
{
reader.BaseStream.SetLength(listElementsBytes + sizeof(ulong));
await
SerializationHelper.ReadBytesAsync(new ArraySegment <byte>(reader.BaseStream.GetBuffer()), listElementsBytes + sizeof(ulong), stream)
.ConfigureAwait(false);
var listElements = new IListElement <T> [listElementsCount];
for (var i = 0; i < listElementsCount; i++)
{
var id = reader.ReadInt64();
var value = stateSerializer.Read(reader); // if this tries to read beyond the end of the stream, listElementsBytes was incorrect (too small)
listElements[i] = DataStore <T> .CreateQueueListElement(id, value, traceType, ListElementState.EnqueueApplied);
}
var readCRC = reader.ReadUInt64();
var calcCRC =
CRC64.ToCRC64(new[] { listElementsCountSegment, listElementsBytesSegment, new ArraySegment <byte>(reader.BaseStream.GetBuffer(), 0, listElementsBytes), });
if (readCRC != calcCRC)
{
throw new InvalidDataException(string.Format("CRC mismatch. Read: {0} Calculated: {1}", readCRC, calcCRC));
}
return(listElements);
}
}
/// <summary>
/// Read all file metadata from the metadata file.
/// </summary>
/// <param name="metadataTable">The metadata table.</param>
/// <param name="filestream">The file stream to read from.</param>
/// <param name="properties">The metadata manager file properties.</param>
/// <param name="traceType">Tracing information.</param>
/// <returns></returns>
private static async Task <int> ReadDiskMetadataAsync(
Dictionary <uint, FileMetadata> metadataTable, Stream filestream, MetadataManagerFileProperties properties, string traceType)
{
var startOffset = properties.MetadataHandle.Offset;
var endOffset = properties.MetadataHandle.EndOffset;
var metadataCount = 0;
var fileId = 0;
// No metadata to read (there are no metadata chunks).
if (startOffset + sizeof(int) >= endOffset)
{
return(fileId);
}
filestream.Position = startOffset;
using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
using (var metadataReader = new InMemoryBinaryReader(metadataStream))
{
// Read the first key chunk size into memory.
metadataStream.SetLength(64 * 1024);
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, PropertyChunkMetadata.Size).ConfigureAwait(false);
var propertyChunkMetadata = PropertyChunkMetadata.Read(metadataReader);
var chunkSize = propertyChunkMetadata.BlockSize;
filestream.Position -= PropertyChunkMetadata.Size;
while (filestream.Position + chunkSize + sizeof(ulong) <= endOffset)
{
// Consistency checks.
if (chunkSize < 0)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_NegativeSize_OneArgs, chunkSize));
}
// Read the entire chunk (plus the checksum and next chunk size) into memory.
metadataStream.SetLength(chunkSize + sizeof(ulong) + sizeof(int));
await filestream.ReadAsync(metadataStream.GetBuffer(), 0, chunkSize + sizeof(ulong) + sizeof(int)).ConfigureAwait(false);
// Read the checksum.
metadataStream.Position = chunkSize;
var checksum = metadataReader.ReadUInt64();
// Re-compute the checksum.
var expectedChecksum = CRC64.ToCRC64(metadataStream.GetBuffer(), 0, chunkSize);
if (checksum != expectedChecksum)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
}
// Deserialize the value into memory.
metadataStream.Position = sizeof(int);
metadataReader.ReadPaddingUntilAligned(true);
while (metadataStream.Position < chunkSize)
{
var fileMetadata = FileMetadata.Read(metadataReader, traceType);
if (metadataTable.ContainsKey(fileMetadata.FileId))
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_DuplicateFileId_Found_OneArgs, fileMetadata.FileId));
}
metadataTable.Add(fileMetadata.FileId, fileMetadata);
metadataCount++;
}
// Read the next chunk size.
chunkSize = BitConverter.ToInt32(metadataStream.GetBuffer(), chunkSize + sizeof(ulong));
filestream.Position -= sizeof(int);
}
// Consistency checks.
if (filestream.Position != endOffset)
{
throw new InvalidDataException(SR.Error_Metadata_Corrupt_IncorrectSize);
}
if (metadataCount != properties.FileCount)
{
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_FileCountMismatch_TwoArgs, metadataCount, properties.FileCount));
}
return(fileId);
}
}
