/// <summary>
/// Deserialzies the FileMetadata from stream.
/// </summary>
/// <param name="reader"></param>
/// <param name="tracer"></param>
/// <returns></returns>
/// <remarks>
/// Name || Size
///
/// TotalNumberOfEntries 8
/// NumberOfValidEntries 8
/// NumberOfDeletedEntries 8
/// TimeStamp 8
///
/// FileId 4
/// CanBeDeleted 1
/// RESERVED 3
///
/// FileName N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// </remarks>
public static FileMetadata Read(InMemoryBinaryReader reader, string tracer)
{
Utility.Assert(reader.IsAligned(), "must be aligned");
var totalNumberOfEntries = reader.ReadInt64();
var numberOfValidEntries = reader.ReadInt64();
var numberOfDeletedEntries = reader.ReadInt64();
var timeStamp = reader.ReadInt64();
var fileId = reader.ReadUInt32();
reader.ReadPaddingUntilAligned(true);
var fileName = reader.ReadString();
reader.ReadPaddingUntilAligned(false);
Utility.Assert(reader.IsAligned(), "must be aligned");
return(new FileMetadata(tracer, fileId,
fileName, totalNumberOfEntries,
numberOfValidEntries, timeStamp,
numberOfDeletedEntries, false,
FileMetadata.InvalidTimeStamp,
FileMetadata.InvalidTimeStamp));
}
/// <summary>
/// Read key from file for merge.
/// </summary>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// KeySize int 4
/// Kind byte 1
/// RESERVED 3
/// VersionSequenceNumber long 8
///
/// (DeletedVersion)
/// TimeStamp long 8
///
/// (Inserted || Updated)
/// Offset long 8
/// ValueChecksum ulong 8
/// ValueSize int 4
/// RESERVED 4
///
/// Key TKey N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Key size supported is int.MaxValue in bytes.
/// </remarks>
public KeyData <TKey, TValue> ReadKey <TKey, TValue>(InMemoryBinaryReader memoryBuffer, IStateSerializer <TKey> keySerializer)
{
memoryBuffer.ThrowIfNotAligned();
// This mirrors WriteKey().
var keySize = memoryBuffer.ReadInt32();
var kind = (RecordKind)memoryBuffer.ReadByte();
memoryBuffer.ReadPaddingUntilAligned(true);
var lsn = memoryBuffer.ReadInt64();
long valueOffset = 0;
var valueSize = 0;
ulong valueChecksum = 0;
long TimeStamp = 0;
if (kind == RecordKind.DeletedVersion)
{
TimeStamp = memoryBuffer.ReadInt64();
}
else
{
valueOffset = memoryBuffer.ReadInt64();
valueChecksum = memoryBuffer.ReadUInt64();
valueSize = memoryBuffer.ReadInt32();
memoryBuffer.ReadPaddingUntilAligned(true);
}
// Protection in case the user's key serializer doesn't leave the stream at the correct end point.
var keyPosition = memoryBuffer.BaseStream.Position;
var key = keySerializer.Read(memoryBuffer);
memoryBuffer.BaseStream.Position = keyPosition + keySize;
memoryBuffer.ReadPaddingUntilAligned(false);
TVersionedItem <TValue> value = null;
switch (kind)
{
case RecordKind.DeletedVersion:
value = new TDeletedItem <TValue>(lsn, this.FileId);
break;
case RecordKind.InsertedVersion:
value = new TInsertedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
case RecordKind.UpdatedVersion:
value = new TUpdatedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum);
break;
default:
throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpointFile_RecordKind, (byte)kind));
}
return(new KeyData <TKey, TValue>(key, value, TimeStamp));
}
public static async Task <IEnumerable <IListElement <T> > > ReadAsync(Stream stream, IStateSerializer <T> stateSerializer, string traceType)
{
var listElementsCountSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
var listElementsBytesSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
var listElementsCount = await SerializationHelper.ReadIntAsync(listElementsCountSegment, stream).ConfigureAwait(false);
if (listElementsCount < 0)
{
throw new InvalidDataException(string.Format("Unexpected listElementsCount: {0}", listElementsCount));
}
var listElementsBytes = await SerializationHelper.ReadIntAsync(listElementsBytesSegment, stream).ConfigureAwait(false);
if (listElementsBytes < 0)
{
throw new InvalidDataException(string.Format("Unexpected listElementsBytes: {0}", listElementsBytes));
}
using (var reader = new InMemoryBinaryReader(new MemoryStream()))
{
reader.BaseStream.SetLength(listElementsBytes + sizeof(ulong));
await
SerializationHelper.ReadBytesAsync(new ArraySegment <byte>(reader.BaseStream.GetBuffer()), listElementsBytes + sizeof(ulong), stream)
.ConfigureAwait(false);
var listElements = new IListElement <T> [listElementsCount];
for (var i = 0; i < listElementsCount; i++)
{
var id = reader.ReadInt64();
var value = stateSerializer.Read(reader); // if this tries to read beyond the end of the stream, listElementsBytes was incorrect (too small)
listElements[i] = DataStore <T> .CreateQueueListElement(id, value, traceType, ListElementState.EnqueueApplied);
}
var readCRC = reader.ReadUInt64();
var calcCRC =
CRC64.ToCRC64(new[] { listElementsCountSegment, listElementsBytesSegment, new ArraySegment <byte>(reader.BaseStream.GetBuffer(), 0, listElementsBytes), });
if (readCRC != calcCRC)
{
throw new InvalidDataException(string.Format("CRC mismatch. Read: {0} Calculated: {1}", readCRC, calcCRC));
}
return(listElements);
}
}