/// <summary> /// Deserialzies the FileMetadata from stream. /// </summary> /// <param name="reader"></param> /// <param name="tracer"></param> /// <returns></returns> /// <remarks> /// Name || Size /// /// TotalNumberOfEntries 8 /// NumberOfValidEntries 8 /// NumberOfDeletedEntries 8 /// TimeStamp 8 /// /// FileId 4 /// CanBeDeleted 1 /// RESERVED 3 /// /// FileName N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// </remarks> public static FileMetadata Read(InMemoryBinaryReader reader, string tracer) { Utility.Assert(reader.IsAligned(), "must be aligned"); var totalNumberOfEntries = reader.ReadInt64(); var numberOfValidEntries = reader.ReadInt64(); var numberOfDeletedEntries = reader.ReadInt64(); var timeStamp = reader.ReadInt64(); var fileId = reader.ReadUInt32(); reader.ReadPaddingUntilAligned(true); var fileName = reader.ReadString(); reader.ReadPaddingUntilAligned(false); Utility.Assert(reader.IsAligned(), "must be aligned"); return(new FileMetadata(tracer, fileId, fileName, totalNumberOfEntries, numberOfValidEntries, timeStamp, numberOfDeletedEntries, false, FileMetadata.InvalidTimeStamp, FileMetadata.InvalidTimeStamp)); }
/// <summary> /// Read key from file for merge. /// </summary> /// <remarks> /// The data is written is 8 bytes aligned. /// /// Name Type Size /// /// KeySize int 4 /// Kind byte 1 /// RESERVED 3 /// VersionSequenceNumber long 8 /// /// (DeletedVersion) /// TimeStamp long 8 /// /// (Inserted || Updated) /// Offset long 8 /// ValueChecksum ulong 8 /// ValueSize int 4 /// RESERVED 4 /// /// Key TKey N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// Note: Larges Key size supported is int.MaxValue in bytes. /// </remarks> public KeyData <TKey, TValue> ReadKey <TKey, TValue>(InMemoryBinaryReader memoryBuffer, IStateSerializer <TKey> keySerializer) { memoryBuffer.ThrowIfNotAligned(); // This mirrors WriteKey(). var keySize = memoryBuffer.ReadInt32(); var kind = (RecordKind)memoryBuffer.ReadByte(); memoryBuffer.ReadPaddingUntilAligned(true); var lsn = memoryBuffer.ReadInt64(); long valueOffset = 0; var valueSize = 0; ulong valueChecksum = 0; long TimeStamp = 0; if (kind == RecordKind.DeletedVersion) { TimeStamp = memoryBuffer.ReadInt64(); } else { valueOffset = memoryBuffer.ReadInt64(); valueChecksum = memoryBuffer.ReadUInt64(); valueSize = memoryBuffer.ReadInt32(); memoryBuffer.ReadPaddingUntilAligned(true); } // Protection in case the user's key serializer doesn't leave the stream at the correct end point. var keyPosition = memoryBuffer.BaseStream.Position; var key = keySerializer.Read(memoryBuffer); memoryBuffer.BaseStream.Position = keyPosition + keySize; memoryBuffer.ReadPaddingUntilAligned(false); TVersionedItem <TValue> value = null; switch (kind) { case RecordKind.DeletedVersion: value = new TDeletedItem <TValue>(lsn, this.FileId); break; case RecordKind.InsertedVersion: value = new TInsertedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum); break; case RecordKind.UpdatedVersion: value = new TUpdatedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum); break; default: throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpointFile_RecordKind, (byte)kind)); } return(new KeyData <TKey, TValue>(key, value, TimeStamp)); }
public static async Task <IEnumerable <IListElement <T> > > ReadAsync(Stream stream, IStateSerializer <T> stateSerializer, string traceType) { var listElementsCountSegment = new ArraySegment <byte>(new byte[sizeof(int)]); var listElementsBytesSegment = new ArraySegment <byte>(new byte[sizeof(int)]); var listElementsCount = await SerializationHelper.ReadIntAsync(listElementsCountSegment, stream).ConfigureAwait(false); if (listElementsCount < 0) { throw new InvalidDataException(string.Format("Unexpected listElementsCount: {0}", listElementsCount)); } var listElementsBytes = await SerializationHelper.ReadIntAsync(listElementsBytesSegment, stream).ConfigureAwait(false); if (listElementsBytes < 0) { throw new InvalidDataException(string.Format("Unexpected listElementsBytes: {0}", listElementsBytes)); } using (var reader = new InMemoryBinaryReader(new MemoryStream())) { reader.BaseStream.SetLength(listElementsBytes + sizeof(ulong)); await SerializationHelper.ReadBytesAsync(new ArraySegment <byte>(reader.BaseStream.GetBuffer()), listElementsBytes + sizeof(ulong), stream) .ConfigureAwait(false); var listElements = new IListElement <T> [listElementsCount]; for (var i = 0; i < listElementsCount; i++) { var id = reader.ReadInt64(); var value = stateSerializer.Read(reader); // if this tries to read beyond the end of the stream, listElementsBytes was incorrect (too small) listElements[i] = DataStore <T> .CreateQueueListElement(id, value, traceType, ListElementState.EnqueueApplied); } var readCRC = reader.ReadUInt64(); var calcCRC = CRC64.ToCRC64(new[] { listElementsCountSegment, listElementsBytesSegment, new ArraySegment <byte>(reader.BaseStream.GetBuffer(), 0, listElementsBytes), }); if (readCRC != calcCRC) { throw new InvalidDataException(string.Format("CRC mismatch. Read: {0} Calculated: {1}", readCRC, calcCRC)); } return(listElements); } }