public static void WriteBadFood(InMemoryBinaryWriter binaryWriter, int size) { var quotient = size / BadFoodLength; for (var i = 0; i < quotient; i++) { binaryWriter.Write(BadFood); } var remainder = size % BadFoodLength; binaryWriter.Write(BadFood, 0, remainder); }
private void WriteValue <TValue>(InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, byte[] value) { // Deleted items don't have values. Only serialize valid items. if (item.Kind != RecordKind.DeletedVersion) { // WriteItemAsync valueSerializer followed by checksum. // Serialize the value. var valueStartPosition = memoryBuffer.BaseStream.Position; memoryBuffer.Write(value); var valueEndPosition = memoryBuffer.BaseStream.Position; Diagnostics.Assert( valueEndPosition >= valueStartPosition, DifferentialStoreConstants.TraceType, "User's value IStateSerializer moved the stream position backwards unexpectedly!"); // Write the checksum of just that value's bytes. var valueSize = checked ((int)(valueEndPosition - valueStartPosition)); var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize); // Update the in-memory offset and size for this item. item.Offset = basePosition + valueStartPosition; item.ValueSize = valueSize; item.ValueChecksum = checksum; // Update checkpoint file in-memory metadata. this.Properties.ValueCount++; } // Update the in-memory metadata about which file this key-value exists in on disk. item.FileId = this.FileId; }
public async Task WriteAsync() { var filePath = Path.Combine(this.directory, this.FileName); using (var stream = FabricFile.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan)) { var versionArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(FileVersion)); if (this.CurrentCheckpoint != null) { ArraySegment <byte> currentCheckpointFileNameArraySegment; using (var writer = new InMemoryBinaryWriter(new MemoryStream())) { writer.Write(this.CurrentCheckpoint.FileName); currentCheckpointFileNameArraySegment = new ArraySegment <byte>(writer.BaseStream.GetBuffer(), 0, (int)writer.BaseStream.Position); } var currentCheckpointFileNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(currentCheckpointFileNameArraySegment.Count)); var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointFileNameLengthArraySegment, currentCheckpointFileNameArraySegment }); var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc)); await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointFileNameLengthArraySegment.Array, currentCheckpointFileNameLengthArraySegment.Offset, currentCheckpointFileNameLengthArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointFileNameArraySegment.Array, currentCheckpointFileNameArraySegment.Offset, currentCheckpointFileNameArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false); } else { var currentCheckpointNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(0)); var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointNameLengthArraySegment }); var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc)); await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointNameLengthArraySegment.Array, currentCheckpointNameLengthArraySegment.Offset, currentCheckpointNameLengthArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false); } } }
public void Write(InMemoryBinaryWriter writer) { writer.AssertIfNotAligned(); writer.Write(this.BlockSize); writer.WritePaddingUntilAligned(); writer.AssertIfNotAligned(); }
/// <summary> /// Serializes the FileMetadata into an in-memory binary writer. /// </summary> /// <param name="writer"></param> /// <remarks> /// This serialization is 8 byte aligned. /// /// Name || Size /// /// TotalNumberOfEntries 8 /// NumberOfValidEntries 8 /// NumberOfDeletedEntries 8 /// TimeStamp 8 /// /// FileId 4 /// CanBeDeleted 1 /// RESERVED 3 /// /// FileName N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// </remarks> public void Write(InMemoryBinaryWriter writer) { Utility.Assert(writer.IsAligned(), "must be aligned"); // Serialize the metadata. writer.Write(this.TotalNumberOfEntries); writer.Write(this.NumberOfValidEntries); writer.Write(this.NumberOfDeletedEntries); writer.Write(this.TimeStamp); // With RESERVE writer.Write(this.FileId); writer.WritePaddingUntilAligned(); // With PADDING writer.Write(this.FileName); writer.WritePaddingUntilAligned(); #if !DotNetCoreClr FabricEvents.Events.CheckpointFileMetadata(this.tracer, DifferentialStoreConstants.FileMetadata_Write, FileName, FileId, TotalNumberOfEntries, NumberOfDeletedEntries, NumberOfInvalidEntries); #endif Utility.Assert(writer.IsAligned(), "must be aligned"); }
/// <summary> /// /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TValue"></typeparam> /// <param name="memoryBuffer"></param> /// <param name="item"></param> /// <param name="keySerializer"></param> /// <param name="timeStamp"></param> /// <remarks> /// The data is written is 8 bytes aligned. /// /// Name Type Size /// /// KeySize int 4 /// Kind byte 1 /// RESERVED 3 /// VersionSequenceNumber long 8 /// /// (DeletedVersion) /// timeStamp long 8 /// /// (Inserted || Updated) /// Offset long 8 /// ValueChecksum ulong 8 /// ValueSize int 4 /// RESERVED 4 /// /// Key TKey N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// Note: Larges Key size supported is int.MaxValue in bytes. /// </remarks> private void WriteKey <TKey, TValue>( InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp) { Utility.Assert(memoryBuffer.IsAligned(), "must be aligned"); var recordPosition = memoryBuffer.BaseStream.Position; memoryBuffer.BaseStream.Position += sizeof(int); memoryBuffer.Write((byte)item.Value.Kind); // RecordKind memoryBuffer.WritePaddingUntilAligned(); // RESERVED memoryBuffer.Write(item.Value.VersionSequenceNumber); // LSN if (item.Value.Kind == RecordKind.DeletedVersion) { memoryBuffer.Write(timeStamp); } else { // Deleted items don't have a value. We only serialize value properties for non-deleted items. memoryBuffer.Write(item.Value.Offset); // value offset memoryBuffer.Write(item.Value.ValueChecksum); // value checksum memoryBuffer.Write(item.Value.ValueSize); // value size memoryBuffer.WritePaddingUntilAligned(); // RESERVED } var keyPosition = memoryBuffer.BaseStream.Position; keySerializer.Write(item.Key, memoryBuffer); var keyEndPosition = memoryBuffer.BaseStream.Position; Diagnostics.Assert( keyEndPosition >= keyPosition, DifferentialStoreConstants.TraceType, "User's key IStateSerializer moved the stream position backwards unexpectedly!"); memoryBuffer.BaseStream.Position = recordPosition; memoryBuffer.Write(checked ((int)(keyEndPosition - keyPosition))); memoryBuffer.BaseStream.Position = keyEndPosition; memoryBuffer.WritePaddingUntilAligned(); Utility.Assert(memoryBuffer.IsAligned(), "must be aligned"); // Update in-memory metadata. this.Properties.KeyCount++; }
private static async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { // Write the chunk length at the beginning of the stream. var chunkSize = checked ((int)memoryBuffer.BaseStream.Position); memoryBuffer.BaseStream.Position = 0; var propertyChunkMetadata = new PropertyChunkMetadata(chunkSize); propertyChunkMetadata.Write(memoryBuffer); // Move to end. memoryBuffer.BaseStream.Position = chunkSize; // Checksum the chunk. var checksum = memoryBuffer.GetChecksum(); memoryBuffer.Write(checksum); // Write to disk. await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false); // Reset the memory buffer. memoryBuffer.BaseStream.Position = 0; }