/// <summary> /// Serializes the FileMetadata into an in-memory binary writer. /// </summary> /// <param name="writer"></param> /// <remarks> /// This serialization is 8 byte aligned. /// /// Name || Size /// /// TotalNumberOfEntries 8 /// NumberOfValidEntries 8 /// NumberOfDeletedEntries 8 /// TimeStamp 8 /// /// FileId 4 /// CanBeDeleted 1 /// RESERVED 3 /// /// FileName N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// </remarks> public void Write(InMemoryBinaryWriter writer) { Utility.Assert(writer.IsAligned(), "must be aligned"); // Serialize the metadata. writer.Write(this.TotalNumberOfEntries); writer.Write(this.NumberOfValidEntries); writer.Write(this.NumberOfDeletedEntries); writer.Write(this.TimeStamp); // With RESERVE writer.Write(this.FileId); writer.WritePaddingUntilAligned(); // With PADDING writer.Write(this.FileName); writer.WritePaddingUntilAligned(); #if !DotNetCoreClr FabricEvents.Events.CheckpointFileMetadata(this.tracer, DifferentialStoreConstants.FileMetadata_Write, FileName, FileId, TotalNumberOfEntries, NumberOfDeletedEntries, NumberOfInvalidEntries); #endif Utility.Assert(writer.IsAligned(), "must be aligned"); }
public BlockAlignedWriter( FileStream valueFileStream, FileStream keyFileStream, ValueCheckpointFile valueCheckpointFile, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter valueBuffer, InMemoryBinaryWriter keyBuffer, IStateSerializer <TValue> valueSerializer, IStateSerializer <TKey> keySerializer, long timeStamp, string traceType) { this.valueCheckpointFile = valueCheckpointFile; this.keyCheckpointFile = keyCheckpointFile; this.timeStamp = timeStamp; this.valueFileStream = valueFileStream; this.keyFileStream = keyFileStream; this.valueBuffer = valueBuffer; this.keyBuffer = keyBuffer; this.valueSerializer = valueSerializer; this.keySerializer = keySerializer; this.traceType = traceType; this.keyBlockAlignedWriter = new KeyBlockAlignedWriter <TKey, TValue>( keyFileStream, keyCheckpointFile, keyBuffer, keySerializer, timeStamp, traceType); this.valueBlockAlignedWriter = new ValueBlockAlignedWriter <TKey, TValue>( valueFileStream, valueCheckpointFile, valueBuffer, valueSerializer, traceType); }
public CheckpointFrame(IStateSerializer <T> valueSerializer) { this.valueSerializer = valueSerializer; this.writer = new InMemoryBinaryWriter(new MemoryStream()); this.ListElementsCount = 0; this.listElementsBytes = 0; }
private void WriteValue <TValue>(InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, byte[] value) { // Deleted items don't have values. Only serialize valid items. if (item.Kind != RecordKind.DeletedVersion) { // WriteItemAsync valueSerializer followed by checksum. // Serialize the value. var valueStartPosition = memoryBuffer.BaseStream.Position; memoryBuffer.Write(value); var valueEndPosition = memoryBuffer.BaseStream.Position; Diagnostics.Assert( valueEndPosition >= valueStartPosition, DifferentialStoreConstants.TraceType, "User's value IStateSerializer moved the stream position backwards unexpectedly!"); // Write the checksum of just that value's bytes. var valueSize = checked ((int)(valueEndPosition - valueStartPosition)); var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize); // Update the in-memory offset and size for this item. item.Offset = basePosition + valueStartPosition; item.ValueSize = valueSize; item.ValueChecksum = checksum; // Update checkpoint file in-memory metadata. this.Properties.ValueCount++; } // Update the in-memory metadata about which file this key-value exists in on disk. item.FileId = this.FileId; }
/// <summary> /// Add a key to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs. /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TValue"></typeparam> /// <param name="memoryBuffer"></param> /// <param name="item"></param> /// <param name="keySerializer"></param> /// <param name="timeStamp"></param> /// <param name="traceType"></param> /// <returns></returns> public void WriteItem <TKey, TValue>( InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp, string traceType) { // Write the key into the memory buffer. this.WriteKey(memoryBuffer, item, keySerializer, timeStamp); }
public void Dispose() { if (this.tempKeyBuffer != null) { this.tempKeyBuffer.Dispose(); this.tempKeyBuffer = null; } }
public async Task WriteAsync() { var filePath = Path.Combine(this.directory, this.FileName); using (var stream = FabricFile.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan)) { var versionArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(FileVersion)); if (this.CurrentCheckpoint != null) { ArraySegment <byte> currentCheckpointFileNameArraySegment; using (var writer = new InMemoryBinaryWriter(new MemoryStream())) { writer.Write(this.CurrentCheckpoint.FileName); currentCheckpointFileNameArraySegment = new ArraySegment <byte>(writer.BaseStream.GetBuffer(), 0, (int)writer.BaseStream.Position); } var currentCheckpointFileNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(currentCheckpointFileNameArraySegment.Count)); var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointFileNameLengthArraySegment, currentCheckpointFileNameArraySegment }); var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc)); await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointFileNameLengthArraySegment.Array, currentCheckpointFileNameLengthArraySegment.Offset, currentCheckpointFileNameLengthArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointFileNameArraySegment.Array, currentCheckpointFileNameArraySegment.Offset, currentCheckpointFileNameArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false); } else { var currentCheckpointNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(0)); var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointNameLengthArraySegment }); var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc)); await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync( currentCheckpointNameLengthArraySegment.Array, currentCheckpointNameLengthArraySegment.Offset, currentCheckpointNameLengthArraySegment.Count).ConfigureAwait(false); await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false); } } }
public void Write(InMemoryBinaryWriter writer) { writer.AssertIfNotAligned(); writer.Write(this.BlockSize); writer.WritePaddingUntilAligned(); writer.AssertIfNotAligned(); }
public async Task WriteToFileBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { // The values are each individually checksummed, so we don't need to separately checksum the block being written. // Write to disk. await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false); // Reset the memory buffer to be able to re-use it. memoryBuffer.BaseStream.Position = 0; }
internal async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { //// Checksum the chunk. //ulong checksum = memoryBuffer.GetChecksum(); //memoryBuffer.Write(checksum); // Write to disk. await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false); // Reset the memory buffer. memoryBuffer.BaseStream.Position = 0; }
public ValueBlockAlignedWriter( FileStream valueFileStream, ValueCheckpointFile valueCheckpointFile, InMemoryBinaryWriter valueBuffer, IStateSerializer <TValue> valueSerializer, string traceType) { this.valueCheckpointFile = valueCheckpointFile; this.valueFileStream = valueFileStream; this.valueBuffer = valueBuffer; this.valueSerializer = valueSerializer; this.traceType = traceType; this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize; }
public static void WriteBadFood(InMemoryBinaryWriter binaryWriter, int size) { var quotient = size / BadFoodLength; for (var i = 0; i < quotient; i++) { binaryWriter.Write(BadFood); } var remainder = size % BadFoodLength; binaryWriter.Write(BadFood, 0, remainder); }
public KeyBlockAlignedWriter( FileStream keyFileStream, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter keyBuffer, IStateSerializer <TKey> keySerializer, long timeStamp, string traceType) { this.keyCheckpointFile = keyCheckpointFile; this.timeStamp = timeStamp; this.keyFileStream = keyFileStream; this.keyBuffer = keyBuffer; this.keySerializer = keySerializer; this.traceType = traceType; this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize; // Most key records are lesser than 4k, when they get beyond 4k, do not shrink the buffer. this.tempKeyBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 4 * 1024)); }
/// <summary> /// /// </summary> /// <typeparam name="TKey"></typeparam> /// <typeparam name="TValue"></typeparam> /// <param name="memoryBuffer"></param> /// <param name="item"></param> /// <param name="keySerializer"></param> /// <param name="timeStamp"></param> /// <remarks> /// The data is written is 8 bytes aligned. /// /// Name Type Size /// /// KeySize int 4 /// Kind byte 1 /// RESERVED 3 /// VersionSequenceNumber long 8 /// /// (DeletedVersion) /// timeStamp long 8 /// /// (Inserted || Updated) /// Offset long 8 /// ValueChecksum ulong 8 /// ValueSize int 4 /// RESERVED 4 /// /// Key TKey N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// Note: Larges Key size supported is int.MaxValue in bytes. /// </remarks> private void WriteKey <TKey, TValue>( InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp) { Utility.Assert(memoryBuffer.IsAligned(), "must be aligned"); var recordPosition = memoryBuffer.BaseStream.Position; memoryBuffer.BaseStream.Position += sizeof(int); memoryBuffer.Write((byte)item.Value.Kind); // RecordKind memoryBuffer.WritePaddingUntilAligned(); // RESERVED memoryBuffer.Write(item.Value.VersionSequenceNumber); // LSN if (item.Value.Kind == RecordKind.DeletedVersion) { memoryBuffer.Write(timeStamp); } else { // Deleted items don't have a value. We only serialize value properties for non-deleted items. memoryBuffer.Write(item.Value.Offset); // value offset memoryBuffer.Write(item.Value.ValueChecksum); // value checksum memoryBuffer.Write(item.Value.ValueSize); // value size memoryBuffer.WritePaddingUntilAligned(); // RESERVED } var keyPosition = memoryBuffer.BaseStream.Position; keySerializer.Write(item.Key, memoryBuffer); var keyEndPosition = memoryBuffer.BaseStream.Position; Diagnostics.Assert( keyEndPosition >= keyPosition, DifferentialStoreConstants.TraceType, "User's key IStateSerializer moved the stream position backwards unexpectedly!"); memoryBuffer.BaseStream.Position = recordPosition; memoryBuffer.Write(checked ((int)(keyEndPosition - keyPosition))); memoryBuffer.BaseStream.Position = keyEndPosition; memoryBuffer.WritePaddingUntilAligned(); Utility.Assert(memoryBuffer.IsAligned(), "must be aligned"); // Update in-memory metadata. this.Properties.KeyCount++; }
/// <summary> /// /// </summary> /// <remarks>On open, metadata table needs to be created.</remarks> private static async Task <MetadataManagerFileProperties> WriteDiskMetadataAsync(Dictionary <uint, FileMetadata> metadataTable, Stream outputstream) { var properties = new MetadataManagerFileProperties(); var startOfChunk = true; using (var metadataStream = new MemoryStream(capacity: 64 * 1024)) using (var metadataWriter = new InMemoryBinaryWriter(metadataStream)) { // Buffer metadata into memory, and periodically flush chunks to disk. foreach (var metadata in metadataTable.Values) { if (startOfChunk) { // Leave space for an int 'size' at the front of the memory stream. // Leave space for size and RESERVED metadataStream.Position += PropertyChunkMetadata.Size; startOfChunk = false; } metadata.Write(metadataWriter); properties.FileCount++; // Flush the memory buffer periodically to disk. if (metadataStream.Position >= MemoryBufferFlushSize) { await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false); startOfChunk = true; } } // If there's any remaining buffered metadata in memory, flush them to disk. if (metadataStream.Position > PropertyChunkMetadata.Size) { await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false); } } // Update properties. properties.MetadataHandle = new BlockHandle(offset: 0, size: outputstream.Position); return(properties); }
/// <summary> /// A Flush indicates that all keys have been written to the checkpoint file (via AddItemAsync), so /// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra /// metadata (e.g. properties and footer), and flush to disk. /// </summary> /// <param name="fileStream"></param> /// <param name="memoryBuffer"></param> /// <returns></returns> public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { // If there's any buffered keys in memory, flush them to disk first. if (memoryBuffer.BaseStream.Position > 0) { await this.FlushMemoryBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false); } // Update the Properties. this.Properties.KeysHandle = new BlockHandle(0, fileStream.Position); // Write the Properties. var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false); // Write the Footer. this.Footer = new FileFooter(propertiesHandle, FileVersion); await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false); // Finally, flush to disk. await fileStream.FlushAsync().ConfigureAwait(false); }
/// <summary> /// A Flush indicates that all values have been written to the checkpoint file (via AddItemAsync), so /// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra /// metadata (e.g. properties and footer), and flush to disk. /// </summary> /// <param name="fileStream"></param> /// <param name="memoryBuffer"></param> /// <returns></returns> public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { // If there's any buffered values in memory, flush them to disk first. if (memoryBuffer.BaseStream.Position > 0) { await this.WriteToFileBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false); } Diagnostics.Assert(fileStream.Position % DefaultBlockAlignmentSize == 0, this.traceType, "Value checkpoint file is incorrectly block aligned at the end."); // Update the Properties. this.Properties.ValuesHandle = new BlockHandle(0, fileStream.Position); // Write the Properties. var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false); // Write the Footer. this.Footer = new FileFooter(propertiesHandle, FileVersion); await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false); // Finally, flush to disk. await fileStream.FlushAsync().ConfigureAwait(false); }
private static async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer) { // Write the chunk length at the beginning of the stream. var chunkSize = checked ((int)memoryBuffer.BaseStream.Position); memoryBuffer.BaseStream.Position = 0; var propertyChunkMetadata = new PropertyChunkMetadata(chunkSize); propertyChunkMetadata.Write(memoryBuffer); // Move to end. memoryBuffer.BaseStream.Position = chunkSize; // Checksum the chunk. var checksum = memoryBuffer.GetChecksum(); memoryBuffer.Write(checksum); // Write to disk. await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false); // Reset the memory buffer. memoryBuffer.BaseStream.Position = 0; }
/// <summary> /// Add a value to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs. /// </summary> /// <typeparam name="TValue"></typeparam> /// <param name="fileStream"></param> /// <param name="memoryBuffer"></param> /// <param name="item"></param> /// <param name="value"></param> /// <param name="traceType"></param> /// <returns></returns> public void WriteItem <TValue>(Stream fileStream, InMemoryBinaryWriter memoryBuffer, TVersionedItem <TValue> item, byte[] value, string traceType) { // Write the value into the memory buffer. this.WriteValue(memoryBuffer, fileStream.Position, item, value); }
/// <summary> /// Create a new <see cref="CheckpointFile"/> from the given file, serializing the given key-values into the file. /// </summary> /// <param name="fileId">File identifier.</param> /// <param name="filename">File to create and write to.</param> /// <param name="sortedItemData">Sorted key-value pairs to include in the table.</param> /// <param name="keySerializer"></param> /// <param name="valueSerializer"></param> /// <param name="timeStamp"></param> /// <param name="traceType">Tracing information.</param> /// <param name="perfCounters">Store performance counters instance.</param> /// <param name="isValueAReferenceType">Indicated if the value type is reference type.</param> /// <returns>The new <see cref="CheckpointFile"/>.</returns> public static async Task <CheckpointFile> CreateAsync <TKey, TValue>( uint fileId, string filename, IEnumerable <KeyValuePair <TKey, TVersionedItem <TValue> > > sortedItemData, IStateSerializer <TKey> keySerializer, IStateSerializer <TValue> valueSerializer, long timeStamp, string traceType, FabricPerformanceCounterSetInstance perfCounters, bool isValueAReferenceType) { var keyFileName = filename + KeyCheckpointFile.FileExtension; var valueFileName = filename + ValueCheckpointFile.FileExtension; var keyFile = new KeyCheckpointFile(keyFileName, fileId, isValueAReferenceType); var valueFile = new ValueCheckpointFile(valueFileName, fileId, traceType); var checkpointFile = new CheckpointFile(filename, keyFile, valueFile, traceType); var perfCounterWriter = new TStoreCheckpointRateCounterWriter(perfCounters); // Create the key checkpoint file and memory buffer. // MCoskun: Creation of checkpoint file's IO priority is not set. // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations. using (var keyFileStream = FabricFile.Open(keyFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous)) using (var keyMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024))) { // Create the value checkpoint file and memory buffer. // MCoskun: Creation of checkpoint file's IO priority is not set. // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations. using (var valueFileStream = FabricFile.Open(valueFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous)) using (var valueMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024))) { var blockAlignedWriter = new BlockAlignedWriter <TKey, TValue>( valueFileStream, keyFileStream, valueFile, keyFile, valueMemoryBuffer, keyMemoryBuffer, valueSerializer, keySerializer, timeStamp, traceType); perfCounterWriter.StartMeasurement(); foreach (var item in sortedItemData) { await blockAlignedWriter.BlockAlignedWriteItemAsync(item, null, true).ConfigureAwait(false); } // Flush both key and value checkpoints to disk. await blockAlignedWriter.FlushAsync().ConfigureAwait(false); perfCounterWriter.StopMeasurement(); } } long writeBytes = checkpointFile.GetFileSize(); long writeBytesPerSec = perfCounterWriter.UpdatePerformanceCounter(writeBytes); #if !DotNetCoreClr FabricEvents.Events.CheckpointFileWriteBytesPerSec(traceType, writeBytesPerSec); #endif return(checkpointFile); }