/// <summary>
/// Serializes the FileMetadata into an in-memory binary writer.
/// </summary>
/// <param name="writer"></param>
/// <remarks>
/// This serialization is 8 byte aligned.
///
/// Name || Size
///
/// TotalNumberOfEntries 8
/// NumberOfValidEntries 8
/// NumberOfDeletedEntries 8
/// TimeStamp 8
///
/// FileId 4
/// CanBeDeleted 1
/// RESERVED 3
///
/// FileName N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// </remarks>
public void Write(InMemoryBinaryWriter writer)
{
Utility.Assert(writer.IsAligned(), "must be aligned");
// Serialize the metadata.
writer.Write(this.TotalNumberOfEntries);
writer.Write(this.NumberOfValidEntries);
writer.Write(this.NumberOfDeletedEntries);
writer.Write(this.TimeStamp);
// With RESERVE
writer.Write(this.FileId);
writer.WritePaddingUntilAligned();
// With PADDING
writer.Write(this.FileName);
writer.WritePaddingUntilAligned();
#if !DotNetCoreClr
FabricEvents.Events.CheckpointFileMetadata(this.tracer, DifferentialStoreConstants.FileMetadata_Write, FileName, FileId,
TotalNumberOfEntries, NumberOfDeletedEntries, NumberOfInvalidEntries);
#endif
Utility.Assert(writer.IsAligned(), "must be aligned");
}
public BlockAlignedWriter(
FileStream valueFileStream, FileStream keyFileStream, ValueCheckpointFile valueCheckpointFile, KeyCheckpointFile keyCheckpointFile,
InMemoryBinaryWriter valueBuffer, InMemoryBinaryWriter keyBuffer, IStateSerializer <TValue> valueSerializer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.valueCheckpointFile = valueCheckpointFile;
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.valueFileStream = valueFileStream;
this.keyFileStream = keyFileStream;
this.valueBuffer = valueBuffer;
this.keyBuffer = keyBuffer;
this.valueSerializer = valueSerializer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.keyBlockAlignedWriter = new KeyBlockAlignedWriter <TKey, TValue>(
keyFileStream,
keyCheckpointFile,
keyBuffer,
keySerializer,
timeStamp,
traceType);
this.valueBlockAlignedWriter = new ValueBlockAlignedWriter <TKey, TValue>(
valueFileStream,
valueCheckpointFile,
valueBuffer,
valueSerializer,
traceType);
}
public CheckpointFrame(IStateSerializer <T> valueSerializer)
{
this.valueSerializer = valueSerializer;
this.writer = new InMemoryBinaryWriter(new MemoryStream());
this.ListElementsCount = 0;
this.listElementsBytes = 0;
}
private void WriteValue <TValue>(InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, byte[] value)
{
// Deleted items don't have values. Only serialize valid items.
if (item.Kind != RecordKind.DeletedVersion)
{
// WriteItemAsync valueSerializer followed by checksum.
// Serialize the value.
var valueStartPosition = memoryBuffer.BaseStream.Position;
memoryBuffer.Write(value);
var valueEndPosition = memoryBuffer.BaseStream.Position;
Diagnostics.Assert(
valueEndPosition >= valueStartPosition,
DifferentialStoreConstants.TraceType,
"User's value IStateSerializer moved the stream position backwards unexpectedly!");
// Write the checksum of just that value's bytes.
var valueSize = checked ((int)(valueEndPosition - valueStartPosition));
var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize);
// Update the in-memory offset and size for this item.
item.Offset = basePosition + valueStartPosition;
item.ValueSize = valueSize;
item.ValueChecksum = checksum;
// Update checkpoint file in-memory metadata.
this.Properties.ValueCount++;
}
// Update the in-memory metadata about which file this key-value exists in on disk.
item.FileId = this.FileId;
}
/// <summary>
/// Add a key to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs.
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="memoryBuffer"></param>
/// <param name="item"></param>
/// <param name="keySerializer"></param>
/// <param name="timeStamp"></param>
/// <param name="traceType"></param>
/// <returns></returns>
public void WriteItem <TKey, TValue>(
InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp,
string traceType)
{
// Write the key into the memory buffer.
this.WriteKey(memoryBuffer, item, keySerializer, timeStamp);
}
public void Dispose()
{
if (this.tempKeyBuffer != null)
{
this.tempKeyBuffer.Dispose();
this.tempKeyBuffer = null;
}
}
public async Task WriteAsync()
{
var filePath = Path.Combine(this.directory, this.FileName);
using (var stream = FabricFile.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan))
{
var versionArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(FileVersion));
if (this.CurrentCheckpoint != null)
{
ArraySegment <byte> currentCheckpointFileNameArraySegment;
using (var writer = new InMemoryBinaryWriter(new MemoryStream()))
{
writer.Write(this.CurrentCheckpoint.FileName);
currentCheckpointFileNameArraySegment = new ArraySegment <byte>(writer.BaseStream.GetBuffer(), 0, (int)writer.BaseStream.Position);
}
var currentCheckpointFileNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(currentCheckpointFileNameArraySegment.Count));
var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointFileNameLengthArraySegment, currentCheckpointFileNameArraySegment });
var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));
await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointFileNameLengthArraySegment.Array,
currentCheckpointFileNameLengthArraySegment.Offset,
currentCheckpointFileNameLengthArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointFileNameArraySegment.Array,
currentCheckpointFileNameArraySegment.Offset,
currentCheckpointFileNameArraySegment.Count).ConfigureAwait(false);
await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
}
else
{
var currentCheckpointNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(0));
var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointNameLengthArraySegment });
var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));
await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointNameLengthArraySegment.Array,
currentCheckpointNameLengthArraySegment.Offset,
currentCheckpointNameLengthArraySegment.Count).ConfigureAwait(false);
await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
}
}
}
public void Write(InMemoryBinaryWriter writer)
{
writer.AssertIfNotAligned();
writer.Write(this.BlockSize);
writer.WritePaddingUntilAligned();
writer.AssertIfNotAligned();
}
public async Task WriteToFileBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// The values are each individually checksummed, so we don't need to separately checksum the block being written.
// Write to disk.
await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false);
// Reset the memory buffer to be able to re-use it.
memoryBuffer.BaseStream.Position = 0;
}
internal async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
//// Checksum the chunk.
//ulong checksum = memoryBuffer.GetChecksum();
//memoryBuffer.Write(checksum);
// Write to disk.
await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false);
// Reset the memory buffer.
memoryBuffer.BaseStream.Position = 0;
}
public ValueBlockAlignedWriter(
FileStream valueFileStream, ValueCheckpointFile valueCheckpointFile, InMemoryBinaryWriter valueBuffer, IStateSerializer <TValue> valueSerializer,
string traceType)
{
this.valueCheckpointFile = valueCheckpointFile;
this.valueFileStream = valueFileStream;
this.valueBuffer = valueBuffer;
this.valueSerializer = valueSerializer;
this.traceType = traceType;
this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
}
public static void WriteBadFood(InMemoryBinaryWriter binaryWriter, int size)
{
var quotient = size / BadFoodLength;
for (var i = 0; i < quotient; i++)
{
binaryWriter.Write(BadFood);
}
var remainder = size % BadFoodLength;
binaryWriter.Write(BadFood, 0, remainder);
}
public KeyBlockAlignedWriter(
FileStream keyFileStream, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter keyBuffer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.keyFileStream = keyFileStream;
this.keyBuffer = keyBuffer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
// Most key records are lesser than 4k, when they get beyond 4k, do not shrink the buffer.
this.tempKeyBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 4 * 1024));
}
/// <summary>
///
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="memoryBuffer"></param>
/// <param name="item"></param>
/// <param name="keySerializer"></param>
/// <param name="timeStamp"></param>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// KeySize int 4
/// Kind byte 1
/// RESERVED 3
/// VersionSequenceNumber long 8
///
/// (DeletedVersion)
/// timeStamp long 8
///
/// (Inserted || Updated)
/// Offset long 8
/// ValueChecksum ulong 8
/// ValueSize int 4
/// RESERVED 4
///
/// Key TKey N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Key size supported is int.MaxValue in bytes.
/// </remarks>
private void WriteKey <TKey, TValue>(
InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp)
{
Utility.Assert(memoryBuffer.IsAligned(), "must be aligned");
var recordPosition = memoryBuffer.BaseStream.Position;
memoryBuffer.BaseStream.Position += sizeof(int);
memoryBuffer.Write((byte)item.Value.Kind); // RecordKind
memoryBuffer.WritePaddingUntilAligned(); // RESERVED
memoryBuffer.Write(item.Value.VersionSequenceNumber); // LSN
if (item.Value.Kind == RecordKind.DeletedVersion)
{
memoryBuffer.Write(timeStamp);
}
else
{
// Deleted items don't have a value. We only serialize value properties for non-deleted items.
memoryBuffer.Write(item.Value.Offset); // value offset
memoryBuffer.Write(item.Value.ValueChecksum); // value checksum
memoryBuffer.Write(item.Value.ValueSize); // value size
memoryBuffer.WritePaddingUntilAligned(); // RESERVED
}
var keyPosition = memoryBuffer.BaseStream.Position;
keySerializer.Write(item.Key, memoryBuffer);
var keyEndPosition = memoryBuffer.BaseStream.Position;
Diagnostics.Assert(
keyEndPosition >= keyPosition,
DifferentialStoreConstants.TraceType,
"User's key IStateSerializer moved the stream position backwards unexpectedly!");
memoryBuffer.BaseStream.Position = recordPosition;
memoryBuffer.Write(checked ((int)(keyEndPosition - keyPosition)));
memoryBuffer.BaseStream.Position = keyEndPosition;
memoryBuffer.WritePaddingUntilAligned();
Utility.Assert(memoryBuffer.IsAligned(), "must be aligned");
// Update in-memory metadata.
this.Properties.KeyCount++;
}
/// <summary>
///
/// </summary>
/// <remarks>On open, metadata table needs to be created.</remarks>
private static async Task <MetadataManagerFileProperties> WriteDiskMetadataAsync(Dictionary <uint, FileMetadata> metadataTable, Stream outputstream)
{
var properties = new MetadataManagerFileProperties();
var startOfChunk = true;
using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
using (var metadataWriter = new InMemoryBinaryWriter(metadataStream))
{
// Buffer metadata into memory, and periodically flush chunks to disk.
foreach (var metadata in metadataTable.Values)
{
if (startOfChunk)
{
// Leave space for an int 'size' at the front of the memory stream.
// Leave space for size and RESERVED
metadataStream.Position += PropertyChunkMetadata.Size;
startOfChunk = false;
}
metadata.Write(metadataWriter);
properties.FileCount++;
// Flush the memory buffer periodically to disk.
if (metadataStream.Position >= MemoryBufferFlushSize)
{
await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false);
startOfChunk = true;
}
}
// If there's any remaining buffered metadata in memory, flush them to disk.
if (metadataStream.Position > PropertyChunkMetadata.Size)
{
await FlushMemoryBufferAsync(outputstream, metadataWriter).ConfigureAwait(false);
}
}
// Update properties.
properties.MetadataHandle = new BlockHandle(offset: 0, size: outputstream.Position);
return(properties);
}
/// <summary>
/// A Flush indicates that all keys have been written to the checkpoint file (via AddItemAsync), so
/// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra
/// metadata (e.g. properties and footer), and flush to disk.
/// </summary>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <returns></returns>
public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// If there's any buffered keys in memory, flush them to disk first.
if (memoryBuffer.BaseStream.Position > 0)
{
await this.FlushMemoryBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false);
}
// Update the Properties.
this.Properties.KeysHandle = new BlockHandle(0, fileStream.Position);
// Write the Properties.
var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false);
// Write the Footer.
this.Footer = new FileFooter(propertiesHandle, FileVersion);
await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false);
// Finally, flush to disk.
await fileStream.FlushAsync().ConfigureAwait(false);
}
/// <summary>
/// A Flush indicates that all values have been written to the checkpoint file (via AddItemAsync), so
/// the checkpoint file can finish flushing any remaining in-memory buffered data, write any extra
/// metadata (e.g. properties and footer), and flush to disk.
/// </summary>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <returns></returns>
public async Task FlushAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// If there's any buffered values in memory, flush them to disk first.
if (memoryBuffer.BaseStream.Position > 0)
{
await this.WriteToFileBufferAsync(fileStream, memoryBuffer).ConfigureAwait(false);
}
Diagnostics.Assert(fileStream.Position % DefaultBlockAlignmentSize == 0, this.traceType, "Value checkpoint file is incorrectly block aligned at the end.");
// Update the Properties.
this.Properties.ValuesHandle = new BlockHandle(0, fileStream.Position);
// Write the Properties.
var propertiesHandle = await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Properties.Write(sectionWriter)).ConfigureAwait(false);
// Write the Footer.
this.Footer = new FileFooter(propertiesHandle, FileVersion);
await FileBlock.WriteBlockAsync(fileStream, (sectionWriter) => this.Footer.Write(sectionWriter)).ConfigureAwait(false);
// Finally, flush to disk.
await fileStream.FlushAsync().ConfigureAwait(false);
}
private static async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// Write the chunk length at the beginning of the stream.
var chunkSize = checked ((int)memoryBuffer.BaseStream.Position);
memoryBuffer.BaseStream.Position = 0;
var propertyChunkMetadata = new PropertyChunkMetadata(chunkSize);
propertyChunkMetadata.Write(memoryBuffer);
// Move to end.
memoryBuffer.BaseStream.Position = chunkSize;
// Checksum the chunk.
var checksum = memoryBuffer.GetChecksum();
memoryBuffer.Write(checksum);
// Write to disk.
await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false);
// Reset the memory buffer.
memoryBuffer.BaseStream.Position = 0;
}
/// <summary>
/// Add a value to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs.
/// </summary>
/// <typeparam name="TValue"></typeparam>
/// <param name="fileStream"></param>
/// <param name="memoryBuffer"></param>
/// <param name="item"></param>
/// <param name="value"></param>
/// <param name="traceType"></param>
/// <returns></returns>
public void WriteItem <TValue>(Stream fileStream, InMemoryBinaryWriter memoryBuffer, TVersionedItem <TValue> item, byte[] value, string traceType)
{
// Write the value into the memory buffer.
this.WriteValue(memoryBuffer, fileStream.Position, item, value);
}
/// <summary>
/// Create a new <see cref="CheckpointFile"/> from the given file, serializing the given key-values into the file.
/// </summary>
/// <param name="fileId">File identifier.</param>
/// <param name="filename">File to create and write to.</param>
/// <param name="sortedItemData">Sorted key-value pairs to include in the table.</param>
/// <param name="keySerializer"></param>
/// <param name="valueSerializer"></param>
/// <param name="timeStamp"></param>
/// <param name="traceType">Tracing information.</param>
/// <param name="perfCounters">Store performance counters instance.</param>
/// <param name="isValueAReferenceType">Indicated if the value type is reference type.</param>
/// <returns>The new <see cref="CheckpointFile"/>.</returns>
public static async Task <CheckpointFile> CreateAsync <TKey, TValue>(
uint fileId,
string filename,
IEnumerable <KeyValuePair <TKey, TVersionedItem <TValue> > > sortedItemData,
IStateSerializer <TKey> keySerializer,
IStateSerializer <TValue> valueSerializer,
long timeStamp,
string traceType,
FabricPerformanceCounterSetInstance perfCounters,
bool isValueAReferenceType)
{
var keyFileName = filename + KeyCheckpointFile.FileExtension;
var valueFileName = filename + ValueCheckpointFile.FileExtension;
var keyFile = new KeyCheckpointFile(keyFileName, fileId, isValueAReferenceType);
var valueFile = new ValueCheckpointFile(valueFileName, fileId, traceType);
var checkpointFile = new CheckpointFile(filename, keyFile, valueFile, traceType);
var perfCounterWriter = new TStoreCheckpointRateCounterWriter(perfCounters);
// Create the key checkpoint file and memory buffer.
// MCoskun: Creation of checkpoint file's IO priority is not set.
// Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
using (var keyFileStream = FabricFile.Open(keyFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
using (var keyMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
{
// Create the value checkpoint file and memory buffer.
// MCoskun: Creation of checkpoint file's IO priority is not set.
// Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
using (var valueFileStream = FabricFile.Open(valueFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
using (var valueMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
{
var blockAlignedWriter = new BlockAlignedWriter <TKey, TValue>(
valueFileStream,
keyFileStream,
valueFile,
keyFile,
valueMemoryBuffer,
keyMemoryBuffer,
valueSerializer,
keySerializer,
timeStamp,
traceType);
perfCounterWriter.StartMeasurement();
foreach (var item in sortedItemData)
{
await blockAlignedWriter.BlockAlignedWriteItemAsync(item, null, true).ConfigureAwait(false);
}
// Flush both key and value checkpoints to disk.
await blockAlignedWriter.FlushAsync().ConfigureAwait(false);
perfCounterWriter.StopMeasurement();
}
}
long writeBytes = checkpointFile.GetFileSize();
long writeBytesPerSec = perfCounterWriter.UpdatePerformanceCounter(writeBytes);
#if !DotNetCoreClr
FabricEvents.Events.CheckpointFileWriteBytesPerSec(traceType, writeBytesPerSec);
#endif
return(checkpointFile);
}
