public static void WriteBadFood(InMemoryBinaryWriter binaryWriter, int size)
{
var quotient = size / BadFoodLength;
for (var i = 0; i < quotient; i++)
{
binaryWriter.Write(BadFood);
}
var remainder = size % BadFoodLength;
binaryWriter.Write(BadFood, 0, remainder);
}
private void WriteValue <TValue>(InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, byte[] value)
{
// Deleted items don't have values. Only serialize valid items.
if (item.Kind != RecordKind.DeletedVersion)
{
// WriteItemAsync valueSerializer followed by checksum.
// Serialize the value.
var valueStartPosition = memoryBuffer.BaseStream.Position;
memoryBuffer.Write(value);
var valueEndPosition = memoryBuffer.BaseStream.Position;
Diagnostics.Assert(
valueEndPosition >= valueStartPosition,
DifferentialStoreConstants.TraceType,
"User's value IStateSerializer moved the stream position backwards unexpectedly!");
// Write the checksum of just that value's bytes.
var valueSize = checked ((int)(valueEndPosition - valueStartPosition));
var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize);
// Update the in-memory offset and size for this item.
item.Offset = basePosition + valueStartPosition;
item.ValueSize = valueSize;
item.ValueChecksum = checksum;
// Update checkpoint file in-memory metadata.
this.Properties.ValueCount++;
}
// Update the in-memory metadata about which file this key-value exists in on disk.
item.FileId = this.FileId;
}
public async Task WriteAsync()
{
var filePath = Path.Combine(this.directory, this.FileName);
using (var stream = FabricFile.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan))
{
var versionArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(FileVersion));
if (this.CurrentCheckpoint != null)
{
ArraySegment <byte> currentCheckpointFileNameArraySegment;
using (var writer = new InMemoryBinaryWriter(new MemoryStream()))
{
writer.Write(this.CurrentCheckpoint.FileName);
currentCheckpointFileNameArraySegment = new ArraySegment <byte>(writer.BaseStream.GetBuffer(), 0, (int)writer.BaseStream.Position);
}
var currentCheckpointFileNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(currentCheckpointFileNameArraySegment.Count));
var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointFileNameLengthArraySegment, currentCheckpointFileNameArraySegment });
var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));
await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointFileNameLengthArraySegment.Array,
currentCheckpointFileNameLengthArraySegment.Offset,
currentCheckpointFileNameLengthArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointFileNameArraySegment.Array,
currentCheckpointFileNameArraySegment.Offset,
currentCheckpointFileNameArraySegment.Count).ConfigureAwait(false);
await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
}
else
{
var currentCheckpointNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(0));
var crc = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointNameLengthArraySegment });
var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));
await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);
await
stream.WriteAsync(
currentCheckpointNameLengthArraySegment.Array,
currentCheckpointNameLengthArraySegment.Offset,
currentCheckpointNameLengthArraySegment.Count).ConfigureAwait(false);
await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
}
}
}
public void Write(InMemoryBinaryWriter writer)
{
writer.AssertIfNotAligned();
writer.Write(this.BlockSize);
writer.WritePaddingUntilAligned();
writer.AssertIfNotAligned();
}
/// <summary>
/// Serializes the FileMetadata into an in-memory binary writer.
/// </summary>
/// <param name="writer"></param>
/// <remarks>
/// This serialization is 8 byte aligned.
///
/// Name || Size
///
/// TotalNumberOfEntries 8
/// NumberOfValidEntries 8
/// NumberOfDeletedEntries 8
/// TimeStamp 8
///
/// FileId 4
/// CanBeDeleted 1
/// RESERVED 3
///
/// FileName N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// </remarks>
public void Write(InMemoryBinaryWriter writer)
{
Utility.Assert(writer.IsAligned(), "must be aligned");
// Serialize the metadata.
writer.Write(this.TotalNumberOfEntries);
writer.Write(this.NumberOfValidEntries);
writer.Write(this.NumberOfDeletedEntries);
writer.Write(this.TimeStamp);
// With RESERVE
writer.Write(this.FileId);
writer.WritePaddingUntilAligned();
// With PADDING
writer.Write(this.FileName);
writer.WritePaddingUntilAligned();
#if !DotNetCoreClr
FabricEvents.Events.CheckpointFileMetadata(this.tracer, DifferentialStoreConstants.FileMetadata_Write, FileName, FileId,
TotalNumberOfEntries, NumberOfDeletedEntries, NumberOfInvalidEntries);
#endif
Utility.Assert(writer.IsAligned(), "must be aligned");
}
/// <summary>
///
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TValue"></typeparam>
/// <param name="memoryBuffer"></param>
/// <param name="item"></param>
/// <param name="keySerializer"></param>
/// <param name="timeStamp"></param>
/// <remarks>
/// The data is written is 8 bytes aligned.
///
/// Name Type Size
///
/// KeySize int 4
/// Kind byte 1
/// RESERVED 3
/// VersionSequenceNumber long 8
///
/// (DeletedVersion)
/// timeStamp long 8
///
/// (Inserted || Updated)
/// Offset long 8
/// ValueChecksum ulong 8
/// ValueSize int 4
/// RESERVED 4
///
/// Key TKey N
/// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8)
///
/// RESERVED: Fixed padding that is usable to add fields in future.
/// PADDING: Due to dynamic size, cannot be used for adding fields.
///
/// Note: Larges Key size supported is int.MaxValue in bytes.
/// </remarks>
private void WriteKey <TKey, TValue>(
InMemoryBinaryWriter memoryBuffer, KeyValuePair <TKey, TVersionedItem <TValue> > item, IStateSerializer <TKey> keySerializer, long timeStamp)
{
Utility.Assert(memoryBuffer.IsAligned(), "must be aligned");
var recordPosition = memoryBuffer.BaseStream.Position;
memoryBuffer.BaseStream.Position += sizeof(int);
memoryBuffer.Write((byte)item.Value.Kind); // RecordKind
memoryBuffer.WritePaddingUntilAligned(); // RESERVED
memoryBuffer.Write(item.Value.VersionSequenceNumber); // LSN
if (item.Value.Kind == RecordKind.DeletedVersion)
{
memoryBuffer.Write(timeStamp);
}
else
{
// Deleted items don't have a value. We only serialize value properties for non-deleted items.
memoryBuffer.Write(item.Value.Offset); // value offset
memoryBuffer.Write(item.Value.ValueChecksum); // value checksum
memoryBuffer.Write(item.Value.ValueSize); // value size
memoryBuffer.WritePaddingUntilAligned(); // RESERVED
}
var keyPosition = memoryBuffer.BaseStream.Position;
keySerializer.Write(item.Key, memoryBuffer);
var keyEndPosition = memoryBuffer.BaseStream.Position;
Diagnostics.Assert(
keyEndPosition >= keyPosition,
DifferentialStoreConstants.TraceType,
"User's key IStateSerializer moved the stream position backwards unexpectedly!");
memoryBuffer.BaseStream.Position = recordPosition;
memoryBuffer.Write(checked ((int)(keyEndPosition - keyPosition)));
memoryBuffer.BaseStream.Position = keyEndPosition;
memoryBuffer.WritePaddingUntilAligned();
Utility.Assert(memoryBuffer.IsAligned(), "must be aligned");
// Update in-memory metadata.
this.Properties.KeyCount++;
}
private static async Task FlushMemoryBufferAsync(Stream fileStream, InMemoryBinaryWriter memoryBuffer)
{
// Write the chunk length at the beginning of the stream.
var chunkSize = checked ((int)memoryBuffer.BaseStream.Position);
memoryBuffer.BaseStream.Position = 0;
var propertyChunkMetadata = new PropertyChunkMetadata(chunkSize);
propertyChunkMetadata.Write(memoryBuffer);
// Move to end.
memoryBuffer.BaseStream.Position = chunkSize;
// Checksum the chunk.
var checksum = memoryBuffer.GetChecksum();
memoryBuffer.Write(checksum);
// Write to disk.
await memoryBuffer.WriteAsync(fileStream).ConfigureAwait(false);
// Reset the memory buffer.
memoryBuffer.BaseStream.Position = 0;
}
