/// <summary>
/// Opens a TStore checkpoint from the given file name.
/// </summary>
/// <param name="filename">The file to open that contains an existing checkpoint.</param>
/// <param name="traceType">Tracing information.</param>
/// <param name="isValueReferenceType"></param>
public static async Task <CheckpointFile> OpenAsync(string filename, string traceType, bool isValueReferenceType)
{
KeyCheckpointFile keyFile = null;
ValueCheckpointFile valueFile = null;
try
{
keyFile = await KeyCheckpointFile.OpenAsync(filename + KeyCheckpointFile.FileExtension, traceType, isValueReferenceType).ConfigureAwait(false);
valueFile = await ValueCheckpointFile.OpenAsync(filename + ValueCheckpointFile.FileExtension, traceType).ConfigureAwait(false);
return(new CheckpointFile(filename, keyFile, valueFile, traceType));
}
catch (Exception)
{
// Ensure the key and value files get disposed quickly if we get an exception.
// Normally the CheckpointFile would dispose them, but it may not get constructed.
if (keyFile != null)
{
keyFile.Dispose();
}
if (valueFile != null)
{
valueFile.Dispose();
}
throw;
}
}
public BlockAlignedWriter(
FileStream valueFileStream, FileStream keyFileStream, ValueCheckpointFile valueCheckpointFile, KeyCheckpointFile keyCheckpointFile,
InMemoryBinaryWriter valueBuffer, InMemoryBinaryWriter keyBuffer, IStateSerializer <TValue> valueSerializer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.valueCheckpointFile = valueCheckpointFile;
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.valueFileStream = valueFileStream;
this.keyFileStream = keyFileStream;
this.valueBuffer = valueBuffer;
this.keyBuffer = keyBuffer;
this.valueSerializer = valueSerializer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.keyBlockAlignedWriter = new KeyBlockAlignedWriter <TKey, TValue>(
keyFileStream,
keyCheckpointFile,
keyBuffer,
keySerializer,
timeStamp,
traceType);
this.valueBlockAlignedWriter = new ValueBlockAlignedWriter <TKey, TValue>(
valueFileStream,
valueCheckpointFile,
valueBuffer,
valueSerializer,
traceType);
}
/// <summary>
/// Opens a <see cref="KeyCheckpointFile"/> from the given file.
/// </summary>
/// <param name="filename">The file to open that contains an existing checkpoint file.</param>
/// <param name="traceType">Tracing information.</param>
/// <param name="isValueReferenceType"></param>
public static async Task <KeyCheckpointFile> OpenAsync(string filename, string traceType, bool isValueReferenceType)
{
FabricEvents.Events.OpenKeyCheckpointFile(traceType, filename, DiskConstants.state_opening);
Diagnostics.Assert(FabricFile.Exists(filename), traceType, "File name {0} does not exist", filename);
KeyCheckpointFile checkpointFile = null;
try
{
checkpointFile = new KeyCheckpointFile(filename, isValueReferenceType);
await checkpointFile.ReadMetadataAsync().ConfigureAwait(false);
}
catch (Exception)
{
// Ensure the checkpoint file get disposed quickly if we get an exception.
if (checkpointFile != null)
{
checkpointFile.Dispose();
}
throw;
}
FabricEvents.Events.OpenKeyCheckpointFile(traceType, filename, DiskConstants.state_complete);
return(checkpointFile);
}
public CheckpointFile(string filename, KeyCheckpointFile keyCheckpointFile, ValueCheckpointFile valueCheckpointName, string traceType)
{
// Reserved for private static CheckpointFile.OpenAsync() or CheckpointFile.CreateAsync().
this.FileNameBase = filename;
this.KeyCheckpointFile = keyCheckpointFile;
this.ValueCheckpointFile = valueCheckpointName;
this.TraceType = traceType;
}
public KeyBlockAlignedWriter(
FileStream keyFileStream, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter keyBuffer, IStateSerializer <TKey> keySerializer,
long timeStamp, string traceType)
{
this.keyCheckpointFile = keyCheckpointFile;
this.timeStamp = timeStamp;
this.keyFileStream = keyFileStream;
this.keyBuffer = keyBuffer;
this.keySerializer = keySerializer;
this.traceType = traceType;
this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
// Most key records are lesser than 4k, when they get beyond 4k, do not shrink the buffer.
this.tempKeyBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 4 * 1024));
}
public KeyCheckpointFileAsyncEnumerator(
KeyCheckpointFile keyCheckpointFile,
IStateSerializer <TKey> keySerializer,
long startOffset,
long endOffset,
string traceType,
Kernel32Types.PRIORITY_HINT priorityHint = Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal)
{
this.traceType = traceType;
this.keyCheckpointFile = keyCheckpointFile;
this.stateZero = true;
this.index = -1;
this.keySerializer = keySerializer;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.priorityHint = priorityHint;
}
/// <summary>
/// Create a new <see cref="CheckpointFile"/> from the given file, serializing the given key-values into the file.
/// </summary>
/// <param name="fileId">File identifier.</param>
/// <param name="filename">File to create and write to.</param>
/// <param name="sortedItemData">Sorted key-value pairs to include in the table.</param>
/// <param name="keySerializer"></param>
/// <param name="valueSerializer"></param>
/// <param name="timeStamp"></param>
/// <param name="traceType">Tracing information.</param>
/// <param name="perfCounters">Store performance counters instance.</param>
/// <param name="isValueAReferenceType">Indicated if the value type is reference type.</param>
/// <returns>The new <see cref="CheckpointFile"/>.</returns>
public static async Task <CheckpointFile> CreateAsync <TKey, TValue>(
uint fileId,
string filename,
IEnumerable <KeyValuePair <TKey, TVersionedItem <TValue> > > sortedItemData,
IStateSerializer <TKey> keySerializer,
IStateSerializer <TValue> valueSerializer,
long timeStamp,
string traceType,
FabricPerformanceCounterSetInstance perfCounters,
bool isValueAReferenceType)
{
var keyFileName = filename + KeyCheckpointFile.FileExtension;
var valueFileName = filename + ValueCheckpointFile.FileExtension;
var keyFile = new KeyCheckpointFile(keyFileName, fileId, isValueAReferenceType);
var valueFile = new ValueCheckpointFile(valueFileName, fileId, traceType);
var checkpointFile = new CheckpointFile(filename, keyFile, valueFile, traceType);
var perfCounterWriter = new TStoreCheckpointRateCounterWriter(perfCounters);
// Create the key checkpoint file and memory buffer.
// MCoskun: Creation of checkpoint file's IO priority is not set.
// Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
using (var keyFileStream = FabricFile.Open(keyFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
using (var keyMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
{
// Create the value checkpoint file and memory buffer.
// MCoskun: Creation of checkpoint file's IO priority is not set.
// Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
using (var valueFileStream = FabricFile.Open(valueFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
using (var valueMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
{
var blockAlignedWriter = new BlockAlignedWriter <TKey, TValue>(
valueFileStream,
keyFileStream,
valueFile,
keyFile,
valueMemoryBuffer,
keyMemoryBuffer,
valueSerializer,
keySerializer,
timeStamp,
traceType);
perfCounterWriter.StartMeasurement();
foreach (var item in sortedItemData)
{
await blockAlignedWriter.BlockAlignedWriteItemAsync(item, null, true).ConfigureAwait(false);
}
// Flush both key and value checkpoints to disk.
await blockAlignedWriter.FlushAsync().ConfigureAwait(false);
perfCounterWriter.StopMeasurement();
}
}
long writeBytes = checkpointFile.GetFileSize();
long writeBytesPerSec = perfCounterWriter.UpdatePerformanceCounter(writeBytes);
#if !DotNetCoreClr
FabricEvents.Events.CheckpointFileWriteBytesPerSec(traceType, writeBytesPerSec);
#endif
return(checkpointFile);
}
