/// <summary> /// Opens a TStore checkpoint from the given file name. /// </summary> /// <param name="filename">The file to open that contains an existing checkpoint.</param> /// <param name="traceType">Tracing information.</param> /// <param name="isValueReferenceType"></param> public static async Task <CheckpointFile> OpenAsync(string filename, string traceType, bool isValueReferenceType) { KeyCheckpointFile keyFile = null; ValueCheckpointFile valueFile = null; try { keyFile = await KeyCheckpointFile.OpenAsync(filename + KeyCheckpointFile.FileExtension, traceType, isValueReferenceType).ConfigureAwait(false); valueFile = await ValueCheckpointFile.OpenAsync(filename + ValueCheckpointFile.FileExtension, traceType).ConfigureAwait(false); return(new CheckpointFile(filename, keyFile, valueFile, traceType)); } catch (Exception) { // Ensure the key and value files get disposed quickly if we get an exception. // Normally the CheckpointFile would dispose them, but it may not get constructed. if (keyFile != null) { keyFile.Dispose(); } if (valueFile != null) { valueFile.Dispose(); } throw; } }
public BlockAlignedWriter( FileStream valueFileStream, FileStream keyFileStream, ValueCheckpointFile valueCheckpointFile, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter valueBuffer, InMemoryBinaryWriter keyBuffer, IStateSerializer <TValue> valueSerializer, IStateSerializer <TKey> keySerializer, long timeStamp, string traceType) { this.valueCheckpointFile = valueCheckpointFile; this.keyCheckpointFile = keyCheckpointFile; this.timeStamp = timeStamp; this.valueFileStream = valueFileStream; this.keyFileStream = keyFileStream; this.valueBuffer = valueBuffer; this.keyBuffer = keyBuffer; this.valueSerializer = valueSerializer; this.keySerializer = keySerializer; this.traceType = traceType; this.keyBlockAlignedWriter = new KeyBlockAlignedWriter <TKey, TValue>( keyFileStream, keyCheckpointFile, keyBuffer, keySerializer, timeStamp, traceType); this.valueBlockAlignedWriter = new ValueBlockAlignedWriter <TKey, TValue>( valueFileStream, valueCheckpointFile, valueBuffer, valueSerializer, traceType); }
/// <summary> /// Opens a <see cref="KeyCheckpointFile"/> from the given file. /// </summary> /// <param name="filename">The file to open that contains an existing checkpoint file.</param> /// <param name="traceType">Tracing information.</param> /// <param name="isValueReferenceType"></param> public static async Task <KeyCheckpointFile> OpenAsync(string filename, string traceType, bool isValueReferenceType) { FabricEvents.Events.OpenKeyCheckpointFile(traceType, filename, DiskConstants.state_opening); Diagnostics.Assert(FabricFile.Exists(filename), traceType, "File name {0} does not exist", filename); KeyCheckpointFile checkpointFile = null; try { checkpointFile = new KeyCheckpointFile(filename, isValueReferenceType); await checkpointFile.ReadMetadataAsync().ConfigureAwait(false); } catch (Exception) { // Ensure the checkpoint file get disposed quickly if we get an exception. if (checkpointFile != null) { checkpointFile.Dispose(); } throw; } FabricEvents.Events.OpenKeyCheckpointFile(traceType, filename, DiskConstants.state_complete); return(checkpointFile); }
public CheckpointFile(string filename, KeyCheckpointFile keyCheckpointFile, ValueCheckpointFile valueCheckpointName, string traceType) { // Reserved for private static CheckpointFile.OpenAsync() or CheckpointFile.CreateAsync(). this.FileNameBase = filename; this.KeyCheckpointFile = keyCheckpointFile; this.ValueCheckpointFile = valueCheckpointName; this.TraceType = traceType; }
public KeyBlockAlignedWriter( FileStream keyFileStream, KeyCheckpointFile keyCheckpointFile, InMemoryBinaryWriter keyBuffer, IStateSerializer <TKey> keySerializer, long timeStamp, string traceType) { this.keyCheckpointFile = keyCheckpointFile; this.timeStamp = timeStamp; this.keyFileStream = keyFileStream; this.keyBuffer = keyBuffer; this.keySerializer = keySerializer; this.traceType = traceType; this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize; // Most key records are lesser than 4k, when they get beyond 4k, do not shrink the buffer. this.tempKeyBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 4 * 1024)); }
public KeyCheckpointFileAsyncEnumerator( KeyCheckpointFile keyCheckpointFile, IStateSerializer <TKey> keySerializer, long startOffset, long endOffset, string traceType, Kernel32Types.PRIORITY_HINT priorityHint = Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal) { this.traceType = traceType; this.keyCheckpointFile = keyCheckpointFile; this.stateZero = true; this.index = -1; this.keySerializer = keySerializer; this.startOffset = startOffset; this.endOffset = endOffset; this.priorityHint = priorityHint; }
/// <summary> /// Create a new <see cref="CheckpointFile"/> from the given file, serializing the given key-values into the file. /// </summary> /// <param name="fileId">File identifier.</param> /// <param name="filename">File to create and write to.</param> /// <param name="sortedItemData">Sorted key-value pairs to include in the table.</param> /// <param name="keySerializer"></param> /// <param name="valueSerializer"></param> /// <param name="timeStamp"></param> /// <param name="traceType">Tracing information.</param> /// <param name="perfCounters">Store performance counters instance.</param> /// <param name="isValueAReferenceType">Indicated if the value type is reference type.</param> /// <returns>The new <see cref="CheckpointFile"/>.</returns> public static async Task <CheckpointFile> CreateAsync <TKey, TValue>( uint fileId, string filename, IEnumerable <KeyValuePair <TKey, TVersionedItem <TValue> > > sortedItemData, IStateSerializer <TKey> keySerializer, IStateSerializer <TValue> valueSerializer, long timeStamp, string traceType, FabricPerformanceCounterSetInstance perfCounters, bool isValueAReferenceType) { var keyFileName = filename + KeyCheckpointFile.FileExtension; var valueFileName = filename + ValueCheckpointFile.FileExtension; var keyFile = new KeyCheckpointFile(keyFileName, fileId, isValueAReferenceType); var valueFile = new ValueCheckpointFile(valueFileName, fileId, traceType); var checkpointFile = new CheckpointFile(filename, keyFile, valueFile, traceType); var perfCounterWriter = new TStoreCheckpointRateCounterWriter(perfCounters); // Create the key checkpoint file and memory buffer. // MCoskun: Creation of checkpoint file's IO priority is not set. // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations. using (var keyFileStream = FabricFile.Open(keyFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous)) using (var keyMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024))) { // Create the value checkpoint file and memory buffer. // MCoskun: Creation of checkpoint file's IO priority is not set. // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations. using (var valueFileStream = FabricFile.Open(valueFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous)) using (var valueMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024))) { var blockAlignedWriter = new BlockAlignedWriter <TKey, TValue>( valueFileStream, keyFileStream, valueFile, keyFile, valueMemoryBuffer, keyMemoryBuffer, valueSerializer, keySerializer, timeStamp, traceType); perfCounterWriter.StartMeasurement(); foreach (var item in sortedItemData) { await blockAlignedWriter.BlockAlignedWriteItemAsync(item, null, true).ConfigureAwait(false); } // Flush both key and value checkpoints to disk. await blockAlignedWriter.FlushAsync().ConfigureAwait(false); perfCounterWriter.StopMeasurement(); } } long writeBytes = checkpointFile.GetFileSize(); long writeBytesPerSec = perfCounterWriter.UpdatePerformanceCounter(writeBytes); #if !DotNetCoreClr FabricEvents.Events.CheckpointFileWriteBytesPerSec(traceType, writeBytesPerSec); #endif return(checkpointFile); }