Ejemplo n.º 1
0
        public async Task FlushAsync()
        {
            if (this.currentBlockPosition > 0)
            {
                this.WriteEndOfBlock();

                // Move the memory stream ahead to a 4k boundary.
                //this.keyBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);
                var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.keyBuffer, skipSize);
            }

            // Flush the memory buffer periodically to disk.
            if (this.keyBuffer.BaseStream.Position > 0)
            {
                await this.keyCheckpointFile.FlushMemoryBufferAsync(this.keyFileStream, this.keyBuffer).ConfigureAwait(false);
            }

            Diagnostics.Assert(
                this.keyFileStream.Position % BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize == 0,
                this.traceType,
                "Key checkpoint file is incorrectly block aligned at the end.");

            await this.keyCheckpointFile.FlushAsync(this.keyFileStream, this.keyBuffer).ConfigureAwait(false);
        }
        public async Task FlushAsync()
        {
            if (this.currentBlockPosition > 0)
            {
                // Move the memory stream ahead to a 4k boundary.
                //this.valueBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);

                var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.valueBuffer, skipSize);
            }

            await this.valueCheckpointFile.FlushAsync(this.valueFileStream, this.valueBuffer).ConfigureAwait(false);
        }
        public async Task BlockAlignedWriteValueAsync(KeyValuePair <TKey, TVersionedItem <TValue> > item, byte[] value, bool shouldSerialize)
        {
            if (item.Value.Kind == RecordKind.DeletedVersion)
            {
                this.valueCheckpointFile.WriteItem(this.valueFileStream, this.valueBuffer, item.Value, this.valueSerializer, this.traceType);

                return;
            }

            // Case 1: The value size is smaller than 4k
            if (item.Value.ValueSize <= this.blockAlignmentSize)
            {
                // If it falls within the block
                if (this.currentBlockPosition + item.Value.ValueSize <= this.blockAlignmentSize)
                {
                    await this.WriteItemAsync(item, value, shouldSerialize).ConfigureAwait(false);
                }
                else
                {
                    // Value does not fit into the current block, move position to the 4k+1
                    //this.valueBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);
                    var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                    BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.valueBuffer, skipSize);

                    // Reset current position.
                    this.currentBlockPosition = 0;

                    // Reset block size to default if value size is smaller, else pick the size correctly.
                    if (item.Value.ValueSize <= BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize)
                    {
                        this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
                    }
                    else
                    {
                        this.SetBlockSize(item);
                    }

                    // Write key and value
                    await this.WriteItemAsync(item, value, shouldSerialize).ConfigureAwait(false);
                }
            }
            else
            {
                // Case 2: Value size is greater than block size
                if (this.currentBlockPosition > 0)
                {
                    // Move the memory stream ahead to a 4k boundary.
                    // Value does not fit into the current block, move position to the 4k+1
                    // this.valueBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);

                    var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                    BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.valueBuffer, skipSize);

                    // Reset current position.
                    this.currentBlockPosition = 0;
                }

                // set block size to be a multiple of 4k bigger than the given value size.
                this.SetBlockSize(item);

                // Write key and value
                await this.WriteItemAsync(item, value, shouldSerialize).ConfigureAwait(false);
            }
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Create a new <see cref="CheckpointFile"/> from the given file, serializing the given key-values into the file.
        /// </summary>
        /// <param name="fileId">File identifier.</param>
        /// <param name="filename">File to create and write to.</param>
        /// <param name="sortedItemData">Sorted key-value pairs to include in the table.</param>
        /// <param name="keySerializer"></param>
        /// <param name="valueSerializer"></param>
        /// <param name="timeStamp"></param>
        /// <param name="traceType">Tracing information.</param>
        /// <param name="perfCounters">Store performance counters instance.</param>
        /// <param name="isValueAReferenceType">Indicated if the value type is reference type.</param>
        /// <returns>The new <see cref="CheckpointFile"/>.</returns>
        public static async Task <CheckpointFile> CreateAsync <TKey, TValue>(
            uint fileId,
            string filename,
            IEnumerable <KeyValuePair <TKey, TVersionedItem <TValue> > > sortedItemData,
            IStateSerializer <TKey> keySerializer,
            IStateSerializer <TValue> valueSerializer,
            long timeStamp,
            string traceType,
            FabricPerformanceCounterSetInstance perfCounters,
            bool isValueAReferenceType)
        {
            var keyFileName   = filename + KeyCheckpointFile.FileExtension;
            var valueFileName = filename + ValueCheckpointFile.FileExtension;

            var keyFile   = new KeyCheckpointFile(keyFileName, fileId, isValueAReferenceType);
            var valueFile = new ValueCheckpointFile(valueFileName, fileId, traceType);

            var checkpointFile = new CheckpointFile(filename, keyFile, valueFile, traceType);

            var perfCounterWriter = new TStoreCheckpointRateCounterWriter(perfCounters);

            // Create the key checkpoint file and memory buffer.
            // MCoskun: Creation of checkpoint file's IO priority is not set.
            // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
            using (var keyFileStream = FabricFile.Open(keyFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
                using (var keyMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
                {
                    // Create the value checkpoint file and memory buffer.
                    // MCoskun: Creation of checkpoint file's IO priority is not set.
                    // Reason: Checkpointing is a life-cycle operation for State Provider that can cause throttling of backup, copy and (extreme cases) write operations.
                    using (var valueFileStream = FabricFile.Open(valueFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.Asynchronous))
                        using (var valueMemoryBuffer = new InMemoryBinaryWriter(new MemoryStream(capacity: 64 * 1024)))
                        {
                            var blockAlignedWriter = new BlockAlignedWriter <TKey, TValue>(
                                valueFileStream,
                                keyFileStream,
                                valueFile,
                                keyFile,
                                valueMemoryBuffer,
                                keyMemoryBuffer,
                                valueSerializer,
                                keySerializer,
                                timeStamp,
                                traceType);

                            perfCounterWriter.StartMeasurement();

                            foreach (var item in sortedItemData)
                            {
                                await blockAlignedWriter.BlockAlignedWriteItemAsync(item, null, true).ConfigureAwait(false);
                            }

                            // Flush both key and value checkpoints to disk.
                            await blockAlignedWriter.FlushAsync().ConfigureAwait(false);

                            perfCounterWriter.StopMeasurement();
                        }
                }

            long writeBytes       = checkpointFile.GetFileSize();
            long writeBytesPerSec = perfCounterWriter.UpdatePerformanceCounter(writeBytes);

#if !DotNetCoreClr
            FabricEvents.Events.CheckpointFileWriteBytesPerSec(traceType, writeBytesPerSec);
#endif

            return(checkpointFile);
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Batches given keys and writes them on disk in 4K aligned batches.
        /// </summary>
        /// <param name="item">Key and the versioned item that contains metadata about the row.</param>
        /// <returns>Task that represents the asynchronous operation.</returns>
        /// <remarks>
        /// The data is written is 8 bytes aligned.
        ///
        /// Name                    Type        Size
        ///
        /// BlockMetadata
        /// BlockSize               int         4
        /// RESERVED                            4
        ///
        /// Block                   byte[]      N
        /// PADDING                             (N % 8 ==0) ? 0 : 8 - (N % 8)
        ///
        /// Checksum                ulong       8
        ///
        /// RESERVED: Fixed padding that is usable to add fields in future.
        /// PADDING:  Due to dynamic size, cannot be used for adding fields.
        ///
        /// Note: Larges Block supported is 2GB
        /// </remarks>
        public async Task BlockAlignedWriteKeyAsync(KeyValuePair <TKey, TVersionedItem <TValue> > item)
        {
            Diagnostics.Assert(this.tempKeyBuffer.BaseStream.Position == 0, this.traceType, "Temp key buffer should always start from position zero.");
            this.keyCheckpointFile.WriteItem <TKey, TValue>(this.tempKeyBuffer, item, this.keySerializer, this.timeStamp, this.traceType);

            // Temp key buffer always gets reset after every key copy, so it is safe to assume position as record size.
            var keyRecordsize = (int)this.tempKeyBuffer.BaseStream.Position;

            // Case1: when record size is lesser than or equal to 4k.
            if (keyRecordsize + ChecksumSize + KeyChunkMetadata.Size <= this.blockAlignmentSize)
            {
                // Check if the current record can fit into this block (just account for checksum).
                if (this.currentBlockPosition + keyRecordsize + ChecksumSize <= this.blockAlignmentSize)
                {
                    // Do a mem copy.
                    await this.CopyStreamAsync().ConfigureAwait(false);
                }
                else
                {
                    Diagnostics.Assert(this.currentBlockPosition > 0, this.traceType, "We should have serialized a least one record");

                    // write checksum since we are done with this 4k block.
                    this.WriteEndOfBlock();

                    // Skip to next 4k boundary.
                    // Value does not fit into the current block, move position to the 4k+1
                    //this.keyBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);
                    var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                    BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.keyBuffer, skipSize);

                    await this.ResetCurrentBlockPositionAndFlushIfNecessaryAsync().ConfigureAwait(false);

                    // Reset block size to default if value size is smaller, else pick the size correctly.
                    if (keyRecordsize + ChecksumSize + KeyChunkMetadata.Size <= BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize)
                    {
                        this.blockAlignmentSize = BlockAlignedWriter <TKey, TValue> .DefaultBlockAlignmentSize;
                    }
                    else
                    {
                        this.GetBlockSize(keyRecordsize + ChecksumSize + KeyChunkMetadata.Size);
                    }

                    // Do a mem copy.
                    await this.CopyStreamAsync().ConfigureAwait(false);
                }
            }
            else
            {
                // Case 2: Value size is greater than block size
                if (this.currentBlockPosition > 0)
                {
                    // write checksum since we are done with this 4k block.
                    this.WriteEndOfBlock();

                    // Move the memory stream ahead to a 4k boundary.
                    // Value does not fit into the current block, move position to the 4k+1
                    //this.keyBuffer.BaseStream.Position += (this.blockAlignmentSize - this.currentBlockPosition);
                    var skipSize = this.blockAlignmentSize - this.currentBlockPosition;
                    BlockAlignedWriter <TKey, TValue> .WriteBadFood(this.keyBuffer, skipSize);

                    // Reset current position.
                    await this.ResetCurrentBlockPositionAndFlushIfNecessaryAsync().ConfigureAwait(false);
                }

                // set block size to be a multiple of 4k bigger than the given value size.
                this.GetBlockSize(keyRecordsize + ChecksumSize + KeyChunkMetadata.Size);

                // Do a mem copy.
                await this.CopyStreamAsync().ConfigureAwait(false);
            }
        }