Exemple #1
0
        /// <summary>
        /// Write a block of log records to the output stream, including the size of the block and a checksum.
        /// </summary>
        /// <param name="outputStream">Backup log stream.</param>
        /// <param name="blockStream">Stream for the block of log records.</param>
        /// <param name="blockWriter">Writer for the block of log records.</param>
        /// <param name="cancellationToken">Token used to signal cancellation.</param>
        /// <returns>Task that represents the asynchronous operation.</returns>
        private async Task WriteLogRecordBlockAsync(
            Stream outputStream,
            MemoryStream blockStream,
            BinaryWriter blockWriter,
            CancellationToken cancellationToken)
        {
            // Get the size of the block.
            var blockSize = checked ((int)blockStream.Position);

            Utility.Assert(blockSize > sizeof(int), "BackupLogFile tried to write a zero-sized block of log records.");

            // Write the block size at the start of the block.
            blockStream.Position = 0;
            blockWriter.Write(blockSize);

            // Checksum this block of data (with the block size included).
            var blockChecksum = CRC64.ToCRC64(blockStream.GetBuffer(), 0, blockSize);

            // Add the checksum at the end of the memory stream.
            blockStream.Position = blockSize;
            blockWriter.Write(blockChecksum);

            // Flush the records block to the output stream.
            await outputStream.WriteAsync(
                blockStream.GetBuffer(),
                0,
                checked ((int)blockStream.Position),
                cancellationToken).ConfigureAwait(false);

            // Reset back to the start (leaving space for the 'int' block size).
            blockStream.Position = sizeof(int);
        }
Exemple #2
0
        /// <summary>
        /// Serialize the block into the given stream.
        /// </summary>
        /// <param name="stream">Stream to write to.</param>
        /// <param name="serializer"></param>
        public static async Task <BlockHandle> WriteBlockAsync(Stream stream, Action <BinaryWriter> serializer)
        {
            using (var memoryStream = new MemoryStream(DefaultMemoryStreamSize))
                using (var memoryWriter = new BinaryWriter(memoryStream))
                {
                    // Serialize the table section.
                    serializer.Invoke(memoryWriter);

                    var blockSize = checked ((int)memoryStream.Position);

                    // Calculate the checksum.
                    var checksum = CRC64.ToCRC64(memoryStream.GetBuffer(), 0, blockSize);

                    // Get the position and size of this section within the stream.
                    var sectionHandle = new BlockHandle(stream.Position, blockSize);

                    // Write the checksum.
                    memoryWriter.Write(checksum);

                    var blockSizeWithChecksum = checked ((int)memoryStream.Position);
                    Utility.Assert((blockSizeWithChecksum - blockSize) == 8, "unexpected block size.");

                    // Write the buffer and checksum to the stream.
                    await stream.WriteAsync(memoryStream.GetBuffer(), 0, blockSizeWithChecksum).ConfigureAwait(false);

                    return(sectionHandle);
                }
        }
        /// <summary>
        /// Write the state provider metadata to the output stream.
        /// </summary>
        /// <param name="outputstream">Output stream.</param>
        /// <param name="metadataList">The state providers' metadata.</param>
        /// <param name="cancellationToken">Token used to signal cancellation.</param>
        /// <returns>Task that represents the asynchronous operation.</returns>
        private async Task <StateManagerFileBlocks> WriteMetadataAsync(
            Stream outputstream,
            IEnumerable <SerializableMetadata> metadataList,
            CancellationToken cancellationToken)
        {
            // Track blocks of records, to improve sequential reads.
            var recordBlockSizes = new List <int>();
            var recordBlockSize  = 0;

            using (var itemStream = new MemoryStream(DesiredBlockSize * 2))
                using (var itemWriter = new InMemoryBinaryWriter(itemStream))
                {
                    foreach (var metadata in metadataList)
                    {
                        // Write each state provider metadata into the memory stream.
                        var recordPosition = checked ((int)itemStream.Position);
                        var recordSize     = WriteMetadata(itemWriter, metadata);

                        // Checksum this metadata record (with the record size included).
                        var recordChecksum = CRC64.ToCRC64(itemStream.GetBuffer(), recordPosition, recordSize);

                        // Add the checksum at the end of the memory stream.
                        itemWriter.Write(recordChecksum);

                        // Track record block sizes.  After enough bytes are buffered in memory, flush to disk.
                        recordBlockSize = checked ((int)itemStream.Position);
                        if (recordBlockSize >= DesiredBlockSize)
                        {
                            // Flush the metadata records to the output stream.
                            await outputstream.WriteAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);

                            recordBlockSizes.Add(recordBlockSize);
                            itemStream.Position = 0;
                        }

                        // Update properties.
                        this.Properties.StateProviderCount++;
                        if (metadata.ParentStateProviderId == DynamicStateManager.EmptyStateProviderId)
                        {
                            this.Properties.RootStateProviderCount++;
                        }
                    }

                    // Flush any remaining bytes to disk, and add the last record block size (if any).
                    recordBlockSize = checked ((int)itemStream.Position);
                    if (recordBlockSize > 0)
                    {
                        await outputstream.WriteAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);

                        recordBlockSizes.Add(recordBlockSize);
                    }
                }

            // Return the record block sizes.
            var blocks = new StateManagerFileBlocks(recordBlockSizes.ToArray());

            return(blocks);
        }
Exemple #4
0
        /// <summary>
        /// Deserialize a block from the given stream.
        /// </summary>
        /// <param name="stream">Stream to read from.</param>
        /// <param name="blockHandle"></param>
        /// <param name="deserializer"></param>
        /// <returns>The block read from the stream.</returns>
        public static async Task <TBlock> ReadBlockAsync <TBlock>(
            Stream stream,
            BlockHandle blockHandle,
            Func <BinaryReader, BlockHandle, TBlock> deserializer)
        {
            // Data validation.
            if (blockHandle.Offset < 0)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_StreamOffset));
            }

            if (blockHandle.Size < 0)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Negative_BlockSize));
            }

            if (blockHandle.EndOffset + sizeof(ulong) > stream.Length)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Block_Beyond_Stream));
            }

            // Move the stream to the block's position within the stream.
            stream.Position = blockHandle.Offset;
            var blockSize             = checked ((int)blockHandle.Size);
            var blockSizeWithChecksum = blockSize + sizeof(ulong);

            // Read the bytes for the block, to calculate the checksum.
            var blockBytes = new byte[blockSizeWithChecksum];
            await stream.ReadAsync(blockBytes, 0, blockBytes.Length).ConfigureAwait(false);

            // Calculate the checksum.
            var blockChecksum = CRC64.ToCRC64(blockBytes, 0, blockSize);

            // Read the actual checksum (checksum is after the block bytes).
            var checksum = BitConverter.ToUInt64(blockBytes, blockSize);

            // Verify the checksum.
            if (checksum != blockChecksum)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FileBlock_Corrupt, SR.Error_Checksum_Mismatch));
            }

            // Deserialize the actual file block.
            using (var memoryStream = new MemoryStream(blockBytes, 0, blockSize, writable: false))
                using (var memoryReader = new BinaryReader(memoryStream))
                {
                    return(deserializer.Invoke(memoryReader, new BlockHandle(0, blockHandle.Size)));
                }
        }
        /// <summary>
        /// Read the list of state providers' metadata from the <see cref="StateManagerFile"/>.
        /// </summary>
        /// <param name="stream">Stream to read from.</param>
        /// <param name="traceType">Tracing type information.</param>
        /// <param name="deserializeTypes">Should the types be deserialized.</param>
        /// <param name="cancellationToken">Token used to signal cancellation.</param>
        /// <returns>The list of state providers' metadata read.</returns>
        private async Task <List <SerializableMetadata> > ReadMetadataAsync(
            Stream stream,
            string traceType,
            bool deserializeTypes,
            CancellationToken cancellationToken)
        {
            var metadataList = new List <SerializableMetadata>((int)this.properties.StateProviderCount);

            // Read the blocks, to improve sequential reads.
            var blocks = await FileBlock.ReadBlockAsync(
                stream,
                this.properties.BlocksHandle,
                (sectionReader, sectionHandle) => StateManagerFileBlocks.Read(sectionReader, sectionHandle)).ConfigureAwait(false);

            // Currently, we expect the block sizes to always be present.
            if (blocks == null || blocks.RecordBlockSizes == null)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_BlockSizesMissing));
            }

            var recordBlockSizes = blocks.RecordBlockSizes;

            // Read blocks from the file.  Each state provider metadata is checksummed individually.
            using (var itemStream = new MemoryStream(DesiredBlockSize * 2))
                using (var itemReader = new InMemoryBinaryReader(itemStream))
                {
                    stream.Position = this.properties.MetadataHandle.Offset;
                    var endOffset = this.properties.MetadataHandle.EndOffset;

                    // Each block has one or more state provider metadata records.
                    foreach (var recordBlockSize in recordBlockSizes)
                    {
                        if (stream.Position + recordBlockSize > endOffset)
                        {
                            throw new InvalidDataException(
                                      string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_BlockExtendPastFile));
                        }

                        // Read the block into memory.
                        itemStream.Position = 0;
                        itemStream.SetLength(recordBlockSize);
                        await stream.ReadAsync(itemStream.GetBuffer(), 0, recordBlockSize, cancellationToken).ConfigureAwait(false);

                        // Read to the end of the metadata section.
                        var endBlockOffset = itemStream.Length;
                        while (itemStream.Position < endBlockOffset)
                        {
                            var position = itemStream.Position;

                            // Read the record size and validate it is not obviously corrupted.
                            if (position + sizeof(int) > endBlockOffset)
                            {
                                throw new InvalidDataException(
                                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MissingRecordSize));
                            }

                            var recordSize             = itemReader.ReadInt32();
                            var recordSizeWithChecksum = recordSize + sizeof(ulong);

                            // We need to do extra validation on the recordSize, because we haven't validated the bits
                            // against the checksum and we need the recordSize to locate the checksum.
                            if (recordSize < 0)
                            {
                                throw new InvalidDataException(
                                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_NegativeRecordSize));
                            }

                            if (position + recordSize > endBlockOffset)
                            {
                                throw new InvalidDataException(
                                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_RecordExtendsPastFile));
                            }

                            if (position + recordSizeWithChecksum > endBlockOffset)
                            {
                                throw new InvalidDataException(
                                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MissingChecksum));
                            }

                            // Compute the checksum.
                            var computedChecksum = CRC64.ToCRC64(
                                itemStream.GetBuffer(),
                                checked ((int)position),
                                recordSize);

                            // Read the checksum (checksum is after the record bytes).
                            itemStream.Position = position + recordSize;
                            var checksum = itemReader.ReadUInt64();

                            // Verify the checksum.
                            if (checksum != computedChecksum)
                            {
                                throw new InvalidDataException(
                                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MismatchedChecksum));
                            }

                            // Read and re-create the state provider metadata, now that the checksum is validated.
                            itemStream.Position = position;
                            var metadata = ReadMetadata(itemReader, traceType, deserializeTypes);
                            metadataList.Add(metadata);
                            itemStream.Position = position + recordSizeWithChecksum;
                        }
                    }
                }

            // Post-validation.
            if (metadataList.Count != (int)this.Properties.StateProviderCount)
            {
                throw new InvalidDataException(
                          string.Format(CultureInfo.CurrentCulture, SR.Error_SMFile_Corrupt_MetadataCountMismatch));
            }

            return(metadataList);
        }
Exemple #6
0
            public async Task <bool> MoveNextAsync(CancellationToken cancellationToken)
            {
                var position  = this.backupLogStream.Position;
                var endOffset = this.logRecordHandle.EndOffset;

                if (position < endOffset)
                {
                    // Read the size of the next block.
                    if (position + sizeof(int) > endOffset)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_MissingBlockSize));
                    }

                    await this.backupLogStream.ReadAsync(this.buffer, 0, sizeof(int)).ConfigureAwait(false);

                    var blockSize             = BitConverter.ToInt32(this.buffer, 0);
                    var blockSizeWithChecksum = blockSize + sizeof(ulong);

                    // We need to do extra validation on the blockSize, because we haven't validated the bits
                    // against the checksum and we need the blockSize to locate the checksum.
                    if (blockSize < 0)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_NegativeBlockSize));
                    }

                    if (position + blockSize > endOffset)
                    {
                        throw new InvalidDataException(
                                  string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_ExtendsPastFileEnd));
                    }

                    if (position + blockSizeWithChecksum > endOffset)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_RecordBlock_MissingChecksum));
                    }

                    // Read the next block into memory (including the size and checksum).
                    this.backupLogStream.Position = position;
                    this.blockStream.Position     = 0;
                    this.blockStream.SetLength(blockSizeWithChecksum);
                    await this.backupLogStream.ReadAsync(this.blockStream.GetBuffer(), 0, blockSizeWithChecksum).ConfigureAwait(false);

                    // Compute the checksum.
                    var computedChecksum = CRC64.ToCRC64(this.blockStream.GetBuffer(), 0, blockSize);

                    // Read the checksum (checksum is after the block bytes).
                    var checksum = BitConverter.ToUInt64(this.blockStream.GetBuffer(), blockSize);
                    // Verify the checksum.
                    if (checksum != computedChecksum)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_BackupLogFileAsyncEnumerator_Corrupt, SR.Error_Checksum_Mismatch));
                    }

                    // Return the prepared block reader, skipping over the int32 'size' field at the start and dropping the checksum.
                    this.blockStream.SetLength(blockSize);
                    this.Current          = this.blockStream;
                    this.Current.Position = sizeof(int);
                    return(true);
                }

                this.Current = null;
                return(false);
            }
 public ulong GetChecksum()
 {
     return(CRC64.ToCRC64(this.BaseStream.GetBuffer(), 0, checked ((int)this.BaseStream.Position)));
 }
Exemple #8
0
 /// <summary>
 /// Checksums the entire buffer.
 /// </summary>
 public ulong GetChecksum(int startOffset, int count)
 {
     return(CRC64.ToCRC64(this.BaseStream.GetBuffer(), startOffset, count));
 }