public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount,
                                          IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data)
            where TInternal : class, IInternalData, new()
        {
            var header = new PersistedDataHeader(name, start, end,
                                                 PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
                                                 sources, this.dimensionSet, dataCount);

            this.WriteDataWithLengthAndCRC32(ms =>
            {
                header.Write(new BufferWriter(ms));
            }, header.SerializedSize, true);

            // The raw data does not compress particularly well. There is some overlap in the keys, but particularly
            // for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which
            // makes for pretty compact data.
            this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false);

            // We can now determine what our block length really is and back-fill our data.
            var currentPosition = this.sourceStream.Position;
            var blockLength     = this.sourceStream.Position - this.blockStartOffset;

            this.sourceStream.Position = this.blockLengthOffset;
            this.sourceStreamWriter.WriteUInt64((ulong)blockLength);
            this.sourceStream.Position = currentPosition;
        }
        // Write a block of (optionally compressed) data with a length prefix and the CRC32 of the uncompressed contents.
        // The length prefix is necessary due to peculiar behavior of .NET's DeflateStream (and GZipStream) which chew through
        // an entire stream instead of reading only the compressed portion and stopping.
        private void WriteDataWithLengthAndCRC32(Action <MemoryStream> writeAction, long suggestedLength, bool shouldCompress)
        {
            using (var ms = this.memoryStreamManager.GetStream("PersistedDataWriter", (int)suggestedLength))
            {
                var startPosition = this.sourceStream.Position;
                // Record a placeholder for the total length of the written data (including optional uncompressed length and CRC32)
                this.sourceStreamWriter.WriteUInt64(0);
                if (shouldCompress)
                {
                    // For compressed data we record both the overall data block length and the uncompressed data length.
                    // Recording the uncompressed data length affords efficient memory allocation when data is read back
                    // from storage.
                    this.sourceStreamWriter.WriteUInt64(0);
                }

                writeAction(ms);

                var uncompressedLength = ms.Position;
                var crc32 = CRC32.Compute(ms.GetBuffer(), 0, uncompressedLength);
                this.sourceStreamWriter.WriteUInt32(crc32);

                ms.Position = 0;
                if (shouldCompress)
                {
                    using (var compressionStream = new DeflateStream(this.sourceStream, CompressionLevel.Fastest, true))
                    {
                        ms.CopyTo(compressionStream);
                        compressionStream.Flush();
                    }
                }
                else
                {
                    ms.CopyTo(this.sourceStream);
                }

                var currentPosition = this.sourceStream.Position;
                var length          = currentPosition - startPosition - sizeof(long);
                length = PersistedDataProtocol.SerializeBufferLengthValue(length, shouldCompress,
                                                                          PersistedDataProtocol.CompressionType.GZip);

                this.sourceStream.Position = startPosition;
                this.sourceStreamWriter.WriteInt64(length);
                if (shouldCompress)
                {
                    this.sourceStreamWriter.WriteInt64(uncompressedLength);
                }
                this.sourceStream.Position = currentPosition;
            }
        }
        private async Task <bool> QuerySources(IList <PersistedDataSource> sources)
        {
            Events.Write.BeginQuerySources(sources);

            var success         = false;
            var transferRequest =
                new TransferRequest
            {
                DataType  = PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
                Timeout   = (this.Timeout.Seconds * 9) / 10,   // 90% of timeout goes to the child
                MaxFanout = this.MaxFanout,
                Sources   = new List <string>()
            };

            foreach (var source in sources)
            {
                source.Status = PersistedDataSourceStatus.Unknown;
                transferRequest.Sources.Add(source.Name);
            }

            var serverOffset   = this.randomNumberGenerator.Next(sources.Count);
            var selectedSource = sources[serverOffset];

            var request = new HttpRequestMessage(HttpMethod.Post, this.CreateRequestUri(selectedSource.Name));

            using (var ms = (this.memoryStreamManager.GetStream()))
                using (var writeStream = new WriterStream(ms, this.memoryStreamManager))
                {
                    var writer = writeStream.CreateCompactBinaryWriter();
                    writer.Write(transferRequest);
                    request.Content = new ByteArrayContent(ms.ToArray());
                }

            try
            {
                Events.Write.BeginSendSourceQuery(selectedSource, request.RequestUri);
                var response =
                    await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);

                Events.Write.EndSendSourceQuery(selectedSource, (int)response.StatusCode, response.ReasonPhrase);

                if (!response.IsSuccessStatusCode)
                {
                    switch (response.StatusCode)
                    {
                    case HttpStatusCode.NotFound:
                        foreach (var source in sources)
                        {
                            source.Status = PersistedDataSourceStatus.Unavailable;
                        }
                        return(true);

                    default:
                        return(false);
                    }
                }

                Events.Write.BeginReceiveSourceResponseBody(selectedSource);
                var length = response.Content.Headers.ContentLength ?? 0;
                using (var dataStream = this.memoryStreamManager.GetStream("PersistedDataAggregator/Http/Read",
                                                                           (int)length, true))
                    using (var responseStream = await response.Content.ReadAsStreamAsync())
                    {
                        responseStream.CopyTo(dataStream);
                        dataStream.Position = 0;
                        using (var dataReader =
                                   new PersistedDataReader(dataStream, this.memoryStreamManager, this.DimensionSet))
                        {
                            Events.Write.EndReceiveSourceResponseBody(selectedSource, (int)dataStream.Length);

                            success = true;
                            if (dataStream.Length == 0)
                            {
                                Events.Write.EmptyResponseReceivedFromSource(selectedSource);
                                // Over the line. Mark it zero. Or just available, but empty (which is okay!)
                                foreach (var source in sources)
                                {
                                    source.Status = PersistedDataSourceStatus.Available;
                                }
                            }
                            else
                            {
                                try
                                {
                                    this.UnpackResponse(dataReader);
                                }
                                catch (PersistedDataException ex)
                                {
                                    Events.Write.PersistedDataExceptionFromSource(selectedSource, ex);
                                    success = false;
                                }
                            }
                        }
                    }
            }
            catch (OperationCanceledException) { }
            catch (Exception ex)
            {
                if (ex is HttpRequestException || ex is IOException || ex is WebException || ex is ObjectDisposedException)
                {
                    Events.Write.HttpExceptionFromSource(selectedSource, ex);
                }
                else
                {
                    throw;
                }
            }

            Events.Write.EndQuerySources(success, sources);
            return(success);
        }
Exemplo n.º 4
0
        private T LoadAndValidateData <T>(Func <MemoryStream, byte[], long, T> readAction)
        {
            MemoryStream memoryStream = null;

            try
            {
                var startPosition = this.sourceStream.Position;

                // Read header values for the next block
                var  valueData = new byte[sizeof(long)];
                long length;
                long dataLength;
                bool compressed;
                uint crc32;
                if (this.usePreviousProtocol)
                {
                    // legacy data is a four byte int, not 8.
                    this.CheckedRead(valueData, sizeof(int));
                    dataLength = length = BitConverter.ToInt32(valueData, 0);
                    // for the legacy data we'll read we know it's never compressed.
                    compressed = false;
                    this.CheckedRead(valueData, sizeof(int));
                    crc32 = BitConverter.ToUInt32(valueData, 0);
                }
                else
                {
                    this.CheckedRead(valueData, sizeof(long));
                    length = BitConverter.ToInt64(valueData, 0);

                    dataLength = length - sizeof(uint);
                    PersistedDataProtocol.CompressionType compressionType;
                    length = PersistedDataProtocol.DeserializeBufferLengthValue(length, out compressed,
                                                                                out compressionType);
                    if (compressed)
                    {
                        this.CheckedRead(valueData, sizeof(long));
                        dataLength = BitConverter.ToInt64(valueData, 0);
                    }
                    this.CheckedRead(valueData, sizeof(uint));
                    crc32 = BitConverter.ToUInt32(valueData, 0);
                }

                memoryStream = this.memoryStreamManager.GetStream("PersistedDataReader", (int)dataLength, true);
                // Because we just use the underlying buffer the length won't be set by conventional methods, so we must do so
                // manually.
                memoryStream.SetLength(dataLength);
                var buffer = memoryStream.GetBuffer();
                if (compressed)
                {
                    using (var compressionStream = new DeflateStream(this.sourceStream, CompressionMode.Decompress, true))
                    {
                        compressionStream.Read(buffer, 0, (int)dataLength);
                    }
                    // DeflateStream (and GZipStream) have this amazingly offensive behavior where they read through the whole
                    // stream instead of stopping at the end of compressed data. Soooo let's deal with that and set the stream
                    // position to what you'd expect.
                    this.sourceStream.Position = startPosition + length + sizeof(long);
                }
                else
                {
                    this.CheckedRead(buffer, dataLength);
                }

                var dataCRC = CRC32.Compute(buffer, dataLength);

                if (crc32 != dataCRC)
                {
                    throw new PersistedDataException(string.Format("CRC failed for data, expected {0} was {1}",
                                                                   crc32, dataCRC));
                }

                return(readAction(memoryStream, buffer, dataLength));
            }
            catch (Exception ex)
            {
                if (memoryStream != null)
                {
                    memoryStream.Dispose();
                }
                if (ex is EndOfStreamException || ex is InvalidDataException)
                {
                    throw new PersistedDataException("Stream data may be truncated", ex);
                }

                throw;
            }
        }