public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount, IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data) where TInternal : class, IInternalData, new() { var header = new PersistedDataHeader(name, start, end, PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)), sources, this.dimensionSet, dataCount); this.WriteDataWithLengthAndCRC32(ms => { header.Write(new BufferWriter(ms)); }, header.SerializedSize, true); // The raw data does not compress particularly well. There is some overlap in the keys, but particularly // for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which // makes for pretty compact data. this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false); // We can now determine what our block length really is and back-fill our data. var currentPosition = this.sourceStream.Position; var blockLength = this.sourceStream.Position - this.blockStartOffset; this.sourceStream.Position = this.blockLengthOffset; this.sourceStreamWriter.WriteUInt64((ulong)blockLength); this.sourceStream.Position = currentPosition; }
// Write a block of (optionally compressed) data with a length prefix and the CRC32 of the uncompressed contents. // The length prefix is necessary due to peculiar behavior of .NET's DeflateStream (and GZipStream) which chew through // an entire stream instead of reading only the compressed portion and stopping. private void WriteDataWithLengthAndCRC32(Action <MemoryStream> writeAction, long suggestedLength, bool shouldCompress) { using (var ms = this.memoryStreamManager.GetStream("PersistedDataWriter", (int)suggestedLength)) { var startPosition = this.sourceStream.Position; // Record a placeholder for the total length of the written data (including optional uncompressed length and CRC32) this.sourceStreamWriter.WriteUInt64(0); if (shouldCompress) { // For compressed data we record both the overall data block length and the uncompressed data length. // Recording the uncompressed data length affords efficient memory allocation when data is read back // from storage. this.sourceStreamWriter.WriteUInt64(0); } writeAction(ms); var uncompressedLength = ms.Position; var crc32 = CRC32.Compute(ms.GetBuffer(), 0, uncompressedLength); this.sourceStreamWriter.WriteUInt32(crc32); ms.Position = 0; if (shouldCompress) { using (var compressionStream = new DeflateStream(this.sourceStream, CompressionLevel.Fastest, true)) { ms.CopyTo(compressionStream); compressionStream.Flush(); } } else { ms.CopyTo(this.sourceStream); } var currentPosition = this.sourceStream.Position; var length = currentPosition - startPosition - sizeof(long); length = PersistedDataProtocol.SerializeBufferLengthValue(length, shouldCompress, PersistedDataProtocol.CompressionType.GZip); this.sourceStream.Position = startPosition; this.sourceStreamWriter.WriteInt64(length); if (shouldCompress) { this.sourceStreamWriter.WriteInt64(uncompressedLength); } this.sourceStream.Position = currentPosition; } }
private async Task <bool> QuerySources(IList <PersistedDataSource> sources) { Events.Write.BeginQuerySources(sources); var success = false; var transferRequest = new TransferRequest { DataType = PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)), Timeout = (this.Timeout.Seconds * 9) / 10, // 90% of timeout goes to the child MaxFanout = this.MaxFanout, Sources = new List <string>() }; foreach (var source in sources) { source.Status = PersistedDataSourceStatus.Unknown; transferRequest.Sources.Add(source.Name); } var serverOffset = this.randomNumberGenerator.Next(sources.Count); var selectedSource = sources[serverOffset]; var request = new HttpRequestMessage(HttpMethod.Post, this.CreateRequestUri(selectedSource.Name)); using (var ms = (this.memoryStreamManager.GetStream())) using (var writeStream = new WriterStream(ms, this.memoryStreamManager)) { var writer = writeStream.CreateCompactBinaryWriter(); writer.Write(transferRequest); request.Content = new ByteArrayContent(ms.ToArray()); } try { Events.Write.BeginSendSourceQuery(selectedSource, request.RequestUri); var response = await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); Events.Write.EndSendSourceQuery(selectedSource, (int)response.StatusCode, response.ReasonPhrase); if (!response.IsSuccessStatusCode) { switch (response.StatusCode) { case HttpStatusCode.NotFound: foreach (var source in sources) { source.Status = PersistedDataSourceStatus.Unavailable; } return(true); default: return(false); } } Events.Write.BeginReceiveSourceResponseBody(selectedSource); var length = response.Content.Headers.ContentLength ?? 0; using (var dataStream = this.memoryStreamManager.GetStream("PersistedDataAggregator/Http/Read", (int)length, true)) using (var responseStream = await response.Content.ReadAsStreamAsync()) { responseStream.CopyTo(dataStream); dataStream.Position = 0; using (var dataReader = new PersistedDataReader(dataStream, this.memoryStreamManager, this.DimensionSet)) { Events.Write.EndReceiveSourceResponseBody(selectedSource, (int)dataStream.Length); success = true; if (dataStream.Length == 0) { Events.Write.EmptyResponseReceivedFromSource(selectedSource); // Over the line. Mark it zero. Or just available, but empty (which is okay!) foreach (var source in sources) { source.Status = PersistedDataSourceStatus.Available; } } else { try { this.UnpackResponse(dataReader); } catch (PersistedDataException ex) { Events.Write.PersistedDataExceptionFromSource(selectedSource, ex); success = false; } } } } } catch (OperationCanceledException) { } catch (Exception ex) { if (ex is HttpRequestException || ex is IOException || ex is WebException || ex is ObjectDisposedException) { Events.Write.HttpExceptionFromSource(selectedSource, ex); } else { throw; } } Events.Write.EndQuerySources(success, sources); return(success); }
private T LoadAndValidateData <T>(Func <MemoryStream, byte[], long, T> readAction) { MemoryStream memoryStream = null; try { var startPosition = this.sourceStream.Position; // Read header values for the next block var valueData = new byte[sizeof(long)]; long length; long dataLength; bool compressed; uint crc32; if (this.usePreviousProtocol) { // legacy data is a four byte int, not 8. this.CheckedRead(valueData, sizeof(int)); dataLength = length = BitConverter.ToInt32(valueData, 0); // for the legacy data we'll read we know it's never compressed. compressed = false; this.CheckedRead(valueData, sizeof(int)); crc32 = BitConverter.ToUInt32(valueData, 0); } else { this.CheckedRead(valueData, sizeof(long)); length = BitConverter.ToInt64(valueData, 0); dataLength = length - sizeof(uint); PersistedDataProtocol.CompressionType compressionType; length = PersistedDataProtocol.DeserializeBufferLengthValue(length, out compressed, out compressionType); if (compressed) { this.CheckedRead(valueData, sizeof(long)); dataLength = BitConverter.ToInt64(valueData, 0); } this.CheckedRead(valueData, sizeof(uint)); crc32 = BitConverter.ToUInt32(valueData, 0); } memoryStream = this.memoryStreamManager.GetStream("PersistedDataReader", (int)dataLength, true); // Because we just use the underlying buffer the length won't be set by conventional methods, so we must do so // manually. memoryStream.SetLength(dataLength); var buffer = memoryStream.GetBuffer(); if (compressed) { using (var compressionStream = new DeflateStream(this.sourceStream, CompressionMode.Decompress, true)) { compressionStream.Read(buffer, 0, (int)dataLength); } // DeflateStream (and GZipStream) have this amazingly offensive behavior where they read through the whole // stream instead of stopping at the end of compressed data. Soooo let's deal with that and set the stream // position to what you'd expect. this.sourceStream.Position = startPosition + length + sizeof(long); } else { this.CheckedRead(buffer, dataLength); } var dataCRC = CRC32.Compute(buffer, dataLength); if (crc32 != dataCRC) { throw new PersistedDataException(string.Format("CRC failed for data, expected {0} was {1}", crc32, dataCRC)); } return(readAction(memoryStream, buffer, dataLength)); } catch (Exception ex) { if (memoryStream != null) { memoryStream.Dispose(); } if (ex is EndOfStreamException || ex is InvalidDataException) { throw new PersistedDataException("Stream data may be truncated", ex); } throw; } }