public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount,
IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data)
where TInternal : class, IInternalData, new()
{
var header = new PersistedDataHeader(name, start, end,
PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
sources, this.dimensionSet, dataCount);
this.WriteDataWithLengthAndCRC32(ms =>
{
header.Write(new BufferWriter(ms));
}, header.SerializedSize, true);
// The raw data does not compress particularly well. There is some overlap in the keys, but particularly
// for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which
// makes for pretty compact data.
this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false);
// We can now determine what our block length really is and back-fill our data.
var currentPosition = this.sourceStream.Position;
var blockLength = this.sourceStream.Position - this.blockStartOffset;
this.sourceStream.Position = this.blockLengthOffset;
this.sourceStreamWriter.WriteUInt64((ulong)blockLength);
this.sourceStream.Position = currentPosition;
}
// Write a block of (optionally compressed) data with a length prefix and the CRC32 of the uncompressed contents.
// The length prefix is necessary due to peculiar behavior of .NET's DeflateStream (and GZipStream) which chew through
// an entire stream instead of reading only the compressed portion and stopping.
private void WriteDataWithLengthAndCRC32(Action <MemoryStream> writeAction, long suggestedLength, bool shouldCompress)
{
using (var ms = this.memoryStreamManager.GetStream("PersistedDataWriter", (int)suggestedLength))
{
var startPosition = this.sourceStream.Position;
// Record a placeholder for the total length of the written data (including optional uncompressed length and CRC32)
this.sourceStreamWriter.WriteUInt64(0);
if (shouldCompress)
{
// For compressed data we record both the overall data block length and the uncompressed data length.
// Recording the uncompressed data length affords efficient memory allocation when data is read back
// from storage.
this.sourceStreamWriter.WriteUInt64(0);
}
writeAction(ms);
var uncompressedLength = ms.Position;
var crc32 = CRC32.Compute(ms.GetBuffer(), 0, uncompressedLength);
this.sourceStreamWriter.WriteUInt32(crc32);
ms.Position = 0;
if (shouldCompress)
{
using (var compressionStream = new DeflateStream(this.sourceStream, CompressionLevel.Fastest, true))
{
ms.CopyTo(compressionStream);
compressionStream.Flush();
}
}
else
{
ms.CopyTo(this.sourceStream);
}
var currentPosition = this.sourceStream.Position;
var length = currentPosition - startPosition - sizeof(long);
length = PersistedDataProtocol.SerializeBufferLengthValue(length, shouldCompress,
PersistedDataProtocol.CompressionType.GZip);
this.sourceStream.Position = startPosition;
this.sourceStreamWriter.WriteInt64(length);
if (shouldCompress)
{
this.sourceStreamWriter.WriteInt64(uncompressedLength);
}
this.sourceStream.Position = currentPosition;
}
}
private async Task <bool> QuerySources(IList <PersistedDataSource> sources)
{
Events.Write.BeginQuerySources(sources);
var success = false;
var transferRequest =
new TransferRequest
{
DataType = PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
Timeout = (this.Timeout.Seconds * 9) / 10, // 90% of timeout goes to the child
MaxFanout = this.MaxFanout,
Sources = new List <string>()
};
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unknown;
transferRequest.Sources.Add(source.Name);
}
var serverOffset = this.randomNumberGenerator.Next(sources.Count);
var selectedSource = sources[serverOffset];
var request = new HttpRequestMessage(HttpMethod.Post, this.CreateRequestUri(selectedSource.Name));
using (var ms = (this.memoryStreamManager.GetStream()))
using (var writeStream = new WriterStream(ms, this.memoryStreamManager))
{
var writer = writeStream.CreateCompactBinaryWriter();
writer.Write(transferRequest);
request.Content = new ByteArrayContent(ms.ToArray());
}
try
{
Events.Write.BeginSendSourceQuery(selectedSource, request.RequestUri);
var response =
await this.httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
Events.Write.EndSendSourceQuery(selectedSource, (int)response.StatusCode, response.ReasonPhrase);
if (!response.IsSuccessStatusCode)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Unavailable;
}
return(true);
default:
return(false);
}
}
Events.Write.BeginReceiveSourceResponseBody(selectedSource);
var length = response.Content.Headers.ContentLength ?? 0;
using (var dataStream = this.memoryStreamManager.GetStream("PersistedDataAggregator/Http/Read",
(int)length, true))
using (var responseStream = await response.Content.ReadAsStreamAsync())
{
responseStream.CopyTo(dataStream);
dataStream.Position = 0;
using (var dataReader =
new PersistedDataReader(dataStream, this.memoryStreamManager, this.DimensionSet))
{
Events.Write.EndReceiveSourceResponseBody(selectedSource, (int)dataStream.Length);
success = true;
if (dataStream.Length == 0)
{
Events.Write.EmptyResponseReceivedFromSource(selectedSource);
// Over the line. Mark it zero. Or just available, but empty (which is okay!)
foreach (var source in sources)
{
source.Status = PersistedDataSourceStatus.Available;
}
}
else
{
try
{
this.UnpackResponse(dataReader);
}
catch (PersistedDataException ex)
{
Events.Write.PersistedDataExceptionFromSource(selectedSource, ex);
success = false;
}
}
}
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
if (ex is HttpRequestException || ex is IOException || ex is WebException || ex is ObjectDisposedException)
{
Events.Write.HttpExceptionFromSource(selectedSource, ex);
}
else
{
throw;
}
}
Events.Write.EndQuerySources(success, sources);
return(success);
}
private T LoadAndValidateData <T>(Func <MemoryStream, byte[], long, T> readAction)
{
MemoryStream memoryStream = null;
try
{
var startPosition = this.sourceStream.Position;
// Read header values for the next block
var valueData = new byte[sizeof(long)];
long length;
long dataLength;
bool compressed;
uint crc32;
if (this.usePreviousProtocol)
{
// legacy data is a four byte int, not 8.
this.CheckedRead(valueData, sizeof(int));
dataLength = length = BitConverter.ToInt32(valueData, 0);
// for the legacy data we'll read we know it's never compressed.
compressed = false;
this.CheckedRead(valueData, sizeof(int));
crc32 = BitConverter.ToUInt32(valueData, 0);
}
else
{
this.CheckedRead(valueData, sizeof(long));
length = BitConverter.ToInt64(valueData, 0);
dataLength = length - sizeof(uint);
PersistedDataProtocol.CompressionType compressionType;
length = PersistedDataProtocol.DeserializeBufferLengthValue(length, out compressed,
out compressionType);
if (compressed)
{
this.CheckedRead(valueData, sizeof(long));
dataLength = BitConverter.ToInt64(valueData, 0);
}
this.CheckedRead(valueData, sizeof(uint));
crc32 = BitConverter.ToUInt32(valueData, 0);
}
memoryStream = this.memoryStreamManager.GetStream("PersistedDataReader", (int)dataLength, true);
// Because we just use the underlying buffer the length won't be set by conventional methods, so we must do so
// manually.
memoryStream.SetLength(dataLength);
var buffer = memoryStream.GetBuffer();
if (compressed)
{
using (var compressionStream = new DeflateStream(this.sourceStream, CompressionMode.Decompress, true))
{
compressionStream.Read(buffer, 0, (int)dataLength);
}
// DeflateStream (and GZipStream) have this amazingly offensive behavior where they read through the whole
// stream instead of stopping at the end of compressed data. Soooo let's deal with that and set the stream
// position to what you'd expect.
this.sourceStream.Position = startPosition + length + sizeof(long);
}
else
{
this.CheckedRead(buffer, dataLength);
}
var dataCRC = CRC32.Compute(buffer, dataLength);
if (crc32 != dataCRC)
{
throw new PersistedDataException(string.Format("CRC failed for data, expected {0} was {1}",
crc32, dataCRC));
}
return(readAction(memoryStream, buffer, dataLength));
}
catch (Exception ex)
{
if (memoryStream != null)
{
memoryStream.Dispose();
}
if (ex is EndOfStreamException || ex is InvalidDataException)
{
throw new PersistedDataException("Stream data may be truncated", ex);
}
throw;
}
}