public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount,
IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data)
where TInternal : class, IInternalData, new()
{
var header = new PersistedDataHeader(name, start, end,
PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)),
sources, this.dimensionSet, dataCount);
this.WriteDataWithLengthAndCRC32(ms =>
{
header.Write(new BufferWriter(ms));
}, header.SerializedSize, true);
// The raw data does not compress particularly well. There is some overlap in the keys, but particularly
// for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which
// makes for pretty compact data.
this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false);
// We can now determine what our block length really is and back-fill our data.
var currentPosition = this.sourceStream.Position;
var blockLength = this.sourceStream.Position - this.blockStartOffset;
this.sourceStream.Position = this.blockLengthOffset;
this.sourceStreamWriter.WriteUInt64((ulong)blockLength);
this.sourceStream.Position = currentPosition;
}
private unsafe PersistedDataHeader LoadLegacyHeader()
{
MemoryStream memoryStream = null;
try
{
// legacy data used a variant of LZ4Net.
var codec = new LZ4Codec(this.memoryStreamManager, LZ4Codec.CodecProvider.LZ4Net);
using (var decompressionStream = codec.GetStream(this.sourceStream, CompressionMode.Decompress, true))
{
var lengthData = new byte[4];
LegacyCheckedRead(decompressionStream, lengthData, sizeof(int));
var length = BitConverter.ToUInt32(lengthData, 0);
var crcData = new byte[4];
LegacyCheckedRead(decompressionStream, crcData, sizeof(int));
var crc32 = BitConverter.ToUInt32(crcData, 0);
memoryStream = this.memoryStreamManager.GetStream("PersistedDataReader", (int)length, true);
memoryStream.SetLength(length);
var buffer = memoryStream.GetBuffer();
LegacyCheckedRead(decompressionStream, buffer, length);
var dataCRC = CRC32.Compute(buffer, length);
if (crc32 != dataCRC)
{
throw new PersistedDataException(string.Format("CRC failed for data, expected {0} was {1}",
crc32, dataCRC));
}
PersistedDataHeader header;
fixed(byte *buf = buffer)
{
header = new PersistedDataHeader(new BufferReader(buf, length));
}
memoryStream.Dispose();
return(header);
}
}
catch (Exception ex)
{
memoryStream?.Dispose();
if (ex is EndOfStreamException || ex is InvalidDataException)
{
throw new PersistedDataException("Stream data may be truncated", ex);
}
throw;
}
}
private unsafe PersistedDataHeader LoadHeader()
{
return(this.LoadAndValidateData((ms, buffer, length) =>
{
// Do not do 'using' here because if an exception leaks we are
// cleaned up elsewhere.
PersistedDataHeader header;
fixed(byte *buf = buffer)
{
header = new PersistedDataHeader(new BufferReader(buf, length));
}
ms.Dispose();
return header;
}));
}