public void WriteData <TInternal>(string name, DateTime start, DateTime end, uint dataCount, IEnumerable <PersistedDataSource> sources, KeyedDataStore <TInternal> data) where TInternal : class, IInternalData, new() { var header = new PersistedDataHeader(name, start, end, PersistedDataProtocol.GetPersistedTypeCodeFromType(typeof(TInternal)), sources, this.dimensionSet, dataCount); this.WriteDataWithLengthAndCRC32(ms => { header.Write(new BufferWriter(ms)); }, header.SerializedSize, true); // The raw data does not compress particularly well. There is some overlap in the keys, but particularly // for histograms they come pre-compressed. We also use VLE heavily in KeyedDataStore.Serialize which // makes for pretty compact data. this.WriteDataWithLengthAndCRC32(data.Serialize, data.SerializedSize, false); // We can now determine what our block length really is and back-fill our data. var currentPosition = this.sourceStream.Position; var blockLength = this.sourceStream.Position - this.blockStartOffset; this.sourceStream.Position = this.blockLengthOffset; this.sourceStreamWriter.WriteUInt64((ulong)blockLength); this.sourceStream.Position = currentPosition; }
private unsafe PersistedDataHeader LoadLegacyHeader() { MemoryStream memoryStream = null; try { // legacy data used a variant of LZ4Net. var codec = new LZ4Codec(this.memoryStreamManager, LZ4Codec.CodecProvider.LZ4Net); using (var decompressionStream = codec.GetStream(this.sourceStream, CompressionMode.Decompress, true)) { var lengthData = new byte[4]; LegacyCheckedRead(decompressionStream, lengthData, sizeof(int)); var length = BitConverter.ToUInt32(lengthData, 0); var crcData = new byte[4]; LegacyCheckedRead(decompressionStream, crcData, sizeof(int)); var crc32 = BitConverter.ToUInt32(crcData, 0); memoryStream = this.memoryStreamManager.GetStream("PersistedDataReader", (int)length, true); memoryStream.SetLength(length); var buffer = memoryStream.GetBuffer(); LegacyCheckedRead(decompressionStream, buffer, length); var dataCRC = CRC32.Compute(buffer, length); if (crc32 != dataCRC) { throw new PersistedDataException(string.Format("CRC failed for data, expected {0} was {1}", crc32, dataCRC)); } PersistedDataHeader header; fixed(byte *buf = buffer) { header = new PersistedDataHeader(new BufferReader(buf, length)); } memoryStream.Dispose(); return(header); } } catch (Exception ex) { memoryStream?.Dispose(); if (ex is EndOfStreamException || ex is InvalidDataException) { throw new PersistedDataException("Stream data may be truncated", ex); } throw; } }
private unsafe PersistedDataHeader LoadHeader() { return(this.LoadAndValidateData((ms, buffer, length) => { // Do not do 'using' here because if an exception leaks we are // cleaned up elsewhere. PersistedDataHeader header; fixed(byte *buf = buffer) { header = new PersistedDataHeader(new BufferReader(buf, length)); } ms.Dispose(); return header; })); }