private void ProcessBinaryMeasurements(IEnumerable <ISupportBinaryImage> measurements, bool useCompactMeasurementFormat)
{
MemoryStream data = new MemoryStream();
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.NoFlags; // No flags means bit is cleared, i.e., unsynchronized
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
data.WriteByte((byte)flags);
// No frame level timestamp is serialized into the data packet since all data is unsynchronized and essentially
// published upon receipt, however timestamps are optionally included in the serialized measurements.
// Serialize total number of measurement values to follow
data.Write(EndianOrder.BigEndian.GetBytes(measurements.Count()), 0, 4);
// Serialize measurements to data buffer
foreach (ISupportBinaryImage measurement in measurements)
{
measurement.CopyBinaryImageToStream(data);
}
// Publish data packet to client
if (m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, data.ToArray());
}
// Track last publication time
m_lastPublishTime = DateTime.UtcNow.Ticks;
}
private void ProcessBinaryMeasurements(IEnumerable <ISupportBinaryImage> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat)
{
MemoryStream data = new MemoryStream();
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.Synchronized;
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
data.WriteByte((byte)flags);
// Serialize frame timestamp into data packet - this only occurs in synchronized data packets,
// unsynchronized subcriptions always include timestamps in the serialized measurements
data.Write(EndianOrder.BigEndian.GetBytes(frameLevelTimestamp), 0, 8);
// Serialize total number of measurement values to follow
data.Write(EndianOrder.BigEndian.GetBytes(measurements.Count()), 0, 4);
// Serialize measurements to data buffer
foreach (ISupportBinaryImage measurement in measurements)
{
measurement.CopyBinaryImageToStream(data);
}
// Publish data packet to client
if (m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, data.ToArray());
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Create working buffer
using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream())
{
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.Synchronized;
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
workingBuffer.WriteByte((byte)flags);
// Serialize frame timestamp into data packet - this only occurs in synchronized data packets,
// unsynchronized subscriptions always include timestamps in the serialized measurements
workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8);
// Serialize total number of measurement values to follow
workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
if (usePayloadCompression && m_compressionModes.HasFlag(CompressionModes.TSSC))
{
throw new InvalidOperationException("TSSC must be processed at the frame level. Please check call stack - this is considered an error.");
}
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, false, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
workingBuffer.Seek(0, SeekOrigin.Begin);
workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray());
}
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Create working buffer
using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream())
{
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.NoFlags; // No flags means bit is cleared, i.e., unsynchronized
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
workingBuffer.WriteByte((byte)flags);
// No frame level timestamp is serialized into the data packet since all data is unsynchronized and essentially
// published upon receipt, however timestamps are optionally included in the serialized measurements.
// Serialize total number of measurement values to follow
workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, m_includeTime, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
workingBuffer.Seek(0, SeekOrigin.Begin);
workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray());
}
// Track last publication time
m_lastPublishTime = DateTime.UtcNow.Ticks;
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Reset working buffer
m_workingBuffer.SetLength(0);
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.Synchronized;
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
m_workingBuffer.WriteByte((byte)flags);
// Serialize frame timestamp into data packet - this only occurs in synchronized data packets,
// unsynchronized subscriptions always include timestamps in the serialized measurements
m_workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8);
// Serialize total number of measurement values to follow
m_workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(m_workingBuffer, m_compressionStrength, false, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(m_workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
m_workingBuffer.Seek(0, SeekOrigin.Begin);
m_workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_workingBuffer, m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, m_workingBuffer.ToArray());
}
}
/// <summary>
/// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
/// </summary>
/// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
/// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
/// <param name="index">Index into buffer where compressed payload begins.</param>
/// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
/// <param name="measurementCount">Number of compressed measurements in the payload.</param>
/// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
{
CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];
// Actual data length has to take into account response byte and in-response-to server command byte in the payload header
//int dataLength = length - index - 2;
int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);
// Copy source data into a decompression buffer
byte[] buffer = new byte[bufferLength];
Buffer.BlockCopy(source, index, buffer, 0, dataLength);
// Check that OS endian-order matches endian-order of compressed data
if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
{
// can be modified to decompress a payload that is in a non-native Endian order
throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
}
// Attempt to decompress buffer
int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);
if (uncompressedSize == 0)
{
throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
}
index = 0;
// Decode ID and state flags
for (int i = 0; i < measurementCount; i++)
{
uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);
measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
{
CompactStateFlags = (byte)(value >> 16),
RuntimeID = (ushort)value
};
index += 4;
}
// Decode values
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
index += 4;
}
if (includeTime)
{
// Decode timestamps
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
index += 8;
}
}
return(measurements);
}
/// <summary>
/// Attempts to compress payload of <see cref="CompactMeasurement"/> values onto the <paramref name="destination"/> stream.
/// </summary>
/// <param name="compactMeasurements">Payload of <see cref="CompactMeasurement"/> values.</param>
/// <param name="destination">Memory based <paramref name="destination"/> stream to hold compressed payload.</param>
/// <param name="compressionStrength">Compression strength to use.</param>
/// <param name="includeTime">Flag that determines if time should be included in the compressed payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns><c>true</c> if payload was compressed and encoded onto <paramref name="destination"/> stream; otherwise <c>false</c>.</returns>
/// <remarks>
/// <para>
/// Compressed payload will only be encoded onto <paramref name="destination"/> stream if compressed size would be smaller
/// than normal serialized size.
/// </para>
/// <para>
/// As an optimization this function uses a compression method that uses pointers to native structures, as such the
/// endian order encoding of the compressed data will always be in the native-endian order of the operating system.
/// This will be an important consideration when writing a endian order neutral payload decompressor. To help with
/// this the actual endian order used during compression is marked in the data flags. However, measurements values
/// are consistently encoded in big-endian order prior to buffer compression.
/// </para>
/// </remarks>
public static bool CompressPayload(this IEnumerable <CompactMeasurement> compactMeasurements, BlockAllocatedMemoryStream destination, byte compressionStrength, bool includeTime, ref DataPacketFlags flags)
{
// Instantiate a buffer that is larger than we'll need
byte[] buffer = new byte[ushort.MaxValue];
// Go ahead an enumerate all the measurements - this will cast all values to compact measurements
CompactMeasurement[] measurements = compactMeasurements.ToArray();
int measurementCount = measurements.Length;
int sizeToBeat = measurementCount * measurements[0].BinaryLength;
int index = 0;
// Encode compact state flags and runtime IDs together --
// Together these are three bytes, so we pad with a zero byte.
// The zero byte and state flags are considered to be more compressible
// than the runtime ID, so these are stored in the higher order bytes.
for (int i = 0; i < measurementCount; i++)
{
uint value = ((uint)measurements[i].CompactStateFlags << 16) | measurements[i].RuntimeID;
index += NativeEndianOrder.Default.CopyBytes(value, buffer, index);
}
// Encode values
for (int i = 0; i < measurementCount; i++)
{
// Encode using adjusted value (accounts for adder and multiplier)
index += NativeEndianOrder.Default.CopyBytes((float)measurements[i].AdjustedValue, buffer, index);
}
if (includeTime)
{
// Encode timestamps
for (int i = 0; i < measurementCount; i++)
{
// Since large majority of 8-byte tick values will be repeated, they should compress well
index += NativeEndianOrder.Default.CopyBytes((long)measurements[i].Timestamp, buffer, index);
}
}
// Attempt to compress buffer
int compressedSize = PatternCompressor.CompressBuffer(buffer, 0, index, ushort.MaxValue, compressionStrength);
// Only encode compressed buffer if compression actually helped payload size
if (compressedSize <= sizeToBeat)
{
// Set payload compression flag
flags |= DataPacketFlags.Compressed;
// Make sure decompressor knows original endian encoding order
if (BitConverter.IsLittleEndian)
{
flags |= DataPacketFlags.LittleEndianCompression;
}
else
{
flags &= ~DataPacketFlags.LittleEndianCompression;
}
// Copy compressed payload onto destination stream
destination.Write(buffer, 0, compressedSize);
return(true);
}
// Clear payload compression flag
flags &= ~DataPacketFlags.Compressed;
return(false);
}
/// <summary>
/// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
/// </summary>
/// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
/// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
/// <param name="index">Index into buffer where compressed payload begins.</param>
/// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
/// <param name="measurementCount">Number of compressed measurements in the payload.</param>
/// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
{
CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];
byte[] buffer = null;
try
{
// Actual data length has to take into account response byte and in-response-to server command byte in the payload header
//int dataLength = length - index - 2;
int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);
// Copy source data into a decompression buffer
buffer = BufferPool.TakeBuffer(bufferLength);
Buffer.BlockCopy(source, index, buffer, 0, dataLength);
// Check that OS endian-order matches endian-order of compressed data
if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
{
// TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it
// can be modified to decompress a payload that is non-native Endian order
throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
}
// Attempt to decompress buffer
int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);
if (uncompressedSize == 0)
throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
index = 0;
// Decode ID and state flags
for (int i = 0; i < measurementCount; i++)
{
uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);
measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
{
CompactStateFlags = (byte)(value >> 16),
RuntimeID = (ushort)value
};
index += 4;
}
// Decode values
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
index += 4;
}
if (includeTime)
{
// Decode timestamps
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
index += 8;
}
}
}
finally
{
if ((object)buffer != null)
BufferPool.ReturnBuffer(buffer);
}
return measurements;
}
/// <summary>
/// Attempts to compress payload of <see cref="CompactMeasurement"/> values onto the <paramref name="destination"/> stream.
/// </summary>
/// <param name="compactMeasurements">Payload of <see cref="CompactMeasurement"/> values.</param>
/// <param name="destination">Memory based <paramref name="destination"/> stream to hold compressed payload.</param>
/// <param name="compressionStrength">Compression strength to use.</param>
/// <param name="includeTime">Flag that determines if time should be included in the compressed payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns><c>true</c> if payload was compressed and encoded onto <paramref name="destination"/> stream; otherwise <c>false</c>.</returns>
/// <remarks>
/// <para>
/// Compressed payload will only be encoded onto <paramref name="destination"/> stream if compressed size would be smaller
/// than normal serialized size.
/// </para>
/// <para>
/// As an optimization this function uses a compression method that uses pointers to native structures, as such the
/// endian order encoding of the compressed data will always be in the native-endian order of the operating system.
/// This will be an important consideration when writing a endian order neutral payload decompressor. To help with
/// this the actual endian order used during compression is marked in the data flags. However, measurements values
/// are consistently encoded in big-endian order prior to buffer compression.
/// </para>
/// </remarks>
public static bool CompressPayload(this IEnumerable<CompactMeasurement> compactMeasurements, MemoryStream destination, byte compressionStrength, bool includeTime, ref DataPacketFlags flags)
{
byte[] buffer = null;
try
{
// Get a buffer that is larger than we'll need
buffer = BufferPool.TakeBuffer(ushort.MaxValue);
// Go ahead an enumerate all the measurements - this will cast all values to compact measurements
CompactMeasurement[] measurements = compactMeasurements.ToArray();
int measurementCount = measurements.Length;
int sizeToBeat = measurementCount * measurements[0].BinaryLength;
int index = 0;
// Encode compact state flags and runtime IDs together --
// Together these are three bytes, so we pad with a zero byte.
// The zero byte and state flags are considered to be more compressible
// than the runtime ID, so these are stored in the higher order bytes.
for (int i = 0; i < measurementCount; i++)
{
uint value = ((uint)measurements[i].CompactStateFlags << 16) | measurements[i].RuntimeID;
index += NativeEndianOrder.Default.CopyBytes(value, buffer, index);
}
// Encode values
for (int i = 0; i < measurementCount; i++)
{
// Encode using adjusted value (accounts for adder and multiplier)
index += NativeEndianOrder.Default.CopyBytes((float)measurements[i].AdjustedValue, buffer, index);
}
if (includeTime)
{
// Encode timestamps
for (int i = 0; i < measurementCount; i++)
{
// Since large majority of 8-byte tick values will be repeated, they should compress well
index += NativeEndianOrder.Default.CopyBytes((long)measurements[i].Timestamp, buffer, index);
}
}
// Attempt to compress buffer
int compressedSize = PatternCompressor.CompressBuffer(buffer, 0, index, ushort.MaxValue, compressionStrength);
// Only encode compressed buffer if compression actually helped payload size
if (compressedSize <= sizeToBeat)
{
// Set payload compression flag
flags |= DataPacketFlags.Compressed;
// Make sure decompressor knows original endian encoding order
if (BitConverter.IsLittleEndian)
flags |= DataPacketFlags.LittleEndianCompression;
else
flags &= ~DataPacketFlags.LittleEndianCompression;
// Copy compressed payload onto destination stream
destination.Write(buffer, 0, compressedSize);
return true;
}
// Clear payload compression flag
flags &= ~DataPacketFlags.Compressed;
return false;
}
finally
{
if ((object)buffer != null)
BufferPool.ReturnBuffer(buffer);
}
}