/// <summary>
/// Queues a sequence of bytes, from the specified data source, onto the stream for parsing.
/// </summary>
/// <param name="source">Identifier of the data source.</param>
/// <param name="buffer">An array of bytes. This method copies count bytes from buffer to the queue.</param>
/// <param name="offset">The zero-based byte offset in buffer at which to begin copying bytes to the current stream.</param>
/// <param name="count">The number of bytes to be written to the current stream.</param>
public override void Parse(SourceChannel source, byte[] buffer, int offset, int count)
{
// When SEL Fast Message is transmitted over Ethernet it is embedded in a Telnet stream. As a result
// any 0xFF will be encoded for Telnet compliance as a duplicate, i.e., 0xFF 0xFF. We remove these
// duplications when encountered to make sure check-sums and parsing work as expected.
int doubleFFPosition = buffer.IndexOfSequence(new byte[] { 0xFF, 0xFF }, offset, count);
while (doubleFFPosition > -1)
{
using (BlockAllocatedMemoryStream newBuffer = new BlockAllocatedMemoryStream())
{
// Write buffer before repeated byte
newBuffer.Write(buffer, offset, doubleFFPosition - offset + 1);
int nextByte = doubleFFPosition + 2;
// Write buffer after repeated byte, if any
if (nextByte < offset + count)
{
newBuffer.Write(buffer, nextByte, offset + count - nextByte);
}
buffer = newBuffer.ToArray();
}
offset = 0;
count = buffer.Length;
// Find next 0xFF 0xFF sequence
doubleFFPosition = buffer.IndexOfSequence(new byte[] { 0xFF, 0xFF }, offset, count);
}
base.Parse(source, buffer, offset, count);
}
// Rotates base time offsets
private void RotateBaseTimes()
{
if ((object)m_parent != null && (object)m_baseTimeRotationTimer != null)
{
if ((object)m_baseTimeOffsets == null)
{
m_baseTimeOffsets = new long[2];
m_baseTimeOffsets[0] = RealTime;
m_baseTimeOffsets[1] = RealTime + (long)m_baseTimeRotationTimer.Interval * Ticks.PerMillisecond;
m_timeIndex = 0;
}
else
{
int oldIndex = m_timeIndex;
// Switch to newer timestamp
m_timeIndex ^= 1;
// Now make older timestamp the newer timestamp
m_baseTimeOffsets[oldIndex] = RealTime + (long)m_baseTimeRotationTimer.Interval * Ticks.PerMillisecond;
}
// Since this function will only be called periodically, there is no real benefit
// to maintaining this memory stream at a member level
using (BlockAllocatedMemoryStream responsePacket = new BlockAllocatedMemoryStream())
{
responsePacket.Write(BigEndian.GetBytes(m_timeIndex), 0, 4);
responsePacket.Write(BigEndian.GetBytes(m_baseTimeOffsets[0]), 0, 8);
responsePacket.Write(BigEndian.GetBytes(m_baseTimeOffsets[1]), 0, 8);
m_parent.SendClientResponse(m_clientID, ServerResponse.UpdateBaseTimes, ServerCommand.Subscribe, responsePacket.ToArray());
}
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Create working buffer
using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream())
{
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.Synchronized;
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
workingBuffer.WriteByte((byte)flags);
// Serialize frame timestamp into data packet - this only occurs in synchronized data packets,
// unsynchronized subscriptions always include timestamps in the serialized measurements
workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8);
// Serialize total number of measurement values to follow
workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
if (usePayloadCompression && m_compressionModes.HasFlag(CompressionModes.TSSC))
{
throw new InvalidOperationException("TSSC must be processed at the frame level. Please check call stack - this is considered an error.");
}
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, false, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
workingBuffer.Seek(0, SeekOrigin.Begin);
workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray());
}
}
}
public void Test4()
{
MemoryStream ms = new MemoryStream();
BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream();
for (int x = 0; x < 10000; x++)
{
int value = Random.Int32;
ms.Write(value);
ms.Write((byte)value);
ms2.Write(value);
ms2.Write((byte)value);
}
for (int x = 0; x < 10000; x++)
{
long position = Random.Int64Between(0, ms.Length - 5);
ms.Position = position;
ms2.Position = position;
if (ms.ReadInt32() != ms2.ReadInt32())
{
throw new Exception();
}
if (ms.ReadNextByte() != ms2.ReadNextByte())
{
throw new Exception();
}
}
for (int x = 0; x < 10000; x++)
{
byte[] buffer1 = new byte[100];
byte[] buffer2 = new byte[100];
long position = Random.Int64Between(0, (long)(ms.Length * 1.1));
int readLength = Random.Int32Between(0, 100);
ms.Position = position;
ms2.Position = position;
if (ms.Read(buffer1, 99 - readLength, readLength) != ms2.Read(buffer2, 99 - readLength, readLength))
{
CompareBytes(buffer1, buffer2);
}
}
Compare(ms, ms2);
}
public void Test()
{
MemoryStream ms = new MemoryStream();
BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream();
for (int x = 0; x < 10000; x++)
{
int value = Random.Int32;
ms.Write(value);
ms.Write((byte)value);
ms2.Write(value);
ms2.Write((byte)value);
}
Compare(ms, ms2);
}
// Decompresses the given buffer based on the compression style and algorithm currently used by the parser.
// The result is placed back in the buffer that was sent to this method.
private void Decompress(ref byte[] buffer)
{
if (CompressionAlgorithm == CompressionAlgorithm.None || CompressionStyle == CompressionStyle.None)
{
return;
}
byte[] readBuffer = new byte[65536];
using (BlockAllocatedMemoryStream readStream = new BlockAllocatedMemoryStream())
{
int readAmount;
// Faster here to use provided buffer as non-expandable memory stream
using (MemoryStream bufferStream = new MemoryStream(buffer))
{
using (ZlibStream inflater = new ZlibStream(bufferStream, CompressionMode.Decompress))
{
do
{
readAmount = inflater.Read(readBuffer, 0, readBuffer.Length);
readStream.Write(readBuffer, 0, readAmount);
}while (readAmount != 0);
}
}
buffer = readStream.ToArray();
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Reset working buffer
m_workingBuffer.SetLength(0);
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.Synchronized;
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
m_workingBuffer.WriteByte((byte)flags);
// Serialize frame timestamp into data packet - this only occurs in synchronized data packets,
// unsynchronized subscriptions always include timestamps in the serialized measurements
m_workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8);
// Serialize total number of measurement values to follow
m_workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(m_workingBuffer, m_compressionStrength, false, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(m_workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
m_workingBuffer.Seek(0, SeekOrigin.Begin);
m_workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_workingBuffer, m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, m_workingBuffer.ToArray());
}
}
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, bool useCompactMeasurementFormat, bool usePayloadCompression)
{
// Create working buffer
using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream())
{
// Serialize data packet flags into response
DataPacketFlags flags = DataPacketFlags.NoFlags; // No flags means bit is cleared, i.e., unsynchronized
if (useCompactMeasurementFormat)
{
flags |= DataPacketFlags.Compact;
}
workingBuffer.WriteByte((byte)flags);
// No frame level timestamp is serialized into the data packet since all data is unsynchronized and essentially
// published upon receipt, however timestamps are optionally included in the serialized measurements.
// Serialize total number of measurement values to follow
workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4);
// Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization
if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, m_includeTime, ref flags))
{
// Serialize measurements to data buffer
foreach (IBinaryMeasurement measurement in measurements)
{
measurement.CopyBinaryImageToStream(workingBuffer);
}
}
// Update data packet flags if it has updated compression flags
if ((flags & DataPacketFlags.Compressed) > 0)
{
workingBuffer.Seek(0, SeekOrigin.Begin);
workingBuffer.WriteByte((byte)flags);
}
// Publish data packet to client
if ((object)m_parent != null)
{
m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray());
}
// Track last publication time
m_lastPublishTime = DateTime.UtcNow.Ticks;
}
}
public void Test3()
{
MemoryStream ms = new MemoryStream();
BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream();
for (int x = 0; x < 10000; x++)
{
long position = Random.Int64Between(0, 100000);
ms.Position = position;
ms2.Position = position;
int value = Random.Int32;
ms.Write(value);
ms2.Write(value);
long length = Random.Int64Between(100000 >> 1, 100000);
ms.SetLength(length);
ms2.SetLength(length);
}
Compare(ms, ms2);
}
public void Test2()
{
MemoryStream ms = new MemoryStream();
BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream();
for (int x = 0; x < 10000; x++)
{
int value = Random.Int32;
ms.Write(value);
ms2.Write(value);
int seek = Random.Int16Between(-10, 20);
if (ms.Position + seek < 0)
{
seek = -seek;
}
ms.Position += seek;
ms2.Position += seek;
}
Compare(ms, ms2);
}
/// <summary>
/// Combines an array of buffers together as a single image.
/// </summary>
/// <param name="buffers">Array of byte buffers.</param>
/// <returns>Combined buffers.</returns>
/// <exception cref="InvalidOperationException">Cannot create a byte array with more than 2,147,483,591 elements.</exception>
/// <remarks>
/// Only use this function if you need a copy of the combined buffers, it will be optimal
/// to use the Linq function <see cref="Enumerable.Concat{T}"/> if you simply need to
/// iterate over the combined buffers.
/// </remarks>
public static byte[] Combine(this byte[][] buffers)
{
if (buffers is null)
{
throw new ArgumentNullException(nameof(buffers));
}
using BlockAllocatedMemoryStream combinedBuffer = new BlockAllocatedMemoryStream();
// Combine all currently queued buffers
for (int x = 0; x < buffers.Length; x++)
{
if (buffers[x] is null)
{
throw new ArgumentNullException($"buffers[{x}]");
}
combinedBuffer.Write(buffers[x], 0, buffers[x].Length);
}
// return combined data buffers
return(combinedBuffer.ToArray());
}
/// <summary>
/// Combines an array of buffers together as a single image.
/// </summary>
/// <param name="buffers">Array of byte buffers.</param>
/// <returns>Combined buffers.</returns>
/// <exception cref="InvalidOperationException">Cannot create a byte array with more than 2,147,483,591 elements.</exception>
/// <remarks>
/// Only use this function if you need a copy of the combined buffers, it will be optimal
/// to use the Linq function <see cref="Enumerable.Concat{T}"/> if you simply need to
/// iterate over the combined buffers.
/// </remarks>
public static byte[] Combine(this byte[][] buffers)
{
if ((object)buffers == null)
{
throw new ArgumentNullException("buffers");
}
using (BlockAllocatedMemoryStream combinedBuffer = new BlockAllocatedMemoryStream())
{
// Combine all currently queued buffers
for (int x = 0; x < buffers.Length; x++)
{
if ((object)buffers[x] == null)
{
throw new ArgumentNullException("buffers[" + x + "]");
}
combinedBuffer.Write(buffers[x], 0, buffers[x].Length);
}
// return combined data buffers
return(combinedBuffer.ToArray());
}
}
/// <summary>
/// Rotates or initializes the crypto keys for this <see cref="ClientConnection"/>.
/// </summary>
public bool RotateCipherKeys()
{
// Make sure at least a second has passed before next key rotation
if ((DateTime.UtcNow.Ticks - m_lastCipherKeyUpdateTime).ToMilliseconds() >= 1000.0D)
{
try
{
// Since this function cannot be not called more than once per second there
// is no real benefit to maintaining these memory streams at a member level
using (BlockAllocatedMemoryStream response = new BlockAllocatedMemoryStream())
{
byte[] bytes, bufferLen;
// Create or update cipher keys and initialization vectors
UpdateKeyIVs();
// Add current cipher index to response
response.WriteByte((byte)m_cipherIndex);
// Serialize new keys
using (BlockAllocatedMemoryStream buffer = new BlockAllocatedMemoryStream())
{
// Write even key
bufferLen = BigEndian.GetBytes(m_keyIVs[EvenKey][KeyIndex].Length);
buffer.Write(bufferLen, 0, bufferLen.Length);
buffer.Write(m_keyIVs[EvenKey][KeyIndex], 0, m_keyIVs[EvenKey][KeyIndex].Length);
// Write even initialization vector
bufferLen = BigEndian.GetBytes(m_keyIVs[EvenKey][IVIndex].Length);
buffer.Write(bufferLen, 0, bufferLen.Length);
buffer.Write(m_keyIVs[EvenKey][IVIndex], 0, m_keyIVs[EvenKey][IVIndex].Length);
// Write odd key
bufferLen = BigEndian.GetBytes(m_keyIVs[OddKey][KeyIndex].Length);
buffer.Write(bufferLen, 0, bufferLen.Length);
buffer.Write(m_keyIVs[OddKey][KeyIndex], 0, m_keyIVs[OddKey][KeyIndex].Length);
// Write odd initialization vector
bufferLen = BigEndian.GetBytes(m_keyIVs[OddKey][IVIndex].Length);
buffer.Write(bufferLen, 0, bufferLen.Length);
buffer.Write(m_keyIVs[OddKey][IVIndex], 0, m_keyIVs[OddKey][IVIndex].Length);
// Get bytes from serialized buffer
bytes = buffer.ToArray();
}
// Encrypt keys using private keys known only to current client and server
if (m_authenticated && !string.IsNullOrWhiteSpace(m_sharedSecret))
{
bytes = bytes.Encrypt(m_sharedSecret, CipherStrength.Aes256);
}
// Add serialized key response
response.Write(bytes, 0, bytes.Length);
// Send cipher key updates
m_parent.SendClientResponse(m_clientID, ServerResponse.UpdateCipherKeys, ServerCommand.Subscribe, response.ToArray());
}
// Send success message
m_parent.SendClientResponse(m_clientID, ServerResponse.Succeeded, ServerCommand.RotateCipherKeys, "New cipher keys established.");
m_parent.OnStatusMessage(MessageLevel.Info, $"{ConnectionID} cipher keys rotated.");
return(true);
}
catch (Exception ex)
{
// Send failure message
m_parent.SendClientResponse(m_clientID, ServerResponse.Failed, ServerCommand.RotateCipherKeys, "Failed to establish new cipher keys: " + ex.Message);
m_parent.OnStatusMessage(MessageLevel.Warning, $"Failed to establish new cipher keys for {ConnectionID}: {ex.Message}");
return(false);
}
}
m_parent.SendClientResponse(m_clientID, ServerResponse.Failed, ServerCommand.RotateCipherKeys, "Cipher key rotation skipped, keys were already rotated within last second.");
m_parent.OnStatusMessage(MessageLevel.Warning, $"Cipher key rotation skipped for {ConnectionID}, keys were already rotated within last second.");
return(false);
}
/// <summary>
/// Attempts to compress payload of <see cref="CompactMeasurement"/> values onto the <paramref name="destination"/> stream.
/// </summary>
/// <param name="compactMeasurements">Payload of <see cref="CompactMeasurement"/> values.</param>
/// <param name="destination">Memory based <paramref name="destination"/> stream to hold compressed payload.</param>
/// <param name="compressionStrength">Compression strength to use.</param>
/// <param name="includeTime">Flag that determines if time should be included in the compressed payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns><c>true</c> if payload was compressed and encoded onto <paramref name="destination"/> stream; otherwise <c>false</c>.</returns>
/// <remarks>
/// <para>
/// Compressed payload will only be encoded onto <paramref name="destination"/> stream if compressed size would be smaller
/// than normal serialized size.
/// </para>
/// <para>
/// As an optimization this function uses a compression method that uses pointers to native structures, as such the
/// endian order encoding of the compressed data will always be in the native-endian order of the operating system.
/// This will be an important consideration when writing a endian order neutral payload decompressor. To help with
/// this the actual endian order used during compression is marked in the data flags. However, measurements values
/// are consistently encoded in big-endian order prior to buffer compression.
/// </para>
/// </remarks>
public static bool CompressPayload(this IEnumerable <CompactMeasurement> compactMeasurements, BlockAllocatedMemoryStream destination, byte compressionStrength, bool includeTime, ref DataPacketFlags flags)
{
// Instantiate a buffer that is larger than we'll need
byte[] buffer = new byte[ushort.MaxValue];
// Go ahead an enumerate all the measurements - this will cast all values to compact measurements
CompactMeasurement[] measurements = compactMeasurements.ToArray();
int measurementCount = measurements.Length;
int sizeToBeat = measurementCount * measurements[0].BinaryLength;
int index = 0;
// Encode compact state flags and runtime IDs together --
// Together these are three bytes, so we pad with a zero byte.
// The zero byte and state flags are considered to be more compressible
// than the runtime ID, so these are stored in the higher order bytes.
for (int i = 0; i < measurementCount; i++)
{
uint value = ((uint)measurements[i].CompactStateFlags << 16) | measurements[i].RuntimeID;
index += NativeEndianOrder.Default.CopyBytes(value, buffer, index);
}
// Encode values
for (int i = 0; i < measurementCount; i++)
{
// Encode using adjusted value (accounts for adder and multiplier)
index += NativeEndianOrder.Default.CopyBytes((float)measurements[i].AdjustedValue, buffer, index);
}
if (includeTime)
{
// Encode timestamps
for (int i = 0; i < measurementCount; i++)
{
// Since large majority of 8-byte tick values will be repeated, they should compress well
index += NativeEndianOrder.Default.CopyBytes((long)measurements[i].Timestamp, buffer, index);
}
}
// Attempt to compress buffer
int compressedSize = PatternCompressor.CompressBuffer(buffer, 0, index, ushort.MaxValue, compressionStrength);
// Only encode compressed buffer if compression actually helped payload size
if (compressedSize <= sizeToBeat)
{
// Set payload compression flag
flags |= DataPacketFlags.Compressed;
// Make sure decompressor knows original endian encoding order
if (BitConverter.IsLittleEndian)
{
flags |= DataPacketFlags.LittleEndianCompression;
}
else
{
flags &= ~DataPacketFlags.LittleEndianCompression;
}
// Copy compressed payload onto destination stream
destination.Write(buffer, 0, compressedSize);
return(true);
}
// Clear payload compression flag
flags &= ~DataPacketFlags.Compressed;
return(false);
}
/// <summary>
/// Writes a sequence of bytes onto the stream for parsing.
/// </summary>
/// <param name="source">Defines the source channel for the data.</param>
/// <param name="buffer">An array of bytes. This method copies count bytes from buffer to the current stream.</param>
/// <param name="offset">The zero-based byte offset in buffer at which to begin copying bytes to the current stream.</param>
/// <param name="count">The number of bytes to be written to the current stream.</param>
public override void Parse(SourceChannel source, byte[] buffer, int offset, int count)
{
// Since the Macrodyne implementation supports both 0xAA and 0xBB as sync-bytes, we must manually check for both during stream initialization,
// base class handles this only then there is a consistently defined set of sync-bytes, not variable.
if (Enabled)
{
// See if there are any 0xAA 0xAA sequences - these must be removed
int syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA, 0xAA }, offset, count);
while (syncBytePosition > -1)
{
using (BlockAllocatedMemoryStream newBuffer = new BlockAllocatedMemoryStream())
{
// Write buffer before repeated byte
newBuffer.Write(buffer, offset, syncBytePosition - offset + 1);
int nextByte = syncBytePosition + 2;
// Write buffer after repeated byte, if any
if (nextByte < offset + count)
{
newBuffer.Write(buffer, nextByte, offset + count - nextByte);
}
buffer = newBuffer.ToArray();
}
offset = 0;
count = buffer.Length;
// Find next 0xAA 0xAA sequence
syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA, 0xAA }, offset, count);
}
if (StreamInitialized)
{
base.Parse(source, buffer, offset, count);
}
else
{
// Initial stream may be anywhere in the middle of a frame, so we attempt to locate sync-bytes to "line-up" data stream,
// First we look for data frame sync-byte:
syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA }, offset, count);
if (syncBytePosition > -1)
{
StreamInitialized = true;
base.Parse(source, buffer, syncBytePosition, count - (syncBytePosition - offset));
}
else
{
// Second we look for command frame response sync-byte:
syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xBB }, offset, count);
if (syncBytePosition > -1)
{
StreamInitialized = true;
base.Parse(source, buffer, syncBytePosition, count - (syncBytePosition - offset));
}
}
}
}
}
