/// <summary> /// Queues a sequence of bytes, from the specified data source, onto the stream for parsing. /// </summary> /// <param name="source">Identifier of the data source.</param> /// <param name="buffer">An array of bytes. This method copies count bytes from buffer to the queue.</param> /// <param name="offset">The zero-based byte offset in buffer at which to begin copying bytes to the current stream.</param> /// <param name="count">The number of bytes to be written to the current stream.</param> public override void Parse(SourceChannel source, byte[] buffer, int offset, int count) { // When SEL Fast Message is transmitted over Ethernet it is embedded in a Telnet stream. As a result // any 0xFF will be encoded for Telnet compliance as a duplicate, i.e., 0xFF 0xFF. We remove these // duplications when encountered to make sure check-sums and parsing work as expected. int doubleFFPosition = buffer.IndexOfSequence(new byte[] { 0xFF, 0xFF }, offset, count); while (doubleFFPosition > -1) { using (BlockAllocatedMemoryStream newBuffer = new BlockAllocatedMemoryStream()) { // Write buffer before repeated byte newBuffer.Write(buffer, offset, doubleFFPosition - offset + 1); int nextByte = doubleFFPosition + 2; // Write buffer after repeated byte, if any if (nextByte < offset + count) { newBuffer.Write(buffer, nextByte, offset + count - nextByte); } buffer = newBuffer.ToArray(); } offset = 0; count = buffer.Length; // Find next 0xFF 0xFF sequence doubleFFPosition = buffer.IndexOfSequence(new byte[] { 0xFF, 0xFF }, offset, count); } base.Parse(source, buffer, offset, count); }
// Rotates base time offsets private void RotateBaseTimes() { if ((object)m_parent != null && (object)m_baseTimeRotationTimer != null) { if ((object)m_baseTimeOffsets == null) { m_baseTimeOffsets = new long[2]; m_baseTimeOffsets[0] = RealTime; m_baseTimeOffsets[1] = RealTime + (long)m_baseTimeRotationTimer.Interval * Ticks.PerMillisecond; m_timeIndex = 0; } else { int oldIndex = m_timeIndex; // Switch to newer timestamp m_timeIndex ^= 1; // Now make older timestamp the newer timestamp m_baseTimeOffsets[oldIndex] = RealTime + (long)m_baseTimeRotationTimer.Interval * Ticks.PerMillisecond; } // Since this function will only be called periodically, there is no real benefit // to maintaining this memory stream at a member level using (BlockAllocatedMemoryStream responsePacket = new BlockAllocatedMemoryStream()) { responsePacket.Write(BigEndian.GetBytes(m_timeIndex), 0, 4); responsePacket.Write(BigEndian.GetBytes(m_baseTimeOffsets[0]), 0, 8); responsePacket.Write(BigEndian.GetBytes(m_baseTimeOffsets[1]), 0, 8); m_parent.SendClientResponse(m_clientID, ServerResponse.UpdateBaseTimes, ServerCommand.Subscribe, responsePacket.ToArray()); } } }
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression) { // Create working buffer using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream()) { // Serialize data packet flags into response DataPacketFlags flags = DataPacketFlags.Synchronized; if (useCompactMeasurementFormat) { flags |= DataPacketFlags.Compact; } workingBuffer.WriteByte((byte)flags); // Serialize frame timestamp into data packet - this only occurs in synchronized data packets, // unsynchronized subscriptions always include timestamps in the serialized measurements workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8); // Serialize total number of measurement values to follow workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4); if (usePayloadCompression && m_compressionModes.HasFlag(CompressionModes.TSSC)) { throw new InvalidOperationException("TSSC must be processed at the frame level. Please check call stack - this is considered an error."); } // Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, false, ref flags)) { // Serialize measurements to data buffer foreach (IBinaryMeasurement measurement in measurements) { measurement.CopyBinaryImageToStream(workingBuffer); } } // Update data packet flags if it has updated compression flags if ((flags & DataPacketFlags.Compressed) > 0) { workingBuffer.Seek(0, SeekOrigin.Begin); workingBuffer.WriteByte((byte)flags); } // Publish data packet to client if ((object)m_parent != null) { m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray()); } } }
public void Test4() { MemoryStream ms = new MemoryStream(); BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream(); for (int x = 0; x < 10000; x++) { int value = Random.Int32; ms.Write(value); ms.Write((byte)value); ms2.Write(value); ms2.Write((byte)value); } for (int x = 0; x < 10000; x++) { long position = Random.Int64Between(0, ms.Length - 5); ms.Position = position; ms2.Position = position; if (ms.ReadInt32() != ms2.ReadInt32()) { throw new Exception(); } if (ms.ReadNextByte() != ms2.ReadNextByte()) { throw new Exception(); } } for (int x = 0; x < 10000; x++) { byte[] buffer1 = new byte[100]; byte[] buffer2 = new byte[100]; long position = Random.Int64Between(0, (long)(ms.Length * 1.1)); int readLength = Random.Int32Between(0, 100); ms.Position = position; ms2.Position = position; if (ms.Read(buffer1, 99 - readLength, readLength) != ms2.Read(buffer2, 99 - readLength, readLength)) { CompareBytes(buffer1, buffer2); } } Compare(ms, ms2); }
public void Test() { MemoryStream ms = new MemoryStream(); BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream(); for (int x = 0; x < 10000; x++) { int value = Random.Int32; ms.Write(value); ms.Write((byte)value); ms2.Write(value); ms2.Write((byte)value); } Compare(ms, ms2); }
// Decompresses the given buffer based on the compression style and algorithm currently used by the parser. // The result is placed back in the buffer that was sent to this method. private void Decompress(ref byte[] buffer) { if (CompressionAlgorithm == CompressionAlgorithm.None || CompressionStyle == CompressionStyle.None) { return; } byte[] readBuffer = new byte[65536]; using (BlockAllocatedMemoryStream readStream = new BlockAllocatedMemoryStream()) { int readAmount; // Faster here to use provided buffer as non-expandable memory stream using (MemoryStream bufferStream = new MemoryStream(buffer)) { using (ZlibStream inflater = new ZlibStream(bufferStream, CompressionMode.Decompress)) { do { readAmount = inflater.Read(readBuffer, 0, readBuffer.Length); readStream.Write(readBuffer, 0, readAmount); }while (readAmount != 0); } } buffer = readStream.ToArray(); } }
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, long frameLevelTimestamp, bool useCompactMeasurementFormat, bool usePayloadCompression) { // Reset working buffer m_workingBuffer.SetLength(0); // Serialize data packet flags into response DataPacketFlags flags = DataPacketFlags.Synchronized; if (useCompactMeasurementFormat) { flags |= DataPacketFlags.Compact; } m_workingBuffer.WriteByte((byte)flags); // Serialize frame timestamp into data packet - this only occurs in synchronized data packets, // unsynchronized subscriptions always include timestamps in the serialized measurements m_workingBuffer.Write(BigEndian.GetBytes(frameLevelTimestamp), 0, 8); // Serialize total number of measurement values to follow m_workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4); // Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(m_workingBuffer, m_compressionStrength, false, ref flags)) { // Serialize measurements to data buffer foreach (IBinaryMeasurement measurement in measurements) { measurement.CopyBinaryImageToStream(m_workingBuffer); } } // Update data packet flags if it has updated compression flags if ((flags & DataPacketFlags.Compressed) > 0) { m_workingBuffer.Seek(0, SeekOrigin.Begin); m_workingBuffer.WriteByte((byte)flags); } // Publish data packet to client if ((object)m_parent != null) { m_parent.SendClientResponse(m_workingBuffer, m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, m_workingBuffer.ToArray()); } }
private void ProcessBinaryMeasurements(IEnumerable <IBinaryMeasurement> measurements, bool useCompactMeasurementFormat, bool usePayloadCompression) { // Create working buffer using (BlockAllocatedMemoryStream workingBuffer = new BlockAllocatedMemoryStream()) { // Serialize data packet flags into response DataPacketFlags flags = DataPacketFlags.NoFlags; // No flags means bit is cleared, i.e., unsynchronized if (useCompactMeasurementFormat) { flags |= DataPacketFlags.Compact; } workingBuffer.WriteByte((byte)flags); // No frame level timestamp is serialized into the data packet since all data is unsynchronized and essentially // published upon receipt, however timestamps are optionally included in the serialized measurements. // Serialize total number of measurement values to follow workingBuffer.Write(BigEndian.GetBytes(measurements.Count()), 0, 4); // Attempt compression when requested - encoding of compressed buffer only happens if size would be smaller than normal serialization if (!usePayloadCompression || !measurements.Cast <CompactMeasurement>().CompressPayload(workingBuffer, m_compressionStrength, m_includeTime, ref flags)) { // Serialize measurements to data buffer foreach (IBinaryMeasurement measurement in measurements) { measurement.CopyBinaryImageToStream(workingBuffer); } } // Update data packet flags if it has updated compression flags if ((flags & DataPacketFlags.Compressed) > 0) { workingBuffer.Seek(0, SeekOrigin.Begin); workingBuffer.WriteByte((byte)flags); } // Publish data packet to client if ((object)m_parent != null) { m_parent.SendClientResponse(m_clientID, ServerResponse.DataPacket, ServerCommand.Subscribe, workingBuffer.ToArray()); } // Track last publication time m_lastPublishTime = DateTime.UtcNow.Ticks; } }
public void Test3() { MemoryStream ms = new MemoryStream(); BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream(); for (int x = 0; x < 10000; x++) { long position = Random.Int64Between(0, 100000); ms.Position = position; ms2.Position = position; int value = Random.Int32; ms.Write(value); ms2.Write(value); long length = Random.Int64Between(100000 >> 1, 100000); ms.SetLength(length); ms2.SetLength(length); } Compare(ms, ms2); }
public void Test2() { MemoryStream ms = new MemoryStream(); BlockAllocatedMemoryStream ms2 = new BlockAllocatedMemoryStream(); for (int x = 0; x < 10000; x++) { int value = Random.Int32; ms.Write(value); ms2.Write(value); int seek = Random.Int16Between(-10, 20); if (ms.Position + seek < 0) { seek = -seek; } ms.Position += seek; ms2.Position += seek; } Compare(ms, ms2); }
/// <summary> /// Combines an array of buffers together as a single image. /// </summary> /// <param name="buffers">Array of byte buffers.</param> /// <returns>Combined buffers.</returns> /// <exception cref="InvalidOperationException">Cannot create a byte array with more than 2,147,483,591 elements.</exception> /// <remarks> /// Only use this function if you need a copy of the combined buffers, it will be optimal /// to use the Linq function <see cref="Enumerable.Concat{T}"/> if you simply need to /// iterate over the combined buffers. /// </remarks> public static byte[] Combine(this byte[][] buffers) { if (buffers is null) { throw new ArgumentNullException(nameof(buffers)); } using BlockAllocatedMemoryStream combinedBuffer = new BlockAllocatedMemoryStream(); // Combine all currently queued buffers for (int x = 0; x < buffers.Length; x++) { if (buffers[x] is null) { throw new ArgumentNullException($"buffers[{x}]"); } combinedBuffer.Write(buffers[x], 0, buffers[x].Length); } // return combined data buffers return(combinedBuffer.ToArray()); }
/// <summary> /// Combines an array of buffers together as a single image. /// </summary> /// <param name="buffers">Array of byte buffers.</param> /// <returns>Combined buffers.</returns> /// <exception cref="InvalidOperationException">Cannot create a byte array with more than 2,147,483,591 elements.</exception> /// <remarks> /// Only use this function if you need a copy of the combined buffers, it will be optimal /// to use the Linq function <see cref="Enumerable.Concat{T}"/> if you simply need to /// iterate over the combined buffers. /// </remarks> public static byte[] Combine(this byte[][] buffers) { if ((object)buffers == null) { throw new ArgumentNullException("buffers"); } using (BlockAllocatedMemoryStream combinedBuffer = new BlockAllocatedMemoryStream()) { // Combine all currently queued buffers for (int x = 0; x < buffers.Length; x++) { if ((object)buffers[x] == null) { throw new ArgumentNullException("buffers[" + x + "]"); } combinedBuffer.Write(buffers[x], 0, buffers[x].Length); } // return combined data buffers return(combinedBuffer.ToArray()); } }
/// <summary> /// Rotates or initializes the crypto keys for this <see cref="ClientConnection"/>. /// </summary> public bool RotateCipherKeys() { // Make sure at least a second has passed before next key rotation if ((DateTime.UtcNow.Ticks - m_lastCipherKeyUpdateTime).ToMilliseconds() >= 1000.0D) { try { // Since this function cannot be not called more than once per second there // is no real benefit to maintaining these memory streams at a member level using (BlockAllocatedMemoryStream response = new BlockAllocatedMemoryStream()) { byte[] bytes, bufferLen; // Create or update cipher keys and initialization vectors UpdateKeyIVs(); // Add current cipher index to response response.WriteByte((byte)m_cipherIndex); // Serialize new keys using (BlockAllocatedMemoryStream buffer = new BlockAllocatedMemoryStream()) { // Write even key bufferLen = BigEndian.GetBytes(m_keyIVs[EvenKey][KeyIndex].Length); buffer.Write(bufferLen, 0, bufferLen.Length); buffer.Write(m_keyIVs[EvenKey][KeyIndex], 0, m_keyIVs[EvenKey][KeyIndex].Length); // Write even initialization vector bufferLen = BigEndian.GetBytes(m_keyIVs[EvenKey][IVIndex].Length); buffer.Write(bufferLen, 0, bufferLen.Length); buffer.Write(m_keyIVs[EvenKey][IVIndex], 0, m_keyIVs[EvenKey][IVIndex].Length); // Write odd key bufferLen = BigEndian.GetBytes(m_keyIVs[OddKey][KeyIndex].Length); buffer.Write(bufferLen, 0, bufferLen.Length); buffer.Write(m_keyIVs[OddKey][KeyIndex], 0, m_keyIVs[OddKey][KeyIndex].Length); // Write odd initialization vector bufferLen = BigEndian.GetBytes(m_keyIVs[OddKey][IVIndex].Length); buffer.Write(bufferLen, 0, bufferLen.Length); buffer.Write(m_keyIVs[OddKey][IVIndex], 0, m_keyIVs[OddKey][IVIndex].Length); // Get bytes from serialized buffer bytes = buffer.ToArray(); } // Encrypt keys using private keys known only to current client and server if (m_authenticated && !string.IsNullOrWhiteSpace(m_sharedSecret)) { bytes = bytes.Encrypt(m_sharedSecret, CipherStrength.Aes256); } // Add serialized key response response.Write(bytes, 0, bytes.Length); // Send cipher key updates m_parent.SendClientResponse(m_clientID, ServerResponse.UpdateCipherKeys, ServerCommand.Subscribe, response.ToArray()); } // Send success message m_parent.SendClientResponse(m_clientID, ServerResponse.Succeeded, ServerCommand.RotateCipherKeys, "New cipher keys established."); m_parent.OnStatusMessage(MessageLevel.Info, $"{ConnectionID} cipher keys rotated."); return(true); } catch (Exception ex) { // Send failure message m_parent.SendClientResponse(m_clientID, ServerResponse.Failed, ServerCommand.RotateCipherKeys, "Failed to establish new cipher keys: " + ex.Message); m_parent.OnStatusMessage(MessageLevel.Warning, $"Failed to establish new cipher keys for {ConnectionID}: {ex.Message}"); return(false); } } m_parent.SendClientResponse(m_clientID, ServerResponse.Failed, ServerCommand.RotateCipherKeys, "Cipher key rotation skipped, keys were already rotated within last second."); m_parent.OnStatusMessage(MessageLevel.Warning, $"Cipher key rotation skipped for {ConnectionID}, keys were already rotated within last second."); return(false); }
/// <summary> /// Attempts to compress payload of <see cref="CompactMeasurement"/> values onto the <paramref name="destination"/> stream. /// </summary> /// <param name="compactMeasurements">Payload of <see cref="CompactMeasurement"/> values.</param> /// <param name="destination">Memory based <paramref name="destination"/> stream to hold compressed payload.</param> /// <param name="compressionStrength">Compression strength to use.</param> /// <param name="includeTime">Flag that determines if time should be included in the compressed payload.</param> /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param> /// <returns><c>true</c> if payload was compressed and encoded onto <paramref name="destination"/> stream; otherwise <c>false</c>.</returns> /// <remarks> /// <para> /// Compressed payload will only be encoded onto <paramref name="destination"/> stream if compressed size would be smaller /// than normal serialized size. /// </para> /// <para> /// As an optimization this function uses a compression method that uses pointers to native structures, as such the /// endian order encoding of the compressed data will always be in the native-endian order of the operating system. /// This will be an important consideration when writing a endian order neutral payload decompressor. To help with /// this the actual endian order used during compression is marked in the data flags. However, measurements values /// are consistently encoded in big-endian order prior to buffer compression. /// </para> /// </remarks> public static bool CompressPayload(this IEnumerable <CompactMeasurement> compactMeasurements, BlockAllocatedMemoryStream destination, byte compressionStrength, bool includeTime, ref DataPacketFlags flags) { // Instantiate a buffer that is larger than we'll need byte[] buffer = new byte[ushort.MaxValue]; // Go ahead an enumerate all the measurements - this will cast all values to compact measurements CompactMeasurement[] measurements = compactMeasurements.ToArray(); int measurementCount = measurements.Length; int sizeToBeat = measurementCount * measurements[0].BinaryLength; int index = 0; // Encode compact state flags and runtime IDs together -- // Together these are three bytes, so we pad with a zero byte. // The zero byte and state flags are considered to be more compressible // than the runtime ID, so these are stored in the higher order bytes. for (int i = 0; i < measurementCount; i++) { uint value = ((uint)measurements[i].CompactStateFlags << 16) | measurements[i].RuntimeID; index += NativeEndianOrder.Default.CopyBytes(value, buffer, index); } // Encode values for (int i = 0; i < measurementCount; i++) { // Encode using adjusted value (accounts for adder and multiplier) index += NativeEndianOrder.Default.CopyBytes((float)measurements[i].AdjustedValue, buffer, index); } if (includeTime) { // Encode timestamps for (int i = 0; i < measurementCount; i++) { // Since large majority of 8-byte tick values will be repeated, they should compress well index += NativeEndianOrder.Default.CopyBytes((long)measurements[i].Timestamp, buffer, index); } } // Attempt to compress buffer int compressedSize = PatternCompressor.CompressBuffer(buffer, 0, index, ushort.MaxValue, compressionStrength); // Only encode compressed buffer if compression actually helped payload size if (compressedSize <= sizeToBeat) { // Set payload compression flag flags |= DataPacketFlags.Compressed; // Make sure decompressor knows original endian encoding order if (BitConverter.IsLittleEndian) { flags |= DataPacketFlags.LittleEndianCompression; } else { flags &= ~DataPacketFlags.LittleEndianCompression; } // Copy compressed payload onto destination stream destination.Write(buffer, 0, compressedSize); return(true); } // Clear payload compression flag flags &= ~DataPacketFlags.Compressed; return(false); }
/// <summary> /// Writes a sequence of bytes onto the stream for parsing. /// </summary> /// <param name="source">Defines the source channel for the data.</param> /// <param name="buffer">An array of bytes. This method copies count bytes from buffer to the current stream.</param> /// <param name="offset">The zero-based byte offset in buffer at which to begin copying bytes to the current stream.</param> /// <param name="count">The number of bytes to be written to the current stream.</param> public override void Parse(SourceChannel source, byte[] buffer, int offset, int count) { // Since the Macrodyne implementation supports both 0xAA and 0xBB as sync-bytes, we must manually check for both during stream initialization, // base class handles this only then there is a consistently defined set of sync-bytes, not variable. if (Enabled) { // See if there are any 0xAA 0xAA sequences - these must be removed int syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA, 0xAA }, offset, count); while (syncBytePosition > -1) { using (BlockAllocatedMemoryStream newBuffer = new BlockAllocatedMemoryStream()) { // Write buffer before repeated byte newBuffer.Write(buffer, offset, syncBytePosition - offset + 1); int nextByte = syncBytePosition + 2; // Write buffer after repeated byte, if any if (nextByte < offset + count) { newBuffer.Write(buffer, nextByte, offset + count - nextByte); } buffer = newBuffer.ToArray(); } offset = 0; count = buffer.Length; // Find next 0xAA 0xAA sequence syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA, 0xAA }, offset, count); } if (StreamInitialized) { base.Parse(source, buffer, offset, count); } else { // Initial stream may be anywhere in the middle of a frame, so we attempt to locate sync-bytes to "line-up" data stream, // First we look for data frame sync-byte: syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xAA }, offset, count); if (syncBytePosition > -1) { StreamInitialized = true; base.Parse(source, buffer, syncBytePosition, count - (syncBytePosition - offset)); } else { // Second we look for command frame response sync-byte: syncBytePosition = buffer.IndexOfSequence(new byte[] { 0xBB }, offset, count); if (syncBytePosition > -1) { StreamInitialized = true; base.Parse(source, buffer, syncBytePosition, count - (syncBytePosition - offset)); } } } } }