private void ProcessMeasurements(IEnumerable <IMeasurement> measurements)
{
List <ISupportBinaryImage> packet = new List <ISupportBinaryImage>();
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat;
int packetSize = 5;
// Wait for any external events, if needed
WaitForExternalEvents();
// If a set of base times has not yet been initialized, initialize a set by rotating
if (!m_initializedBaseTimeOffsets)
{
if (m_parent.UseBaseTimeOffsets)
{
RotateBaseTimes();
}
m_initializedBaseTimeOffsets = true;
}
foreach (IMeasurement measurement in measurements)
{
ISupportBinaryImage binaryMeasurement;
int binaryLength;
// Serialize the current measurement.
if (useCompactMeasurementFormat)
{
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
}
else
{
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(m_clientID));
}
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on the max
// packet size, process the current packet and start a new packet.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat);
packet.Clear();
packetSize = 5;
}
// Add the current measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
// Process the remaining measurements.
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat);
}
/// <summary>
/// Publish <see cref="IFrame"/> of time-aligned collection of <see cref="IMeasurement"/> values that arrived within the
/// concentrator's defined <see cref="ConcentratorBase.LagTime"/>.
/// </summary>
/// <param name="frame"><see cref="IFrame"/> of measurements with the same timestamp that arrived within <see cref="ConcentratorBase.LagTime"/> that are ready for processing.</param>
/// <param name="index">Index of <see cref="IFrame"/> within a second ranging from zero to <c><see cref="ConcentratorBase.FramesPerSecond"/> - 1</c>.</param>
protected override void PublishFrame(IFrame frame, int index)
{
if (!m_disposed)
{
List <ISupportBinaryImage> packet = new List <ISupportBinaryImage>();
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat;
long frameLevelTimestamp = frame.Timestamp;
int packetSize = 13;
foreach (IMeasurement measurement in frame.Measurements.Values)
{
ISupportBinaryImage binaryMeasurement;
int binaryLength;
// Serialize the current measurement.
if (useCompactMeasurementFormat)
{
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, false, null, 0, false);
}
else
{
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(ClientID));
}
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on
// the max packet size, process the packet and start a new one.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat);
packet.Clear();
packetSize = 13;
}
// Add the measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
// Process the remaining measurements.
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat);
}
}
/// <summary>
/// Publish <see cref="IFrame"/> of time-aligned collection of <see cref="IMeasurement"/> values that arrived within the
/// concentrator's defined <see cref="ConcentratorBase.LagTime"/>.
/// </summary>
/// <param name="frame"><see cref="IFrame"/> of measurements with the same timestamp that arrived within <see cref="ConcentratorBase.LagTime"/> that are ready for processing.</param>
/// <param name="index">Index of <see cref="IFrame"/> within a second ranging from zero to <c><see cref="ConcentratorBase.FramesPerSecond"/> - 1</c>.</param>
protected override void PublishFrame(IFrame frame, int index)
{
if ((object)m_parent == null || m_disposed)
return;
// Includes data packet flags, frame level timestamp and measurement count
const int PacketHeaderSize = DataPublisher.ClientResponseHeaderSize + 13;
List<IBinaryMeasurement> packet = new List<IBinaryMeasurement>();
bool usePayloadCompression = m_usePayloadCompression;
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat || usePayloadCompression;
long frameLevelTimestamp = frame.Timestamp;
BufferBlockMeasurement bufferBlockMeasurement;
IBinaryMeasurement binaryMeasurement;
byte[] bufferBlock;
int binaryLength;
int packetSize = PacketHeaderSize;
long publishTime;
foreach (IMeasurement measurement in frame.Measurements.Values)
{
bufferBlockMeasurement = measurement as BufferBlockMeasurement;
if ((object)bufferBlockMeasurement != null)
{
// Still sending buffer block measurements to client; we are expecting
// confirmations which will indicate whether retransmission is necessary,
// so we will restart the retransmission timer
m_bufferBlockRetransmissionTimer.Stop();
// Handle buffer block measurements as a special case - this can be any kind of data,
// measurement subscriber will need to know how to interpret buffer
bufferBlock = new byte[4 + bufferBlockMeasurement.Length];
// Prepend sequence number
BigEndian.CopyBytes(m_bufferBlockSequenceNumber, bufferBlock, 0);
m_bufferBlockSequenceNumber++;
// Append measurement data and send
Buffer.BlockCopy(bufferBlockMeasurement.Buffer, 0, bufferBlock, 4, bufferBlockMeasurement.Length);
m_parent.SendClientResponse(m_clientID, ServerResponse.BufferBlock, ServerCommand.Subscribe, bufferBlock);
lock (m_bufferBlockCacheLock)
{
// Cache buffer block for retransmission
m_bufferBlockCache.Add(bufferBlock);
}
// Start the retransmission timer in case we never receive a confirmation
m_bufferBlockRetransmissionTimer.Start();
}
else
{
// Serialize the current measurement.
if (useCompactMeasurementFormat)
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, false);
else
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(ClientID));
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on
// the max packet size, process the packet and start a new one.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression);
packet.Clear();
packetSize = PacketHeaderSize;
}
// Add the measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
}
// Process the remaining measurements.
if (packet.Count > 0)
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression);
// Update latency statistics
publishTime = DateTime.UtcNow.Ticks;
m_parent.UpdateLatencyStatistics(frame.Measurements.Values.Select(m => (long)(publishTime - m.Timestamp)));
}
/// <summary>
/// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
/// </summary>
/// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
/// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
/// <param name="index">Index into buffer where compressed payload begins.</param>
/// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
/// <param name="measurementCount">Number of compressed measurements in the payload.</param>
/// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
{
CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];
// Actual data length has to take into account response byte and in-response-to server command byte in the payload header
//int dataLength = length - index - 2;
int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);
// Copy source data into a decompression buffer
byte[] buffer = new byte[bufferLength];
Buffer.BlockCopy(source, index, buffer, 0, dataLength);
// Check that OS endian-order matches endian-order of compressed data
if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
{
// can be modified to decompress a payload that is in a non-native Endian order
throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
}
// Attempt to decompress buffer
int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);
if (uncompressedSize == 0)
{
throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
}
index = 0;
// Decode ID and state flags
for (int i = 0; i < measurementCount; i++)
{
uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);
measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
{
CompactStateFlags = (byte)(value >> 16),
RuntimeID = (ushort)value
};
index += 4;
}
// Decode values
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
index += 4;
}
if (includeTime)
{
// Decode timestamps
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
index += 8;
}
}
return(measurements);
}
private void ProcessMeasurements(IEnumerable <IMeasurement> measurements)
{
// Includes data packet flags and measurement count
const int PacketHeaderSize = DataPublisher.ClientResponseHeaderSize + 5;
List <IBinaryMeasurement> packet;
int packetSize;
bool usePayloadCompression;
bool useCompactMeasurementFormat;
BufferBlockMeasurement bufferBlockMeasurement;
byte[] bufferBlock;
ushort bufferBlockSignalIndex;
IBinaryMeasurement binaryMeasurement;
int binaryLength;
try
{
if (!Enabled)
{
return;
}
packet = new List <IBinaryMeasurement>();
packetSize = PacketHeaderSize;
usePayloadCompression = m_usePayloadCompression;
useCompactMeasurementFormat = m_useCompactMeasurementFormat || usePayloadCompression;
foreach (IMeasurement measurement in measurements)
{
bufferBlockMeasurement = measurement as BufferBlockMeasurement;
if ((object)bufferBlockMeasurement != null)
{
// Still sending buffer block measurements to client; we are expecting
// confirmations which will indicate whether retransmission is necessary,
// so we will restart the retransmission timer
m_bufferBlockRetransmissionTimer.Stop();
// Handle buffer block measurements as a special case - this can be any kind of data,
// measurement subscriber will need to know how to interpret buffer
bufferBlock = new byte[6 + bufferBlockMeasurement.Length];
// Prepend sequence number
BigEndian.CopyBytes(m_bufferBlockSequenceNumber, bufferBlock, 0);
m_bufferBlockSequenceNumber++;
// Copy signal index into buffer
bufferBlockSignalIndex = m_signalIndexCache.GetSignalIndex(bufferBlockMeasurement.ID);
BigEndian.CopyBytes(bufferBlockSignalIndex, bufferBlock, 4);
// Append measurement data and send
Buffer.BlockCopy(bufferBlockMeasurement.Buffer, 0, bufferBlock, 6, bufferBlockMeasurement.Length);
m_parent.SendClientResponse(m_workingBuffer, m_clientID, ServerResponse.BufferBlock, ServerCommand.Subscribe, bufferBlock);
lock (m_bufferBlockCacheLock)
{
// Cache buffer block for retransmission
m_bufferBlockCache.Add(bufferBlock);
}
// Start the retransmission timer in case we never receive a confirmation
m_bufferBlockRetransmissionTimer.Start();
}
else
{
// Serialize the current measurement.
if (useCompactMeasurementFormat)
{
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
}
else
{
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(m_clientID));
}
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on the max
// packet size, process the current packet and start a new packet.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat, usePayloadCompression);
packet.Clear();
packetSize = PacketHeaderSize;
}
// Add the current measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
}
// Process the remaining measurements.
if (packet.Count > 0)
{
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat, usePayloadCompression);
}
IncrementProcessedMeasurements(measurements.Count());
// Update latency statistics
m_parent.UpdateLatencyStatistics(measurements.Select(m => (long)(m_lastPublishTime - m.Timestamp)));
}
catch (Exception ex)
{
string message = string.Format("Error processing measurements: {0}", ex.Message);
OnProcessException(new InvalidOperationException(message, ex));
}
}
private void ProcessServerResponse(byte[] buffer, int length)
{
// Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing
// is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread...
if (buffer != null && length > 0)
{
try
{
Dictionary<Guid, DeviceStatisticsHelper<SubscribedDevice>> subscribedDevicesLookup;
DeviceStatisticsHelper<SubscribedDevice> statisticsHelper;
ServerResponse responseCode = (ServerResponse)buffer[0];
ServerCommand commandCode = (ServerCommand)buffer[1];
int responseLength = BigEndian.ToInt32(buffer, 2);
int responseIndex = DataPublisher.ClientResponseHeaderSize;
bool solicited = false;
byte[][][] keyIVs;
// See if this was a solicited response to a requested server command
if (responseCode.IsSolicited())
{
lock (m_requests)
{
int index = m_requests.BinarySearch(commandCode);
if (index >= 0)
{
solicited = true;
m_requests.RemoveAt(index);
}
}
// Disconnect any established UDP data channel upon successful unsubscribe
if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded)
DataChannel = null;
}
OnReceivedServerResponse(responseCode, commandCode);
switch (responseCode)
{
case ServerResponse.Succeeded:
if (solicited)
{
switch (commandCode)
{
case ServerCommand.Authenticate:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_authenticated = true;
OnConnectionAuthenticated();
break;
case ServerCommand.Subscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = true;
break;
case ServerCommand.Unsubscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = false;
if ((object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = false;
break;
case ServerCommand.RotateCipherKeys:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.MetaDataRefresh:
OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
m_metadataRefreshPending = false;
break;
}
}
else
{
switch (commandCode)
{
case ServerCommand.MetaDataRefresh:
// Meta-data refresh may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
m_metadataRefreshPending = false;
break;
case ServerCommand.RotateCipherKeys:
// Key rotation may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.Subscribe:
OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
default:
OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode));
break;
}
}
break;
case ServerResponse.Failed:
if (solicited)
OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
else
OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode));
if (commandCode == ServerCommand.MetaDataRefresh)
m_metadataRefreshPending = false;
break;
case ServerResponse.DataPacket:
long now = DateTime.UtcNow.Ticks;
// Deserialize data packet
List<IMeasurement> measurements = new List<IMeasurement>();
DataPacketFlags flags;
Ticks timestamp = 0;
int count;
if (m_totalBytesReceived == 0)
{
// At the point when data is being received, data monitor should be enabled
if ((object)m_dataStreamMonitor != null && !m_dataStreamMonitor.Enabled)
m_dataStreamMonitor.Enabled = true;
// Establish run-time log for subscriber
if (m_autoConnect || m_dataGapRecoveryEnabled)
{
if ((object)m_runTimeLog == null)
{
m_runTimeLog = new RunTimeLog();
m_runTimeLog.FileName = GetLoggingPath(Name + "_RunTimeLog.txt");
m_runTimeLog.ProcessException += m_runTimeLog_ProcessException;
m_runTimeLog.Initialize();
}
else
{
// Mark the start of any data transmissions
m_runTimeLog.StartTime = DateTime.UtcNow;
m_runTimeLog.Enabled = true;
}
}
// The duration between last disconnection and start of data transmissions
// represents a gap in data - if data gap recovery is enabled, we log
// this as a gap for recovery:
if (m_dataGapRecoveryEnabled && (object)m_dataGapRecoverer != null)
m_dataGapRecoverer.LogDataGap(m_runTimeLog.StopTime, DateTime.UtcNow);
}
// Track total data packet bytes received from any channel
m_totalBytesReceived += m_lastBytesReceived;
m_monitoredBytesReceived += m_lastBytesReceived;
// Get data packet flags
flags = (DataPacketFlags)buffer[responseIndex];
responseIndex++;
bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0);
bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0);
bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0);
int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0;
// Decrypt data packet payload if keys are available
if ((object)m_keyIVs != null)
{
// Get a local copy of volatile keyIVs reference since this can change at any time
keyIVs = m_keyIVs;
// Decrypt payload portion of data packet
buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]);
responseIndex = 0;
responseLength = buffer.Length;
}
// Synchronized packets contain a frame level timestamp
if (synchronizedMeasurements)
{
timestamp = BigEndian.ToInt64(buffer, responseIndex);
responseIndex += 8;
}
// Deserialize number of measurements that follow
count = BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
if (compressedPayload)
{
if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
{
if (m_lastMissingCacheWarning != 0L)
{
// Warning message for missing signal index cache
OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
}
m_lastMissingCacheWarning = now;
}
else
{
try
{
if (CompressionModes.HasFlag(CompressionModes.TSSC))
{
// Use TSSC compression to decompress measurements
if ((object)m_decompressionBlock == null)
m_decompressionBlock = new MeasurementDecompressionBlock();
MemoryStream bufferStream = new MemoryStream(buffer, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize);
bool eos = false;
while (!eos)
{
Measurement measurement;
Tuple<Guid, string, uint> tuple;
ushort id;
long time;
uint quality;
float value;
byte command;
switch (m_decompressionBlock.GetMeasurement(out id, out time, out quality, out value, out command))
{
case DecompressionExitCode.EndOfStreamOccured:
if (bufferStream.Position != bufferStream.Length)
m_decompressionBlock.Fill(bufferStream);
else
eos = true;
break;
case DecompressionExitCode.CommandRead:
break;
case DecompressionExitCode.MeasurementRead:
// Attempt to restore signal identification
if (m_signalIndexCache.Reference.TryGetValue(id, out tuple))
{
measurement = new Measurement();
measurement.Key = MeasurementKey.LookUpOrCreate(tuple.Item1, tuple.Item2, tuple.Item3);
measurement.Timestamp = time;
measurement.StateFlags = (MeasurementStateFlags)quality;
measurement.Value = value;
measurements.Add(measurement);
}
break;
}
}
}
else
{
// Decompress compact measurements from payload
measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags));
}
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex));
}
}
}
else
{
// Deserialize measurements
for (int i = 0; i < count; i++)
{
if (!compactMeasurementFormat)
{
// Deserialize full measurement format
SerializableMeasurement measurement = new SerializableMeasurement(m_encoding);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
measurements.Add(measurement);
}
else if ((object)m_signalIndexCache != null)
{
// Deserialize compact measurement format
CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
// Apply timestamp from frame if not included in transmission
if (!measurement.IncludeTime)
measurement.Timestamp = timestamp;
measurements.Add(measurement);
}
else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
{
if (m_lastMissingCacheWarning != 0L)
{
// Warning message for missing signal index cache
OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
}
m_lastMissingCacheWarning = now;
}
}
}
// Calculate statistics
subscribedDevicesLookup = m_subscribedDevicesLookup;
statisticsHelper = null;
if ((object)subscribedDevicesLookup != null)
{
IEnumerable<IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement>> deviceGroups = measurements
.Where(measurement => subscribedDevicesLookup.TryGetValue(measurement.ID, out statisticsHelper))
.Select(measurement => Tuple.Create(statisticsHelper, measurement))
.ToList()
.GroupBy(tuple => tuple.Item1, tuple => tuple.Item2);
foreach (IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement> deviceGroup in deviceGroups)
{
statisticsHelper = deviceGroup.Key;
foreach (IGrouping<Ticks, IMeasurement> frame in deviceGroup.GroupBy(measurement => measurement.Timestamp))
{
// Determine the number of measurements received with valid values
int measurementsReceived = frame.Count(measurement => !double.IsNaN(measurement.Value));
IMeasurement statusFlags = null;
IMeasurement frequency = null;
IMeasurement deltaFrequency = null;
// Attempt to update real-time
if (!m_useLocalClockAsRealTime && frame.Key > m_realTime)
m_realTime = frame.Key;
// Search the frame for status flags, frequency, and delta frequency
foreach (IMeasurement measurement in frame)
{
if (measurement.ID == statisticsHelper.Device.StatusFlagsID)
statusFlags = measurement;
else if (measurement.ID == statisticsHelper.Device.FrequencyID)
frequency = measurement;
else if (measurement.ID == statisticsHelper.Device.DeltaFrequencyID)
deltaFrequency = measurement;
}
// If we are receiving status flags for this device,
// count the data quality, time quality, and device errors
if ((object)statusFlags != null)
{
uint commonStatusFlags = (uint)statusFlags.Value;
if ((commonStatusFlags & (uint)Bits.Bit19) > 0)
statisticsHelper.Device.DataQualityErrors++;
if ((commonStatusFlags & (uint)Bits.Bit18) > 0)
statisticsHelper.Device.TimeQualityErrors++;
if ((commonStatusFlags & (uint)Bits.Bit16) > 0)
statisticsHelper.Device.DeviceErrors++;
measurementsReceived--;
}
// Zero is not a valid value for frequency.
// If frequency is zero, invalidate both frequency and delta frequency
if ((object)frequency != null && frequency.Value == 0.0D)
{
if ((object)deltaFrequency != null)
measurementsReceived -= 2;
else
measurementsReceived--;
}
// Track the number of measurements received
statisticsHelper.AddToMeasurementsReceived(measurementsReceived);
}
}
}
// Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer
if ((object)m_localConcentrator != null)
m_localConcentrator.SortMeasurements(measurements);
else
OnNewMeasurements(measurements);
// Gather statistics on received data
DateTime timeReceived = RealTime;
if (!m_useLocalClockAsRealTime && timeReceived.Ticks - m_lastStatisticsHelperUpdate > Ticks.PerSecond)
{
UpdateStatisticsHelpers();
m_lastStatisticsHelperUpdate = m_realTime;
}
m_lifetimeMeasurements += measurements.Count;
UpdateMeasurementsPerSecond(timeReceived, measurements.Count);
for (int x = 0; x < measurements.Count; x++)
{
long latency = timeReceived.Ticks - (long)measurements[x].Timestamp;
// Throw out latencies that exceed one hour as invalid
if (Math.Abs(latency) > Time.SecondsPerHour * Ticks.PerSecond)
continue;
if (m_lifetimeMinimumLatency > latency || m_lifetimeMinimumLatency == 0)
m_lifetimeMinimumLatency = latency;
if (m_lifetimeMaximumLatency < latency || m_lifetimeMaximumLatency == 0)
m_lifetimeMaximumLatency = latency;
m_lifetimeTotalLatency += latency;
m_lifetimeLatencyMeasurements++;
}
break;
case ServerResponse.BufferBlock:
// Buffer block received - wrap as a buffer block measurement and expose back to consumer
uint sequenceNumber = BigEndian.ToUInt32(buffer, responseIndex);
int cacheIndex = (int)(sequenceNumber - m_expectedBufferBlockSequenceNumber);
BufferBlockMeasurement bufferBlockMeasurement;
Tuple<Guid, string, uint> measurementKey;
ushort signalIndex;
// Check if this buffer block has already been processed (e.g., mistaken retransmission due to timeout)
if (cacheIndex >= 0 && (cacheIndex >= m_bufferBlockCache.Count || (object)m_bufferBlockCache[cacheIndex] == null))
{
// Send confirmation that buffer block is received
SendServerCommand(ServerCommand.ConfirmBufferBlock, buffer.BlockCopy(responseIndex, 4));
// Get measurement key from signal index cache
signalIndex = BigEndian.ToUInt16(buffer, responseIndex + 4);
if (!m_signalIndexCache.Reference.TryGetValue(signalIndex, out measurementKey))
throw new InvalidOperationException("Failed to find associated signal identification for runtime ID " + signalIndex);
// Skip the sequence number and signal index when creating the buffer block measurement
bufferBlockMeasurement = new BufferBlockMeasurement(buffer, responseIndex + 6, responseLength - 6)
{
Key = MeasurementKey.LookUpOrCreate(measurementKey.Item1, measurementKey.Item2, measurementKey.Item3)
};
// Determine if this is the next buffer block in the sequence
if (sequenceNumber == m_expectedBufferBlockSequenceNumber)
{
List<IMeasurement> bufferBlockMeasurements = new List<IMeasurement>();
int i;
// Add the buffer block measurement to the list of measurements to be published
bufferBlockMeasurements.Add(bufferBlockMeasurement);
m_expectedBufferBlockSequenceNumber++;
// Add cached buffer block measurements to the list of measurements to be published
for (i = 1; i < m_bufferBlockCache.Count; i++)
{
if ((object)m_bufferBlockCache[i] == null)
break;
bufferBlockMeasurements.Add(m_bufferBlockCache[i]);
m_expectedBufferBlockSequenceNumber++;
}
// Remove published measurements from the buffer block queue
if (m_bufferBlockCache.Count > 0)
m_bufferBlockCache.RemoveRange(0, i);
// Publish measurements
OnNewMeasurements(bufferBlockMeasurements);
}
else
{
// Ensure that the list has at least as many
// elements as it needs to cache this measurement
for (int i = m_bufferBlockCache.Count; i <= cacheIndex; i++)
m_bufferBlockCache.Add(null);
// Insert this buffer block into the proper location in the list
m_bufferBlockCache[cacheIndex] = bufferBlockMeasurement;
}
}
m_lifetimeMeasurements += 1;
UpdateMeasurementsPerSecond(DateTime.UtcNow, 1);
break;
case ServerResponse.DataStartTime:
// Raise data start time event
OnDataStartTime(BigEndian.ToInt64(buffer, responseIndex));
break;
case ServerResponse.ProcessingComplete:
// Raise input processing completed event
OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerResponse.UpdateSignalIndexCache:
// Deserialize new signal index cache
m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength));
m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
FixExpectedMeasurementCounts();
break;
case ServerResponse.UpdateBaseTimes:
// Get active time index
m_timeIndex = BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
// Deserialize new base time offsets
m_baseTimeOffsets = new[] { BigEndian.ToInt64(buffer, responseIndex), BigEndian.ToInt64(buffer, responseIndex + 8) };
break;
case ServerResponse.UpdateCipherKeys:
// Move past active cipher index (not currently used anywhere else)
responseIndex++;
// Extract remaining response
byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1);
// Decrypt response payload if subscription is authenticated
if (m_authenticated)
bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256);
// Deserialize new cipher keys
keyIVs = new byte[2][][];
keyIVs[EvenKey] = new byte[2][];
keyIVs[OddKey] = new byte[2][];
int index = 0;
int bufferLen;
// Read even key size
bufferLen = BigEndian.ToInt32(bytes, index);
index += 4;
// Read even key
keyIVs[EvenKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read even initialization vector size
bufferLen = BigEndian.ToInt32(bytes, index);
index += 4;
// Read even initialization vector
keyIVs[EvenKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen);
index += bufferLen;
// Read odd key size
bufferLen = BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd key
keyIVs[OddKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read odd initialization vector size
bufferLen = BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd initialization vector
keyIVs[OddKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen);
//index += bufferLen;
// Exchange keys
m_keyIVs = keyIVs;
OnStatusMessage("Successfully established new cipher keys for data packet transmissions.");
break;
case ServerResponse.Notify:
// Skip the 4-byte hash
string message = m_encoding.GetString(buffer, responseIndex + 4, responseLength - 4);
// Display notification
OnStatusMessage("NOTIFICATION: {0}", message);
OnNotificationReceived(message);
// Send confirmation of receipt of the notification
SendServerCommand(ServerCommand.ConfirmNotification, buffer.BlockCopy(responseIndex, 4));
break;
case ServerResponse.ConfigurationChanged:
OnStatusMessage("Received notification from publisher that configuration has changed.");
OnServerConfigurationChanged();
// Initiate meta-data refresh when publisher configuration has changed - we only do this
// for automatic connections since API style connections have to manually initiate a
// meta-data refresh. API style connection should attach to server configuration changed
// event and request meta-data refresh to complete automated cycle.
if (m_autoConnect && m_autoSynchronizeMetadata)
SendServerCommand(ServerCommand.MetaDataRefresh, m_metadataFilters);
break;
}
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex));
}
}
}
/// <summary>
/// Publish <see cref="IFrame"/> of time-aligned collection of <see cref="IMeasurement"/> values that arrived within the
/// concentrator's defined <see cref="ConcentratorBase.LagTime"/>.
/// </summary>
/// <param name="frame"><see cref="IFrame"/> of measurements with the same timestamp that arrived within <see cref="ConcentratorBase.LagTime"/> that are ready for processing.</param>
/// <param name="index">Index of <see cref="IFrame"/> within a second ranging from zero to <c><see cref="ConcentratorBase.FramesPerSecond"/> - 1</c>.</param>
protected override void PublishFrame(IFrame frame, int index)
{
if (!m_disposed)
{
List<ISupportBinaryImage> packet = new List<ISupportBinaryImage>();
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat;
long frameLevelTimestamp = frame.Timestamp;
int packetSize = 13;
foreach (IMeasurement measurement in frame.Measurements.Values)
{
ISupportBinaryImage binaryMeasurement;
int binaryLength;
// Serialize the current measurement.
if (useCompactMeasurementFormat)
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, false, null, 0, false);
else
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(ClientID));
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on
// the max packet size, process the packet and start a new one.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat);
packet.Clear();
packetSize = 13;
}
// Add the measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
// Process the remaining measurements.
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat);
}
}
/// <summary>
/// Publish <see cref="IFrame"/> of time-aligned collection of <see cref="IMeasurement"/> values that arrived within the
/// concentrator's defined <see cref="ConcentratorBase.LagTime"/>.
/// </summary>
/// <param name="frame"><see cref="IFrame"/> of measurements with the same timestamp that arrived within <see cref="ConcentratorBase.LagTime"/> that are ready for processing.</param>
/// <param name="index">Index of <see cref="IFrame"/> within a second ranging from zero to <c><see cref="ConcentratorBase.FramesPerSecond"/> - 1</c>.</param>
protected override void PublishFrame(IFrame frame, int index)
{
if ((object)m_parent == null || m_disposed)
{
return;
}
if (m_usePayloadCompression && m_compressionModes.HasFlag(CompressionModes.TSSC))
{
ProcessTSSCMeasurements(frame);
return;
}
// Includes data packet flags, frame level timestamp and measurement count
const int PacketHeaderSize = DataPublisher.ClientResponseHeaderSize + 13;
List <IBinaryMeasurement> packet = new List <IBinaryMeasurement>();
bool usePayloadCompression = m_usePayloadCompression;
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat || usePayloadCompression;
long frameLevelTimestamp = frame.Timestamp;
BufferBlockMeasurement bufferBlockMeasurement;
IBinaryMeasurement binaryMeasurement;
byte[] bufferBlock;
int binaryLength;
int packetSize = PacketHeaderSize;
long publishTime;
foreach (IMeasurement measurement in frame.Measurements.Values)
{
bufferBlockMeasurement = measurement as BufferBlockMeasurement;
if ((object)bufferBlockMeasurement != null)
{
// Still sending buffer block measurements to client; we are expecting
// confirmations which will indicate whether retransmission is necessary,
// so we will restart the retransmission timer
m_bufferBlockRetransmissionTimer.Stop();
// Handle buffer block measurements as a special case - this can be any kind of data,
// measurement subscriber will need to know how to interpret buffer
bufferBlock = new byte[4 + bufferBlockMeasurement.Length];
// Prepend sequence number
BigEndian.CopyBytes(m_bufferBlockSequenceNumber, bufferBlock, 0);
m_bufferBlockSequenceNumber++;
// Append measurement data and send
Buffer.BlockCopy(bufferBlockMeasurement.Buffer, 0, bufferBlock, 4, bufferBlockMeasurement.Length);
m_parent.SendClientResponse(m_clientID, ServerResponse.BufferBlock, ServerCommand.Subscribe, bufferBlock);
lock (m_bufferBlockCacheLock)
{
// Cache buffer block for retransmission
m_bufferBlockCache.Add(bufferBlock);
}
// Start the retransmission timer in case we never receive a confirmation
m_bufferBlockRetransmissionTimer.Start();
}
else
{
// Serialize the current measurement.
if (useCompactMeasurementFormat)
{
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, false);
}
else
{
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(ClientID));
}
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on
// the max packet size, process the packet and start a new one.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression);
packet.Clear();
packetSize = PacketHeaderSize;
}
// Add the measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
}
// Process the remaining measurements.
if (packet.Count > 0)
{
ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression);
}
// Update latency statistics
publishTime = DateTime.UtcNow.Ticks;
m_parent.UpdateLatencyStatistics(frame.Measurements.Values.Select(m => (long)(publishTime - m.Timestamp)));
}
// Retrieves the measurements from the database.
private void GetDbMeasurements(object state)
{
IDbConnection connection = null;
// Get measurements from the database.
try
{
SignalIndexCache signalIndexCache = new SignalIndexCache();
CompactMeasurement measurement;
long startTime = DateTime.UtcNow.Ticks;
if (m_cacheFileName != null && File.Exists(m_cacheFileName))
{
OnStatusMessage("Loading cached input data...");
try
{
using (FileStream data = File.OpenRead(m_cacheFileName))
{
byte[] buffer = new byte[4];
int signalIndexCacheImageSize;
int compactMeasurementSize;
int totalMeasurements;
// Read the signal index cache image size from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
signalIndexCacheImageSize = LittleEndian.ToInt32(buffer, 0);
// Resize buffer to accomodate exact signal index cache
buffer = new byte[signalIndexCacheImageSize];
// Read the signal index cache image from the file
if (data.Read(buffer, 0, signalIndexCacheImageSize) != signalIndexCacheImageSize)
throw new EndOfStreamException();
// Deserialize the signal index cache
signalIndexCache = Serialization.Deserialize<SignalIndexCache>(buffer, SerializationFormat.Binary);
// Read the size of each compact measurement from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
compactMeasurementSize = LittleEndian.ToInt32(buffer, 0);
// Read the total number of compact measurements from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
totalMeasurements = LittleEndian.ToInt32(buffer, 0);
// Resize buffer to accomodate compact measurement if needed (not likely)
if (buffer.Length < compactMeasurementSize)
buffer = new byte[compactMeasurementSize];
// Read each compact measurement image from the file
for (int i = 0; i < totalMeasurements; i++)
{
if (data.Read(buffer, 0, compactMeasurementSize) != compactMeasurementSize)
throw new EndOfStreamException();
// Parse compact measurement
measurement = new CompactMeasurement(signalIndexCache);
measurement.ParseBinaryImage(buffer, 0, compactMeasurementSize);
m_dbMeasurements.Add(measurement);
if (m_dbMeasurements.Count % 50000 == 0)
OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
}
OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2));
}
}
catch (Exception ex)
{
if (ex is EndOfStreamException)
throw (EndOfStreamException)ex;
throw new EndOfStreamException(ex.Message, ex);
}
}
else
{
OnStatusMessage("Loading database input data...");
const string MeasurementTable = "ActiveMeasurements";
Dictionary<string, string> dataProviderSettings = m_dataProviderString.ParseKeyValuePairs();
Assembly assm = Assembly.Load(dataProviderSettings["AssemblyName"]);
Type connectionType = assm.GetType(dataProviderSettings["ConnectionType"]);
Dictionary<Guid, MeasurementKey> lookupCache = new Dictionary<Guid, MeasurementKey>();
IDbCommand command;
IDataReader dbReader;
MeasurementKey key;
Guid id;
ushort index = 0;
connection = (IDbConnection)Activator.CreateInstance(connectionType);
connection.ConnectionString = m_dbConnectionString;
connection.Open();
command = connection.CreateCommand();
command.CommandText = string.Format("SELECT * FROM {0}", m_dbTableName);
using (dbReader = command.ExecuteReader())
{
while (dbReader.Read())
{
measurement = new CompactMeasurement(signalIndexCache);
foreach (string fieldName in m_fieldNames.Keys)
{
object value = dbReader[fieldName];
string propertyName = m_fieldNames[fieldName];
switch (propertyName)
{
case "Timestamp":
// If the value is a timestamp, use the timestamp format
// specified by the user when reading the timestamp.
if (m_timestampFormat == null)
measurement.Timestamp = long.Parse(value.ToNonNullString());
else
measurement.Timestamp = DateTime.ParseExact(value.ToNonNullString(), m_timestampFormat, CultureInfo.CurrentCulture);
break;
case "ID":
if (Guid.TryParse(value.ToString(), out id))
{
if (!lookupCache.TryGetValue(id, out key))
{
if (DataSource.Tables.Contains(MeasurementTable))
{
DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("SignalID = '{0}'", id));
if (filteredRows.Length > 0)
key = MeasurementKey.LookUpOrCreate(id, filteredRows[0]["ID"].ToString());
}
if (key != MeasurementKey.Undefined)
{
// Cache measurement key associated with ID
lookupCache[id] = key;
// Assign a runtime index optimization for distinct measurements
signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(id, key.Source, key.ID));
}
}
measurement.Key = key;
}
break;
case "Key":
if (MeasurementKey.TryParse(value.ToString(), out key))
{
if (!lookupCache.ContainsKey(key.SignalID))
{
// Cache measurement key associated with ID
lookupCache[key.SignalID] = key;
// Assign a runtime index optimization for distinct measurements
signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(key.SignalID, key.Source, key.ID));
}
measurement.Key = key;
}
break;
case "Value":
measurement.Value = Convert.ToDouble(value);
break;
default:
PropertyInfo property = GetAllProperties(typeof(IMeasurement)).FirstOrDefault(propertyInfo => propertyInfo.Name == propertyName);
if (property != null)
{
Type propertyType = property.PropertyType;
Type valueType = value.GetType();
if (property.PropertyType.IsAssignableFrom(value.GetType()))
{
property.SetValue(measurement, value, null);
}
else if (property.PropertyType == typeof(string))
{
property.SetValue(measurement, value.ToNonNullString(), null);
}
else if (valueType == typeof(string))
{
MethodInfo parseMethod = propertyType.GetMethod("Parse", new[] { typeof(string) });
if (parseMethod != null && parseMethod.IsStatic)
property.SetValue(measurement, parseMethod.Invoke(null, new[] { value }), null);
}
else
{
string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1}.", fieldName, propertyName);
OnProcessException(new InvalidCastException(exceptionMessage));
}
}
else
{
string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1} - no property match was found.", fieldName, propertyName);
OnProcessException(new InvalidCastException(exceptionMessage));
}
break;
}
m_dbMeasurements.Add(measurement);
if (m_dbMeasurements.Count % 50000 == 0)
OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
}
}
}
OnStatusMessage("Sorting data by time...");
m_dbMeasurements = m_dbMeasurements.OrderBy(m => (long)m.Timestamp).ToList();
OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2));
if (m_cacheFileName != null)
{
OnStatusMessage("Caching data for next initialization...");
using (FileStream data = File.OpenWrite(m_cacheFileName))
{
byte[] signalIndexCacheImage = Serialization.Serialize(signalIndexCache, SerializationFormat.Binary);
int compactMeasurementSize = (new CompactMeasurement(signalIndexCache)).BinaryLength;
// Write the signal index cache image size to the file
data.Write(LittleEndian.GetBytes(signalIndexCacheImage.Length), 0, 4);
// Write the signal index cache image to the file
data.Write(signalIndexCacheImage, 0, signalIndexCacheImage.Length);
// Write the size of each compact measurement to the file
data.Write(LittleEndian.GetBytes(compactMeasurementSize), 0, 4);
// Write the total number of compact measurements to the file
data.Write(LittleEndian.GetBytes(m_dbMeasurements.Count), 0, 4);
// Write each compact measurement image to the file
for (int i = 0; i < m_dbMeasurements.Count; i++)
{
((ISupportBinaryImage)m_dbMeasurements[i]).CopyBinaryImageToStream(data);
}
}
}
}
OnStatusMessage("Entering data read cycle...");
ThreadPool.QueueUserWorkItem(PublishData);
}
catch (EndOfStreamException ex)
{
OnProcessException(new EndOfStreamException(string.Format("Failed load cached data from {0} due to file corruption{1} cache will be recreated from database", m_cacheFileName, string.IsNullOrWhiteSpace(ex.Message) ? "," : ": " + ex.Message + " - ")));
// If the cached file is corrupt, delete it and load from the database
if (File.Exists(m_cacheFileName))
File.Delete(m_cacheFileName);
m_dbMeasurements.Clear();
GetDbMeasurements(null);
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed during data load: " + ex.Message, ex));
}
finally
{
if (connection != null)
connection.Close();
}
}
/// <summary>
/// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
/// </summary>
/// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
/// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
/// <param name="index">Index into buffer where compressed payload begins.</param>
/// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
/// <param name="measurementCount">Number of compressed measurements in the payload.</param>
/// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
{
CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];
// Actual data length has to take into account response byte and in-response-to server command byte in the payload header
//int dataLength = length - index - 2;
int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);
// Copy source data into a decompression buffer
byte[] buffer = new byte[bufferLength];
Buffer.BlockCopy(source, index, buffer, 0, dataLength);
// Check that OS endian-order matches endian-order of compressed data
if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
{
// TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it
// can be modified to decompress a payload that is in a non-native Endian order
throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
}
// Attempt to decompress buffer
int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);
if (uncompressedSize == 0)
throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
index = 0;
// Decode ID and state flags
for (int i = 0; i < measurementCount; i++)
{
uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);
measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
{
CompactStateFlags = (byte)(value >> 16),
RuntimeID = (ushort)value
};
index += 4;
}
// Decode values
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
index += 4;
}
if (includeTime)
{
// Decode timestamps
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
index += 8;
}
}
return measurements;
}
private void ProcessMeasurements(IEnumerable<IMeasurement> measurements)
{
if (m_usePayloadCompression && m_compressionModes.HasFlag(CompressionModes.TSSC))
{
ProcessTSSCMeasurements(measurements);
return;
}
// Includes data packet flags and measurement count
const int PacketHeaderSize = DataPublisher.ClientResponseHeaderSize + 5;
List<IBinaryMeasurement> packet;
int packetSize;
bool usePayloadCompression;
bool useCompactMeasurementFormat;
BufferBlockMeasurement bufferBlockMeasurement;
byte[] bufferBlock;
ushort bufferBlockSignalIndex;
IBinaryMeasurement binaryMeasurement;
int binaryLength;
try
{
if (!Enabled)
return;
packet = new List<IBinaryMeasurement>();
packetSize = PacketHeaderSize;
usePayloadCompression = m_usePayloadCompression;
useCompactMeasurementFormat = m_useCompactMeasurementFormat || usePayloadCompression;
foreach (IMeasurement measurement in measurements)
{
bufferBlockMeasurement = measurement as BufferBlockMeasurement;
if ((object)bufferBlockMeasurement != null)
{
// Still sending buffer block measurements to client; we are expecting
// confirmations which will indicate whether retransmission is necessary,
// so we will restart the retransmission timer
m_bufferBlockRetransmissionTimer.Stop();
// Handle buffer block measurements as a special case - this can be any kind of data,
// measurement subscriber will need to know how to interpret buffer
bufferBlock = new byte[6 + bufferBlockMeasurement.Length];
// Prepend sequence number
BigEndian.CopyBytes(m_bufferBlockSequenceNumber, bufferBlock, 0);
m_bufferBlockSequenceNumber++;
// Copy signal index into buffer
bufferBlockSignalIndex = m_signalIndexCache.GetSignalIndex(bufferBlockMeasurement.Key);
BigEndian.CopyBytes(bufferBlockSignalIndex, bufferBlock, 4);
// Append measurement data and send
Buffer.BlockCopy(bufferBlockMeasurement.Buffer, 0, bufferBlock, 6, bufferBlockMeasurement.Length);
m_parent.SendClientResponse(m_clientID, ServerResponse.BufferBlock, ServerCommand.Subscribe, bufferBlock);
lock (m_bufferBlockCacheLock)
{
// Cache buffer block for retransmission
m_bufferBlockCache.Add(bufferBlock);
}
// Start the retransmission timer in case we never receive a confirmation
m_bufferBlockRetransmissionTimer.Start();
}
else
{
// Serialize the current measurement.
if (useCompactMeasurementFormat)
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
else
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(m_clientID));
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on the max
// packet size, process the current packet and start a new packet.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat, usePayloadCompression);
packet.Clear();
packetSize = PacketHeaderSize;
}
// Add the current measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
}
// Process the remaining measurements.
if (packet.Count > 0)
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat, usePayloadCompression);
IncrementProcessedMeasurements(measurements.Count());
// Update latency statistics
m_parent.UpdateLatencyStatistics(measurements.Select(m => (long)(m_lastPublishTime - m.Timestamp)));
}
catch (Exception ex)
{
string message = $"Error processing measurements: {ex.Message}";
OnProcessException(MessageLevel.Info, new InvalidOperationException(message, ex));
}
}
private void ProcessMeasurements(IEnumerable<IMeasurement> measurements)
{
List<ISupportBinaryImage> packet = new List<ISupportBinaryImage>();
bool useCompactMeasurementFormat = m_useCompactMeasurementFormat;
int packetSize = 5;
// Wait for any external events, if needed
WaitForExternalEvents();
// If a set of base times has not yet been initialized, initialize a set by rotating
if (!m_initializedBaseTimeOffsets)
{
if (m_parent.UseBaseTimeOffsets)
RotateBaseTimes();
m_initializedBaseTimeOffsets = true;
}
foreach (IMeasurement measurement in measurements)
{
ISupportBinaryImage binaryMeasurement;
int binaryLength;
// Serialize the current measurement.
if (useCompactMeasurementFormat)
binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
else
binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(m_clientID));
// Determine the size of the measurement in bytes.
binaryLength = binaryMeasurement.BinaryLength;
// If the current measurement will not fit in the packet based on the max
// packet size, process the current packet and start a new packet.
if (packetSize + binaryLength > DataPublisher.MaxPacketSize)
{
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat);
packet.Clear();
packetSize = 5;
}
// Add the current measurement to the packet.
packet.Add(binaryMeasurement);
packetSize += binaryLength;
}
// Process the remaining measurements.
ProcessBinaryMeasurements(packet, useCompactMeasurementFormat);
}
private void ProcessServerResponse(byte[] buffer, int length)
{
// Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing
// is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread...
if (buffer != null && length > 0)
{
try
{
ServerResponse responseCode = (ServerResponse)buffer[0];
ServerCommand commandCode = (ServerCommand)buffer[1];
int responseLength = EndianOrder.BigEndian.ToInt32(buffer, 2);
int responseIndex = 6;
bool solicited = false;
byte[][][] keyIVs;
// See if this was a solicited response to a requested server command
if (responseCode.IsSolicited())
{
lock (m_requests)
{
int index = m_requests.BinarySearch(commandCode);
if (index >= 0)
{
solicited = true;
m_requests.RemoveAt(index);
}
}
// Disconnect any established UDP data channel upon successful unsubscribe
if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded)
DataChannel = null;
}
switch (responseCode)
{
case ServerResponse.Succeeded:
if (solicited)
{
switch (commandCode)
{
case ServerCommand.Authenticate:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_authenticated = true;
OnConnectionAuthenticated();
break;
case ServerCommand.Subscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = true;
if ((object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = true;
break;
case ServerCommand.Unsubscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = false;
if ((object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = false;
break;
case ServerCommand.RotateCipherKeys:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.MetaDataRefresh:
OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
break;
}
}
else
{
switch (commandCode)
{
case ServerCommand.MetaDataRefresh:
// Meta-data refresh may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
break;
case ServerCommand.RotateCipherKeys:
// Key rotation may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.Subscribe:
OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
default:
OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode));
break;
}
}
break;
case ServerResponse.Failed:
if (solicited)
OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
else
OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode));
break;
case ServerResponse.DataPacket:
long now = DateTime.UtcNow.Ticks;
// Deserialize data packet
List<IMeasurement> measurements = new List<IMeasurement>();
DataPacketFlags flags;
Ticks timestamp = 0;
int count;
// Track total data packet bytes received from any channel
m_totalBytesReceived += m_lastBytesReceived;
m_monitoredBytesReceived += m_lastBytesReceived;
// Get data packet flags
flags = (DataPacketFlags)buffer[responseIndex];
responseIndex++;
bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0);
bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0);
int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0;
// Decrypt data packet payload if keys are available
// if (m_keyIVs != null)
// {
// // Get a local copy of volatile keyIVs reference since this can change at any time
// keyIVs = m_keyIVs;
//
// // Decrypt payload portion of data packet
// buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]);
// responseIndex = 0;
// responseLength = buffer.Length;
// }
// Synchronized packets contain a frame level timestamp
if (synchronizedMeasurements)
{
timestamp = EndianOrder.BigEndian.ToInt64(buffer, responseIndex);
responseIndex += 8;
}
// Deserialize number of measurements that follow
count = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
// Deserialize measurements
for (int i = 0; i < count; i++)
{
if (!compactMeasurementFormat)
{
// Deserialize full measurement format
SerializableMeasurement measurement = new SerializableMeasurement(m_encoding);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
measurements.Add(measurement);
}
else if ((object)m_signalIndexCache != null)
{
// Deserialize compact measurement format
CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
// Apply timestamp from frame if not included in transmission
if (!measurement.IncludeTime)
measurement.Timestamp = timestamp;
measurements.Add(measurement);
}
else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
{
if (m_lastMissingCacheWarning != 0L)
{
// Warning message for missing signal index cache
OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
}
m_lastMissingCacheWarning = now;
}
}
// Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer
if ((object)m_localConcentrator != null)
m_localConcentrator.SortMeasurements(measurements);
else
OnNewMeasurements(measurements);
break;
case ServerResponse.DataStartTime:
// Raise data start time event
OnDataStartTime(EndianOrder.BigEndian.ToInt64(buffer, responseIndex));
break;
case ServerResponse.ProcessingComplete:
// Raise input processing completed event
OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerResponse.UpdateSignalIndexCache:
// Deserialize new signal index cache
m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength));
m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
break;
case ServerResponse.UpdateBaseTimes:
// Get active time index
m_timeIndex = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
// Deserialize new base time offsets
m_baseTimeOffsets = new long[] { EndianOrder.BigEndian.ToInt64(buffer, responseIndex), EndianOrder.BigEndian.ToInt64(buffer, responseIndex + 8) };
break;
case ServerResponse.UpdateCipherKeys:
// Get active cipher index
m_cipherIndex = buffer[responseIndex++];
// Extract remaining response
byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1);
// // Decrypt response payload if subscription is authenticated
// if (m_authenticated)
// bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256);
// Deserialize new cipher keys
keyIVs = new byte[2][][];
keyIVs[EvenKey] = new byte[2][];
keyIVs[OddKey] = new byte[2][];
int index = 0;
int bufferLen;
// Read even key size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read even key
keyIVs[EvenKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read even initialization vector size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read even initialization vector
keyIVs[EvenKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen);
index += bufferLen;
// Read odd key size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd key
keyIVs[OddKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read odd initialization vector size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd initialization vector
keyIVs[OddKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen);
index += bufferLen;
// Exchange keys
m_keyIVs = keyIVs;
OnStatusMessage("Successfully established new cipher keys for data packet transmissions.");
break;
}
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex));
}
}
}