/// <summary>
/// Creates a new local system cache from one that was received remotely.
/// </summary>
/// <param name="dataSource"><see cref="DataSet"/> based data source used to interpret local measurement keys.</param>
/// <param name="remoteCache">Deserialized remote signal index cache.</param>
public SignalIndexCache(DataSet dataSource, SignalIndexCache remoteCache)
{
m_subscriberID = remoteCache.SubscriberID;
// If active measurements are defined, interpret signal cache in context of current measurement key definitions
if (dataSource != null && dataSource.Tables != null && dataSource.Tables.Contains("ActiveMeasurements"))
{
DataTable activeMeasurements = dataSource.Tables["ActiveMeasurements"];
m_reference = new ConcurrentDictionary<ushort, Tuple<Guid, string, uint>>();
foreach (KeyValuePair<ushort, Tuple<Guid, string, uint>> signalIndex in remoteCache.Reference)
{
Guid signalID = signalIndex.Value.Item1;
DataRow[] filteredRows = activeMeasurements.Select("SignalID = '" + signalID.ToString() + "'");
if (filteredRows.Length > 0)
{
DataRow row = filteredRows[0];
MeasurementKey key = MeasurementKey.Parse(row["ID"].ToNonNullString(MeasurementKey.Undefined.ToString()), signalID);
m_reference.TryAdd(signalIndex.Key, new Tuple<Guid, string, uint>(signalID, key.Source, key.ID));
}
}
m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
}
else
{
// Just use remote signal index cache as-is if no local configuration exists
m_reference = remoteCache.Reference;
m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
}
}
private SignalIndexCache DeserializeSignalIndexCache(byte[] buffer)
{
GatewayCompressionMode gatewayCompressionMode = (GatewayCompressionMode)(m_operationalModes & OperationalModes.CompressionModeMask);
bool useCommonSerializationFormat = (m_operationalModes & OperationalModes.UseCommonSerializationFormat) > 0;
bool compressSignalIndexCache = (m_operationalModes & OperationalModes.CompressSignalIndexCache) > 0;
SignalIndexCache deserializedCache;
MemoryStream compressedData = null;
GZipStream inflater = null;
if (compressSignalIndexCache && gatewayCompressionMode == GatewayCompressionMode.GZip)
{
try
{
compressedData = new MemoryStream(buffer);
inflater = new GZipStream(compressedData, CompressionMode.Decompress);
buffer = inflater.ReadStream();
}
finally
{
if ((object)inflater != null)
inflater.Close();
if ((object)compressedData != null)
compressedData.Close();
}
}
if (useCommonSerializationFormat)
{
deserializedCache = new SignalIndexCache();
deserializedCache.Encoding = m_encoding;
deserializedCache.ParseBinaryImage(buffer, 0, buffer.Length);
}
else
{
deserializedCache = Serialization.Deserialize<SignalIndexCache>(buffer, TVA.SerializationFormat.Binary);
}
return deserializedCache;
}
/// <summary>
/// Handles auto-connection metadata synchronization to local system.
/// </summary>
/// <param name="state"><see cref="DataSet"/> metadata collection passed into state parameter.</param>
/// <remarks>
/// This function is normally called from thread pool since synchronization can take some time.
/// </remarks>
protected virtual void SynchronizeMetadata(object state)
{
const int MetadataSynchronizationTimeout = 0;
try
{
DataSet metadata = state as DataSet;
if ((object)metadata != null)
{
bool dataMonitoringEnabled = false;
// Reset data stream monitor while meta-data synchronization is in progress
if ((object)m_dataStreamMonitor != null && m_dataStreamMonitor.Enabled)
{
m_dataStreamMonitor.Enabled = false;
dataMonitoringEnabled = true;
}
// Track total meta-data synchronization process time
Ticks startTime = DateTime.UtcNow.Ticks;
// Open the configuration database using settings found in the config file
using (AdoDataConnection database = new AdoDataConnection("systemSettings"))
using (IDbCommand command = database.Connection.CreateCommand())
using (IDbTransaction transaction = database.Connection.BeginTransaction())
{
try
{
command.Transaction = transaction;
// Query the actual record ID based on the known run-time ID for this subscriber device
int parentID = Convert.ToInt32(command.ExecuteScalar(string.Format("SELECT SourceID FROM Runtime WHERE ID = {0} AND SourceTable='Device'", ID), MetadataSynchronizationTimeout));
// Validate that the subscriber device is marked as a concentrator (we are about to associate children devices with it)
if (!command.ExecuteScalar(string.Format("SELECT IsConcentrator FROM Device WHERE ID = {0}", parentID), MetadataSynchronizationTimeout).ToString().ParseBoolean())
command.ExecuteNonQuery(string.Format("UPDATE Device SET IsConcentrator = 1 WHERE ID = {0}", parentID), MetadataSynchronizationTimeout);
// Get any historian associated with the subscriber device
object historianID = command.ExecuteScalar(string.Format("SELECT HistorianID FROM Device WHERE ID = {0}", parentID), MetadataSynchronizationTimeout);
// Determine the active node ID - we cache this since this value won't change for the lifetime of this class
if (m_nodeID == Guid.Empty)
m_nodeID = Guid.Parse(command.ExecuteScalar(string.Format("SELECT NodeID FROM IaonInputAdapter WHERE ID = {0}", ID), MetadataSynchronizationTimeout).ToString());
// Determine the protocol record auto-inc ID value for the gateway transport protocol (GEP) - this value is also cached since it shouldn't change for the lifetime of this class
if (m_gatewayProtocolID == 0)
m_gatewayProtocolID = int.Parse(command.ExecuteScalar("SELECT ID FROM Protocol WHERE Acronym='GatewayTransport'", MetadataSynchronizationTimeout).ToString());
// Prefix all children devices with the name of the parent since the same device names could appear in different connections (helps keep device names unique)
string sourcePrefix = Name + "!";
Dictionary<string, int> deviceIDs = new Dictionary<string, int>(StringComparer.InvariantCultureIgnoreCase);
string selectSql, insertSql, updateSql, deleteSql, deviceAcronym, signalTypeAcronym;
int deviceID;
// Check to see if data for the "DeviceDetail" table was included in the meta-data
if (metadata.Tables.Contains("DeviceDetail"))
{
List<Guid> uniqueIDs = new List<Guid>();
foreach (DataRow row in metadata.Tables["DeviceDetail"].Rows)
{
Guid uniqueID = Guid.Parse(row.Field<object>("UniqueID").ToString()); // adoDatabase.Guid(row, "UniqueID"); // row.Field<Guid>("UniqueID");
// Track unique device Guids in this meta-data session, we'll need to remove any old associated devices that no longer exist
uniqueIDs.Add(uniqueID);
// We will synchronize metadata only if the source owns this device and it's not defined as a concentrator (these should normally be filtered by publisher - but we check just in case).
if (!row["IsConcentrator"].ToNonNullString("0").ParseBoolean())
{
// Define query to determine if this device is already defined (this should always be based on the unique device Guid)
selectSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0}", "deviceGuid");
if (Convert.ToInt32(command.ExecuteScalar(selectSql, MetadataSynchronizationTimeout, database.Guid(uniqueID))) == 0)
{
// Insert new device record
insertSql = database.ParameterizedQueryString("INSERT INTO Device(NodeID, ParentID, HistorianID, Acronym, Name, ProtocolID, IsConcentrator, Enabled, OriginalSource) " +
"VALUES ({0}, {1}, {2}, {3}, {4}, {5}, 0, 1, {6})", "nodeID", "parentID", "historianID", "acronym", "name", "protocolID", "originalSource");
command.ExecuteNonQuery(insertSql, MetadataSynchronizationTimeout, database.Guid(m_nodeID), parentID, historianID, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), m_gatewayProtocolID,
m_internal ? (object)DBNull.Value : string.IsNullOrEmpty(row.Field<string>("ParentAcronym")) ? sourcePrefix + row.Field<string>("Acronym") : sourcePrefix + row.Field<string>("ParentAcronym"));
// Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database
// scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned.
updateSql = database.ParameterizedQueryString("UPDATE Device SET UniqueID = {0} WHERE Acronym = {1}", "uniqueID", "acronym");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, database.Guid(uniqueID), sourcePrefix + row.Field<string>("Acronym"));
}
else
{
selectSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0} AND (ParentID <> {1} OR ParentID IS NULL)", "deviceGuid", "parentID");
// Update existing device record
if (Convert.ToInt32(command.ExecuteScalar(selectSql, MetadataSynchronizationTimeout, database.Guid(uniqueID), parentID)) > 0)
continue;
if (m_internal)
{
// Gateway is assuming ownership of the device records when the "internal" flag is true - this means the device's measurements can be forwarded to another party.
// From a device record perspective, ownership is inferred by setting 'OriginalSource' to null.
updateSql = database.ParameterizedQueryString("UPDATE Device SET Acronym = {0}, Name = {1}, OriginalSource = {2}, ProtocolID = {3}, HistorianID = {4} WHERE UniqueID = {5}", "acronym", "name", "originalSource", "protocolID", "historianID", "uniqueID");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), (object)DBNull.Value, m_gatewayProtocolID, historianID, database.Guid(uniqueID));
}
else
{
// When gateway doesn't own device records (i.e., the "internal" flag is false), this means the device's measurements can only be consumed locally. From a device
// record perspective this means the 'OriginalSource' field is set to the acronym of the PDC or PMU that generated the source measurements. This field allows a
// mirrored source restriction to be implemented later to ensure all devices in an output protocol came from the same original source connection.
updateSql = database.ParameterizedQueryString("UPDATE Device SET Acronym = {0}, Name = {1}, ProtocolID = {2}, HistorianID = {3} WHERE UniqueID = {4}", "acronym", "name", "protocolID", "historianID", "uniqueID");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), m_gatewayProtocolID, historianID, database.Guid(uniqueID));
}
}
}
// Capture local device ID auto-inc value for measurement association
selectSql = database.ParameterizedQueryString("SELECT ID FROM Device WHERE UniqueID = {0}", "deviceGuid");
deviceIDs[row.Field<string>("Acronym")] = Convert.ToInt32(command.ExecuteScalar(selectSql, MetadataSynchronizationTimeout, database.Guid(uniqueID)));
}
// Remove any device records associated with this subscriber that no longer exist in the meta-data
if (uniqueIDs.Count > 0)
{
deleteSql = string.Format("DELETE FROM Device WHERE ParentID = {0} AND UniqueID NOT IN ({1})", parentID, uniqueIDs.Select(uniqueID => string.Format("'{0}'", uniqueID.ToString().ToLower())).ToDelimitedString(", "));
command.ExecuteNonQuery(deleteSql, MetadataSynchronizationTimeout);
}
}
// Check to see if data for the "MeasurementDetail" table was included in the meta-data
if (metadata.Tables.Contains("MeasurementDetail"))
{
List<Guid> signalIDs = new List<Guid>();
DataRow[] measurementRows;
// Load signal type ID's from local database associated with their acronym for proper signal type translation
Dictionary<string, int> signalTypeIDs = new Dictionary<string, int>(StringComparer.InvariantCultureIgnoreCase);
foreach (DataRow row in command.RetrieveData(database.AdapterType, "SELECT ID, Acronym FROM SignalType").Rows)
{
signalTypeAcronym = row.Field<string>("Acronym");
if (!string.IsNullOrWhiteSpace(signalTypeAcronym))
signalTypeIDs[signalTypeAcronym] = row.ConvertField<int>("ID");
}
foreach (DataRow row in metadata.Tables["MeasurementDetail"].Rows)
{
// Get device and signal type acronyms
deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty;
signalTypeAcronym = row.Field<string>("SignalAcronym") ?? string.Empty;
// Make sure we have an associated device and signal type already defined for the measurement
if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym) && !string.IsNullOrWhiteSpace(signalTypeAcronym) && signalTypeIDs.ContainsKey(signalTypeAcronym))
{
// Prefix the tag name with the "updated" device name
deviceID = deviceIDs[deviceAcronym];
string pointTag = sourcePrefix + row.Field<string>("PointTag");
Guid signalID = Guid.Parse(row.Field<object>("SignalID").ToString()); // adoDatabase.Guid(row, "SignalID"); // row.Field<Guid>("SignalID");
// Track unique measurement signal Guids in this meta-data session, we'll need to remove any old associated measurements that no longer exist
signalIDs.Add(signalID);
// Define query to determine if this measurement is already defined (this should always be based on the unique signal ID Guid)
selectSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Measurement WHERE SignalID = {0}", "signalID");
if (Convert.ToInt32(command.ExecuteScalar(selectSql, MetadataSynchronizationTimeout, database.Guid(signalID))) == 0)
{
string alternateTag = Guid.NewGuid().ToString();
// Insert new measurement record
insertSql = database.ParameterizedQueryString("INSERT INTO Measurement (DeviceID, HistorianID, PointTag, AlternateTag, SignalTypeID, SignalReference, Description, Internal, Subscribed, Enabled) VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, 0, 1)", "deviceID", "historianID", "pointTag", "alternateTag", "signalTypeID", "signalReference", "description", "internal");
command.ExecuteNonQuery(insertSql, MetadataSynchronizationTimeout, deviceID, historianID, pointTag, alternateTag, signalTypeIDs[signalTypeAcronym], sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal));
// Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database
// scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned.
updateSql = database.ParameterizedQueryString("UPDATE Measurement SET SignalID = {0}, AlternateTag = NULL WHERE AlternateTag = {1}", "signalID", "alternateTag");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, database.Guid(signalID), alternateTag);
}
else
{
// Update existing measurement record. Note that this update assumes that measurements will remain associated with a static source device.
updateSql = database.ParameterizedQueryString("UPDATE Measurement SET HistorianID = {0}, PointTag = {1}, SignalTypeID = {2}, SignalReference = {3}, Description = {4}, Internal = {5} WHERE SignalID = {6}", "historianID", "pointTag", "signalTypeID", "signalReference", "description", "internal", "signalID");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, historianID, pointTag, signalTypeIDs[signalTypeAcronym], sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal), database.Guid(signalID));
}
}
}
// Remove any measurement records associated with existing devices in this session but no longer exist in the meta-data
if (deviceIDs.Count > 0 && signalIDs.Count > 0)
{
deleteSql = string.Format("DELETE FROM Measurement WHERE DeviceID IN ({0}) AND SignalID NOT IN ({1})", deviceIDs.Values.ToDelimitedString(", "), signalIDs.Select(uniqueID => string.Format("'{0}'", uniqueID.ToString())).ToDelimitedString(", "));
command.ExecuteNonQuery(deleteSql, MetadataSynchronizationTimeout);
}
}
// Check to see if data for the "PhasorDetail" table was included in the meta-data
if (metadata.Tables.Contains("PhasorDetail"))
{
Dictionary<int, int> maxSourceIndicies = new Dictionary<int, int>();
int sourceIndex;
// Phasor data is normally only needed so that the user can property generate a mirrored IEEE C37.118 output stream from the source data.
// This is necessary since, in this protocol, the phasors are described (i.e., labeled) as a unit (i.e., as a complex number) instead of
// as two distinct angle and magnitude measurements.
foreach (DataRow row in metadata.Tables["PhasorDetail"].Rows)
{
// Get device acronym
deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty;
// Make sure we have an associated device already defined for the phasor record
if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym))
{
deviceID = deviceIDs[deviceAcronym];
// Define query to determine if this phasor record is already defined, this is no Guid for these simple label records
selectSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Phasor WHERE DeviceID = {0} AND SourceIndex = {1}", "deviceID", "sourceIndex");
if (Convert.ToInt32(command.ExecuteScalar(selectSql, MetadataSynchronizationTimeout, deviceID, row.ConvertField<int>("SourceIndex"))) == 0)
{
// Insert new phasor record
insertSql = database.ParameterizedQueryString("INSERT INTO Phasor (DeviceID, Label, Type, Phase, SourceIndex) VALUES ({0}, {1}, {2}, {3}, {4})", "deviceID", "label", "type", "phase", "sourceIndex");
command.ExecuteNonQuery(insertSql, MetadataSynchronizationTimeout, deviceID, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), row.ConvertField<int>("SourceIndex"));
}
else
{
// Update existing phasor record
updateSql = database.ParameterizedQueryString("UPDATE Phasor SET Label = {0}, Type = {1}, Phase = {2} WHERE DeviceID = {3} AND SourceIndex = {4}", "label", "type", "phase", "deviceID", "sourceIndex");
command.ExecuteNonQuery(updateSql, MetadataSynchronizationTimeout, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), deviceID, row.ConvertField<int>("SourceIndex"));
}
// Track largest source index for each device
maxSourceIndicies.TryGetValue(deviceID, out sourceIndex);
if (row.ConvertField<int>("SourceIndex") > sourceIndex)
maxSourceIndicies[deviceID] = row.ConvertField<int>("SourceIndex");
}
}
// Remove any phasor records associated with existing devices in this session but no longer exist in the meta-data
if (maxSourceIndicies.Count > 0)
{
foreach (KeyValuePair<int, int> deviceIndexPair in maxSourceIndicies)
{
deleteSql = string.Format("DELETE FROM Phasor WHERE DeviceID = {0} AND SourceIndex > {1}", deviceIndexPair.Key, deviceIndexPair.Value);
command.ExecuteNonQuery(deleteSql, MetadataSynchronizationTimeout);
}
}
}
transaction.Commit();
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex));
try
{
transaction.Rollback();
}
catch (Exception rollbackException)
{
OnProcessException(new InvalidOperationException("Failed to roll back database transaction due to exception: " + rollbackException.Message, rollbackException));
}
return;
}
}
// New signals may have been defined, take original remote signal index cache and apply changes
if (m_remoteSignalIndexCache != null)
m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
OnStatusMessage("Meta-data synchronization completed successfully in {0}", (DateTime.UtcNow.Ticks - startTime).ToElapsedTimeString(3));
// Restart data stream monitor after meta-data synchronization if it was originally enabled
if (dataMonitoringEnabled && (object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = true;
}
else
{
OnStatusMessage("WARNING: Meta-data synchronization was not performed, deserialized dataset was empty.");
}
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex));
}
}
private void ProcessServerResponse(byte[] buffer, int length)
{
// Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing
// is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread...
if (buffer != null && length > 0)
{
try
{
ServerResponse responseCode = (ServerResponse)buffer[0];
ServerCommand commandCode = (ServerCommand)buffer[1];
int responseLength = EndianOrder.BigEndian.ToInt32(buffer, 2);
int responseIndex = DataPublisher.ClientResponseHeaderSize;
bool solicited = false;
byte[][][] keyIVs;
// See if this was a solicited response to a requested server command
if (responseCode.IsSolicited())
{
lock (m_requests)
{
int index = m_requests.BinarySearch(commandCode);
if (index >= 0)
{
solicited = true;
m_requests.RemoveAt(index);
}
}
// Disconnect any established UDP data channel upon successful unsubscribe
if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded)
DataChannel = null;
}
switch (responseCode)
{
case ServerResponse.Succeeded:
if (solicited)
{
switch (commandCode)
{
case ServerCommand.Authenticate:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_authenticated = true;
OnConnectionAuthenticated();
break;
case ServerCommand.Subscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = true;
if ((object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = true;
break;
case ServerCommand.Unsubscribe:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
m_subscribed = false;
if ((object)m_dataStreamMonitor != null)
m_dataStreamMonitor.Enabled = false;
break;
case ServerCommand.RotateCipherKeys:
OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.MetaDataRefresh:
OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
break;
}
}
else
{
switch (commandCode)
{
case ServerCommand.MetaDataRefresh:
// Meta-data refresh may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode);
OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
break;
case ServerCommand.RotateCipherKeys:
// Key rotation may be unsolicited
OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerCommand.Subscribe:
OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
default:
OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode));
break;
}
}
break;
case ServerResponse.Failed:
if (solicited)
OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
else
OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode));
break;
case ServerResponse.DataPacket:
long now = DateTime.UtcNow.Ticks;
// Deserialize data packet
List<IMeasurement> measurements = new List<IMeasurement>();
DataPacketFlags flags;
Ticks timestamp = 0;
int count;
// Track total data packet bytes received from any channel
m_totalBytesReceived += m_lastBytesReceived;
m_monitoredBytesReceived += m_lastBytesReceived;
// Get data packet flags
flags = (DataPacketFlags)buffer[responseIndex];
responseIndex++;
bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0);
bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0);
bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0);
int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0;
// Decrypt data packet payload if keys are available
if (m_keyIVs != null)
{
// Get a local copy of volatile keyIVs reference since this can change at any time
keyIVs = m_keyIVs;
// Decrypt payload portion of data packet
buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]);
responseIndex = 0;
responseLength = buffer.Length;
}
// Synchronized packets contain a frame level timestamp
if (synchronizedMeasurements)
{
timestamp = EndianOrder.BigEndian.ToInt64(buffer, responseIndex);
responseIndex += 8;
}
// Deserialize number of measurements that follow
count = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
if (compressedPayload)
{
if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
{
if (m_lastMissingCacheWarning != 0L)
{
// Warning message for missing signal index cache
OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
}
m_lastMissingCacheWarning = now;
}
else
{
try
{
// Decompress compact measurements from payload
measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags));
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex));
}
}
}
else
{
// Deserialize measurements
for (int i = 0; i < count; i++)
{
if (!compactMeasurementFormat)
{
// Deserialize full measurement format
SerializableMeasurement measurement = new SerializableMeasurement(m_encoding);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
measurements.Add(measurement);
}
else if ((object)m_signalIndexCache != null)
{
// Deserialize compact measurement format
CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
// Apply timestamp from frame if not included in transmission
if (!measurement.IncludeTime)
measurement.Timestamp = timestamp;
measurements.Add(measurement);
}
else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
{
if (m_lastMissingCacheWarning != 0L)
{
// Warning message for missing signal index cache
OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
}
m_lastMissingCacheWarning = now;
}
}
}
// Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer
if ((object)m_localConcentrator != null)
m_localConcentrator.SortMeasurements(measurements);
else
OnNewMeasurements(measurements);
break;
case ServerResponse.DataStartTime:
// Raise data start time event
OnDataStartTime(EndianOrder.BigEndian.ToInt64(buffer, responseIndex));
break;
case ServerResponse.ProcessingComplete:
// Raise input processing completed event
OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength));
break;
case ServerResponse.UpdateSignalIndexCache:
// Deserialize new signal index cache
m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength));
m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
break;
case ServerResponse.UpdateBaseTimes:
// Get active time index
m_timeIndex = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
responseIndex += 4;
// Deserialize new base time offsets
m_baseTimeOffsets = new long[] { EndianOrder.BigEndian.ToInt64(buffer, responseIndex), EndianOrder.BigEndian.ToInt64(buffer, responseIndex + 8) };
break;
case ServerResponse.UpdateCipherKeys:
// Get active cipher index
m_cipherIndex = buffer[responseIndex++];
// Extract remaining response
byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1);
// Decrypt response payload if subscription is authenticated
if (m_authenticated)
bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256);
// Deserialize new cipher keys
keyIVs = new byte[2][][];
keyIVs[EvenKey] = new byte[2][];
keyIVs[OddKey] = new byte[2][];
int index = 0;
int bufferLen;
// Read even key size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read even key
keyIVs[EvenKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read even initialization vector size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read even initialization vector
keyIVs[EvenKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen);
index += bufferLen;
// Read odd key size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd key
keyIVs[OddKey][KeyIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen);
index += bufferLen;
// Read odd initialization vector size
bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
index += 4;
// Read odd initialization vector
keyIVs[OddKey][IVIndex] = new byte[bufferLen];
Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen);
index += bufferLen;
// Exchange keys
m_keyIVs = keyIVs;
OnStatusMessage("Successfully established new cipher keys for data packet transmissions.");
break;
}
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex));
}
}
}
/// <summary>
/// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
/// </summary>
/// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
/// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
/// <param name="index">Index into buffer where compressed payload begins.</param>
/// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
/// <param name="measurementCount">Number of compressed measurements in the payload.</param>
/// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
/// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
/// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
{
CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];
byte[] buffer = null;
try
{
// Actual data length has to take into account response byte and in-response-to server command byte in the payload header
//int dataLength = length - index - 2;
int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);
// Copy source data into a decompression buffer
buffer = BufferPool.TakeBuffer(bufferLength);
Buffer.BlockCopy(source, index, buffer, 0, dataLength);
// Check that OS endian-order matches endian-order of compressed data
if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
{
// TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it
// can be modified to decompress a payload that is non-native Endian order
throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
}
// Attempt to decompress buffer
int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);
if (uncompressedSize == 0)
throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
index = 0;
// Decode ID and state flags
for (int i = 0; i < measurementCount; i++)
{
uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);
measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
{
CompactStateFlags = (byte)(value >> 16),
RuntimeID = (ushort)value
};
index += 4;
}
// Decode values
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
index += 4;
}
if (includeTime)
{
// Decode timestamps
for (int i = 0; i < measurementCount; i++)
{
measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
index += 8;
}
}
}
finally
{
if ((object)buffer != null)
BufferPool.ReturnBuffer(buffer);
}
return measurements;
}
/// <summary>
/// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value.
/// </summary>
/// <param name="measurement">Source <see cref="IMeasurement"/> value.</param>
/// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
/// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
/// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
/// <param name="timeIndex">Time index to use for base offset.</param>
/// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false)
{
ID = measurement.ID;
Key = measurement.Key;
Value = measurement.Value;
Adder = measurement.Adder;
Multiplier = measurement.Multiplier;
Timestamp = measurement.Timestamp;
StateFlags = measurement.StateFlags;
m_signalIndexCache = signalIndexCache;
m_includeTime = includeTime;
// We keep a clone of the base time offsets, if provided, since array contents can change at any time
if (baseTimeOffsets == null)
m_baseTimeOffsets = s_emptyBaseTimeOffsets;
else
m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] };
m_timeIndex = timeIndex;
m_useMillisecondResolution = useMillisecondResolution;
}
/// <summary>
/// Creates a new <see cref="CompactMeasurement"/>.
/// </summary>
/// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
/// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
/// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
/// <param name="timeIndex">Time index to use for base offset.</param>
/// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
public CompactMeasurement(SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false)
{
m_signalIndexCache = signalIndexCache;
m_includeTime = includeTime;
// We keep a clone of the base time offsets, if provided, since array contents can change at any time
if (baseTimeOffsets == null)
m_baseTimeOffsets = s_emptyBaseTimeOffsets;
else
m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] };
m_timeIndex = timeIndex;
m_useMillisecondResolution = useMillisecondResolution;
}
// Retrieves the measurements from the database.
private void GetDbMeasurements(object state)
{
IDbConnection connection = null;
// Get measurements from the database.
try
{
SignalIndexCache signalIndexCache = new SignalIndexCache();
CompactMeasurement measurement;
long startTime = DateTime.UtcNow.Ticks;
if (m_cacheFileName != null && File.Exists(m_cacheFileName))
{
OnStatusMessage("Loading cached input data...");
try
{
using (FileStream data = File.OpenRead(m_cacheFileName))
{
byte[] buffer = new byte[4];
int signalIndexCacheImageSize;
int compactMeasurementSize;
int totalMeasurements;
// Read the signal index cache image size from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
signalIndexCacheImageSize = EndianOrder.LittleEndian.ToInt32(buffer, 0);
// Resize buffer to accomodate exact signal index cache
buffer = new byte[signalIndexCacheImageSize];
// Read the signal index cache image from the file
if (data.Read(buffer, 0, signalIndexCacheImageSize) != signalIndexCacheImageSize)
throw new EndOfStreamException();
// Deserialize the signal index cache
signalIndexCache = Serialization.Deserialize<SignalIndexCache>(buffer, TVA.SerializationFormat.Binary);
// Read the size of each compact measurement from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
compactMeasurementSize = EndianOrder.LittleEndian.ToInt32(buffer, 0);
// Read the total number of compact measurements from the file
if (data.Read(buffer, 0, 4) != 4)
throw new EndOfStreamException();
totalMeasurements = EndianOrder.LittleEndian.ToInt32(buffer, 0);
// Resize buffer to accomodate compact measurement if needed (not likely)
if (buffer.Length < compactMeasurementSize)
buffer = new byte[compactMeasurementSize];
// Read each compact measurement image from the file
for (int i = 0; i < totalMeasurements; i++)
{
if (data.Read(buffer, 0, compactMeasurementSize) != compactMeasurementSize)
throw new EndOfStreamException();
// Parse compact measurement
measurement = new CompactMeasurement(signalIndexCache);
measurement.ParseBinaryImage(buffer, 0, compactMeasurementSize);
m_dbMeasurements.Add(measurement);
if (m_dbMeasurements.Count % 50000 == 0)
OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
}
OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(4));
}
}
catch (Exception ex)
{
if (ex is EndOfStreamException)
throw (EndOfStreamException)ex;
throw new EndOfStreamException(ex.Message, ex);
}
}
else
{
OnStatusMessage("Loading database input data...");
const string MeasurementTable = "ActiveMeasurements";
Dictionary<string, string> dataProviderSettings = m_dataProviderString.ParseKeyValuePairs();
Assembly assm = Assembly.Load(dataProviderSettings["AssemblyName"]);
Type connectionType = assm.GetType(dataProviderSettings["ConnectionType"]);
Dictionary<Guid, MeasurementKey> lookupCache = new Dictionary<Guid, MeasurementKey>();
IDbCommand command;
IDataReader dbReader;
MeasurementKey key;
Guid id;
ushort index = 0;
connection = (IDbConnection)Activator.CreateInstance(connectionType);
connection.ConnectionString = m_dbConnectionString;
connection.Open();
command = connection.CreateCommand();
command.CommandText = string.Format("SELECT * FROM {0}", m_dbTableName);
using (dbReader = command.ExecuteReader())
{
while (dbReader.Read())
{
measurement = new CompactMeasurement(signalIndexCache);
foreach (string fieldName in m_fieldNames.Keys)
{
object value = dbReader[fieldName];
string propertyName = m_fieldNames[fieldName];
switch (propertyName)
{
case "Timestamp":
// If the value is a timestamp, use the timestamp format
// specified by the user when reading the timestamp.
if (m_timestampFormat == null)
measurement.Timestamp = long.Parse(value.ToNonNullString());
else
measurement.Timestamp = DateTime.ParseExact(value.ToNonNullString(), m_timestampFormat, CultureInfo.CurrentCulture);
break;
case "ID":
if (Guid.TryParse(value.ToString(), out id))
{
if (!lookupCache.TryGetValue(id, out key))
{
if (DataSource.Tables.Contains(MeasurementTable))
{
DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("SignalID = '{0}'", id));
if (filteredRows.Length > 0)
MeasurementKey.TryParse(filteredRows[0]["ID"].ToString(), id, out key);
}
if (key != default(MeasurementKey))
{
// Cache measurement key associated with ID
lookupCache[id] = key;
// Assign a runtime index optimization for distinct measurements
signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(id, key.Source, key.ID));
}
}
measurement.ID = id;
measurement.Key = key;
}
break;
case "Key":
if (MeasurementKey.TryParse(value.ToString(), Guid.Empty, out key))
{
// Attempt to update empty signal ID if available
if (key.SignalID == Guid.Empty)
{
if (DataSource.Tables.Contains(MeasurementTable))
{
DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("ID = '{0}'", key.ToString()));
if (filteredRows.Length > 0)
key.SignalID = filteredRows[0]["SignalID"].ToNonNullString(Guid.Empty.ToString()).ConvertToType<Guid>();
}
}
if (key.SignalID != Guid.Empty)
{
measurement.ID = key.SignalID;
if (!lookupCache.ContainsKey(measurement.ID))
{
// Cache measurement key associated with ID
lookupCache[measurement.ID] = key;
// Assign a runtime index optimization for distinct measurements
signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(measurement.ID, key.Source, key.ID));
}
}
measurement.Key = key;
}
break;
case "Value":
measurement.Value = Convert.ToDouble(value);
break;
default:
PropertyInfo property = GetAllProperties(typeof(IMeasurement)).FirstOrDefault(propertyInfo => propertyInfo.Name == propertyName);
if (property != null)
{
Type propertyType = property.PropertyType;
Type valueType = value.GetType();
if (property.PropertyType.IsAssignableFrom(value.GetType()))
{
property.SetValue(measurement, value, null);
}
else if (property.PropertyType == typeof(string))
{
property.SetValue(measurement, value.ToNonNullString(), null);
}
else if (valueType == typeof(string))
{
MethodInfo parseMethod = propertyType.GetMethod("Parse", new Type[] { typeof(string) });
if (parseMethod != null && parseMethod.IsStatic)
property.SetValue(measurement, parseMethod.Invoke(null, new object[] { value }), null);
}
else
{
string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1}.", fieldName, propertyName);
OnProcessException(new InvalidCastException(exceptionMessage));
}
}
else
{
string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1} - no property match was found.", fieldName, propertyName);
OnProcessException(new InvalidCastException(exceptionMessage));
}
break;
}
m_dbMeasurements.Add(measurement);
if (m_dbMeasurements.Count % 50000 == 0)
OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
}
}
}
OnStatusMessage("Sorting data by time...");
m_dbMeasurements = m_dbMeasurements.OrderBy(m => (long)m.Timestamp).ToList();
OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(4));
if (m_cacheFileName != null)
{
OnStatusMessage("Caching data for next initialization...");
using (FileStream data = File.OpenWrite(m_cacheFileName))
{
byte[] signalIndexCacheImage = Serialization.Serialize(signalIndexCache, TVA.SerializationFormat.Binary);
int compactMeasurementSize = (new CompactMeasurement(signalIndexCache)).BinaryLength;
// Write the signal index cache image size to the file
data.Write(EndianOrder.LittleEndian.GetBytes(signalIndexCacheImage.Length), 0, 4);
// Write the signal index cache image to the file
data.Write(signalIndexCacheImage, 0, signalIndexCacheImage.Length);
// Write the size of each compact measurement to the file
data.Write(EndianOrder.LittleEndian.GetBytes(compactMeasurementSize), 0, 4);
// Write the total number of compact measurements to the file
data.Write(EndianOrder.LittleEndian.GetBytes(m_dbMeasurements.Count), 0, 4);
// Write each compact measurement image to the file
for (int i = 0; i < m_dbMeasurements.Count; i++)
{
((ISupportBinaryImage)m_dbMeasurements[i]).CopyBinaryImageToStream(data);
}
}
}
}
OnStatusMessage("Entering data read cycle...");
ThreadPool.QueueUserWorkItem(PublishData);
}
catch (EndOfStreamException ex)
{
OnProcessException(new EndOfStreamException(string.Format("Failed load cached data from {0} due to file corruption{1} cache will be recreated from database", m_cacheFileName, string.IsNullOrWhiteSpace(ex.Message) ? "," : ": " + ex.Message + " - ")));
// If the cached file is corrupt, delete it and load from the database
if (File.Exists(m_cacheFileName))
File.Delete(m_cacheFileName);
m_dbMeasurements.Clear();
GetDbMeasurements(null);
}
catch (Exception ex)
{
OnProcessException(new InvalidOperationException("Failed during data load: " + ex.Message, ex));
}
finally
{
if (connection != null)
connection.Close();
}
}
/// <summary>
/// Creates a new <see cref="SynchronizedClientSubscription"/>.
/// </summary>
/// <param name="parent">Reference to parent.</param>
/// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
/// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
{
// Pass parent reference into base class
AssignParentCollection(parent);
m_parent = parent;
m_clientID = clientID;
m_subscriberID = subscriberID;
m_signalIndexCache = new SignalIndexCache()
{
SubscriberID = subscriberID
};
}
