/// <summary> /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value. /// </summary> /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param> /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param> /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param> /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param> /// <param name="timeIndex">Time index to use for base offset.</param> /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param> public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false) { Key = measurement.Key; Value = measurement.Value; Adder = measurement.Adder; Multiplier = measurement.Multiplier; Timestamp = measurement.Timestamp; StateFlags = measurement.StateFlags; m_signalIndexCache = signalIndexCache; m_includeTime = includeTime; // We keep a clone of the base time offsets, if provided, since array contents can change at any time if ((object)baseTimeOffsets == null) { m_baseTimeOffsets = s_emptyBaseTimeOffsets; } else { m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] } }; m_timeIndex = timeIndex; m_useMillisecondResolution = useMillisecondResolution; }
/// <summary> /// Creates a new local system cache from one that was received remotely. /// </summary> /// <param name="dataSource"><see cref="DataSet"/> based data source used to interpret local measurement keys.</param> /// <param name="remoteCache">Deserialized remote signal index cache.</param> public SignalIndexCache(DataSet dataSource, SignalIndexCache remoteCache) { m_subscriberID = remoteCache.SubscriberID; // If active measurements are defined, interpret signal cache in context of current measurement key definitions if (dataSource != null && dataSource.Tables != null && dataSource.Tables.Contains("ActiveMeasurements")) { DataTable activeMeasurements = dataSource.Tables["ActiveMeasurements"]; m_reference = new ConcurrentDictionary<ushort, Tuple<Guid, string, uint>>(); foreach (KeyValuePair<ushort, Tuple<Guid, string, uint>> signalIndex in remoteCache.Reference) { Guid signalID = signalIndex.Value.Item1; DataRow[] filteredRows = activeMeasurements.Select("SignalID = '" + signalID.ToString() + "'"); if (filteredRows.Length > 0) { DataRow row = filteredRows[0]; MeasurementKey key = MeasurementKey.Parse(row["ID"].ToNonNullString(MeasurementKey.Undefined.ToString()), signalID); m_reference.TryAdd(signalIndex.Key, new Tuple<Guid, string, uint>(signalID, key.Source, key.ID)); } } m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs; } else { // Just use remote signal index cache as-is if no local configuration exists m_reference = remoteCache.Reference; m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs; } }
/// <summary> /// Creates a new local system cache from one that was received remotely. /// </summary> /// <param name="dataSource"><see cref="DataSet"/> based data source used to interpret local measurement keys.</param> /// <param name="remoteCache">Deserialized remote signal index cache.</param> public SignalIndexCache(DataSet dataSource, SignalIndexCache remoteCache) { m_subscriberID = remoteCache.SubscriberID; // If active measurements are defined, interpret signal cache in context of current measurement key definitions if (dataSource != null && dataSource.Tables.Contains("ActiveMeasurements")) { DataTable activeMeasurements = dataSource.Tables["ActiveMeasurements"]; m_reference = new ConcurrentDictionary <ushort, MeasurementKey>(); foreach (KeyValuePair <ushort, MeasurementKey> signalIndex in remoteCache.Reference) { Guid signalID = signalIndex.Value.SignalID; DataRow[] filteredRows = activeMeasurements.Select("SignalID = '" + signalID.ToString() + "'"); if (filteredRows.Length > 0) { DataRow row = filteredRows[0]; MeasurementKey key = MeasurementKey.LookUpOrCreate(signalID, row["ID"].ToNonNullString(MeasurementKey.Undefined.ToString())); m_reference.TryAdd(signalIndex.Key, key); } } m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs; } else { // Just use remote signal index cache as-is if no local configuration exists m_reference = remoteCache.Reference; m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs; } }
/// <summary> /// Creates a new <see cref="SynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID) { // Pass parent reference into base class AssignParentCollection(parent); m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_signalIndexCache = new SignalIndexCache() { SubscriberID = subscriberID }; }
/// <summary> /// Creates a new <see cref="UnsynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID) { m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_signalIndexCache = new SignalIndexCache(); m_signalIndexCache.SubscriberID = subscriberID; m_workingBuffer = new BlockAllocatedMemoryStream(); m_bufferBlockCache = new List <byte[]>(); m_bufferBlockCacheLock = new object(); }
/// <summary> /// Creates a new <see cref="SynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> /// <param name="compressionModes"><see cref="CompressionModes"/> requested by client.</param> public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID, CompressionModes compressionModes) { m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_compressionModes = compressionModes; m_signalIndexCache = new SignalIndexCache(); m_signalIndexCache.SubscriberID = subscriberID; m_bufferBlockCache = new List <byte[]>(); m_bufferBlockCacheLock = new object(); }
/// <summary> /// Creates a new <see cref="CompactMeasurement"/>. /// </summary> /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param> /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param> /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param> /// <param name="timeIndex">Time index to use for base offset.</param> /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param> public CompactMeasurement(SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false) { m_signalIndexCache = signalIndexCache; m_includeTime = includeTime; // We keep a clone of the base time offsets, if provided, since array contents can change at any time if ((object)baseTimeOffsets == null) { m_baseTimeOffsets = s_emptyBaseTimeOffsets; } else { m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] } }; m_timeIndex = timeIndex; m_useMillisecondResolution = useMillisecondResolution; }
/// <summary> /// Creates a new <see cref="UnsynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID) { // Pass parent reference into base class AssignParentCollection(parent); m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_signalIndexCache = new SignalIndexCache() { SubscriberID = subscriberID }; m_processQueue = new AsyncQueue <IEnumerable <IMeasurement> >() { ProcessItemFunction = ProcessMeasurements }; m_processQueue.ProcessException += m_processQueue_ProcessException; }
/// <summary> /// Creates a new <see cref="SynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> /// <param name="compressionModes"><see cref="CompressionModes"/> requested by client.</param> public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID, CompressionModes compressionModes) { m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_compressionModes = compressionModes; m_signalIndexCache = new SignalIndexCache(); m_signalIndexCache.SubscriberID = subscriberID; m_bufferBlockCache = new List<byte[]>(); m_bufferBlockCacheLock = new object(); }
/// <summary> /// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer. /// </summary> /// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param> /// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param> /// <param name="index">Index into buffer where compressed payload begins.</param> /// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param> /// <param name="measurementCount">Number of compressed measurements in the payload.</param> /// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param> /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param> /// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns> public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags) { CompactMeasurement[] measurements = new CompactMeasurement[measurementCount]; // Actual data length has to take into account response byte and in-response-to server command byte in the payload header //int dataLength = length - index - 2; int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength); // Copy source data into a decompression buffer byte[] buffer = new byte[bufferLength]; Buffer.BlockCopy(source, index, buffer, 0, dataLength); // Check that OS endian-order matches endian-order of compressed data if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0)) { // can be modified to decompress a payload that is in a non-native Endian order throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order."); } // Attempt to decompress buffer int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength); if (uncompressedSize == 0) { throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption."); } index = 0; // Decode ID and state flags for (int i = 0; i < measurementCount; i++) { uint value = NativeEndianOrder.Default.ToUInt32(buffer, index); measurements[i] = new CompactMeasurement(signalIndexCache, includeTime) { CompactStateFlags = (byte)(value >> 16), RuntimeID = (ushort)value }; index += 4; } // Decode values for (int i = 0; i < measurementCount; i++) { measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index); index += 4; } if (includeTime) { // Decode timestamps for (int i = 0; i < measurementCount; i++) { measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index); index += 8; } } return(measurements); }
private SignalIndexCache DeserializeSignalIndexCache(byte[] buffer) { CompressionModes compressionModes = (CompressionModes)(m_operationalModes & OperationalModes.CompressionModeMask); bool useCommonSerializationFormat = (m_operationalModes & OperationalModes.UseCommonSerializationFormat) > 0; bool compressSignalIndexCache = (m_operationalModes & OperationalModes.CompressSignalIndexCache) > 0; SignalIndexCache deserializedCache; GZipStream inflater = null; if (compressSignalIndexCache && compressionModes.HasFlag(CompressionModes.GZip)) { try { using (MemoryStream compressedData = new MemoryStream(buffer)) { inflater = new GZipStream(compressedData, CompressionMode.Decompress, true); buffer = inflater.ReadStream(); } } finally { if ((object)inflater != null) inflater.Close(); } } if (useCommonSerializationFormat) { deserializedCache = new SignalIndexCache(); deserializedCache.Encoding = m_encoding; deserializedCache.ParseBinaryImage(buffer, 0, buffer.Length); } else { deserializedCache = Serialization.Deserialize<SignalIndexCache>(buffer, SerializationFormat.Binary); } return deserializedCache; }
/// <summary> /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value. /// </summary> /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param> /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param> /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param> /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param> /// <param name="timeIndex">Time index to use for base offset.</param> /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param> public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false) { Metadata = measurement.Metadata; Value = measurement.Value; Timestamp = measurement.Timestamp; StateFlags = measurement.StateFlags; m_signalIndexCache = signalIndexCache; m_includeTime = includeTime; // We keep a clone of the base time offsets, if provided, since array contents can change at any time if ((object)baseTimeOffsets == null) m_baseTimeOffsets = s_emptyBaseTimeOffsets; else m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] }; m_timeIndex = timeIndex; m_useMillisecondResolution = useMillisecondResolution; }
private SignalIndexCache DeserializeSignalIndexCache(byte[] buffer) { GatewayCompressionMode gatewayCompressionMode = (GatewayCompressionMode)(m_operationalModes & OperationalModes.CompressionModeMask); bool useCommonSerializationFormat = (m_operationalModes & OperationalModes.UseCommonSerializationFormat) > 0; bool compressSignalIndexCache = (m_operationalModes & OperationalModes.CompressSignalIndexCache) > 0; SignalIndexCache deserializedCache; MemoryStream compressedData = null; GZipStream inflater = null; if (compressSignalIndexCache && gatewayCompressionMode == GatewayCompressionMode.GZip) { try { compressedData = new MemoryStream(buffer); inflater = new GZipStream(compressedData, CompressionMode.Decompress); buffer = inflater.ReadStream(); } finally { if ((object)inflater != null) inflater.Close(); if ((object)compressedData != null) compressedData.Close(); } } if (useCommonSerializationFormat) { deserializedCache = new SignalIndexCache(); deserializedCache.Encoding = m_encoding; deserializedCache.ParseBinaryImage(buffer, 0, buffer.Length); } else { deserializedCache = Serialization.Deserialize<SignalIndexCache>(buffer, GSF.SerializationFormat.Binary); } return deserializedCache; }
/// <summary> /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value. /// </summary> /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param> /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param> /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param> /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param> /// <param name="timeIndex">Time index to use for base offset.</param> /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param> public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime, long[] baseTimeOffsets, int timeIndex, bool useMillisecondResolution) { ID = measurement.ID; Key = measurement.Key; Value = measurement.Value; Adder = measurement.Adder; Multiplier = measurement.Multiplier; Timestamp = measurement.Timestamp; StateFlags = measurement.StateFlags; m_signalIndexCache = signalIndexCache; m_includeTime = includeTime; // We keep a clone of the base time offsets, if provided, since array contents can change at any time if (baseTimeOffsets == null) m_baseTimeOffsets = s_emptyBaseTimeOffsets; else m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] }; m_timeIndex = timeIndex; m_useMillisecondResolution = useMillisecondResolution; }
// Retrieves the measurements from the database. private void GetDbMeasurements(object state) { IDbConnection connection = null; // Get measurements from the database. try { SignalIndexCache signalIndexCache = new SignalIndexCache(); CompactMeasurement measurement; long startTime = DateTime.UtcNow.Ticks; if (m_cacheFileName != null && File.Exists(m_cacheFileName)) { OnStatusMessage("Loading cached input data..."); try { using (FileStream data = File.OpenRead(m_cacheFileName)) { byte[] buffer = new byte[4]; int signalIndexCacheImageSize; int compactMeasurementSize; int totalMeasurements; // Read the signal index cache image size from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); signalIndexCacheImageSize = LittleEndian.ToInt32(buffer, 0); // Resize buffer to accomodate exact signal index cache buffer = new byte[signalIndexCacheImageSize]; // Read the signal index cache image from the file if (data.Read(buffer, 0, signalIndexCacheImageSize) != signalIndexCacheImageSize) throw new EndOfStreamException(); // Deserialize the signal index cache signalIndexCache = Serialization.Deserialize<SignalIndexCache>(buffer, SerializationFormat.Binary); // Read the size of each compact measurement from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); compactMeasurementSize = LittleEndian.ToInt32(buffer, 0); // Read the total number of compact measurements from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); totalMeasurements = LittleEndian.ToInt32(buffer, 0); // Resize buffer to accomodate compact measurement if needed (not likely) if (buffer.Length < compactMeasurementSize) buffer = new byte[compactMeasurementSize]; // Read each compact measurement image from the file for (int i = 0; i < totalMeasurements; i++) { if (data.Read(buffer, 0, compactMeasurementSize) != compactMeasurementSize) throw new EndOfStreamException(); // Parse compact measurement measurement = new CompactMeasurement(signalIndexCache); measurement.ParseBinaryImage(buffer, 0, compactMeasurementSize); m_dbMeasurements.Add(measurement); if (m_dbMeasurements.Count % 50000 == 0) OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count); } OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2)); } } catch (Exception ex) { if (ex is EndOfStreamException) throw (EndOfStreamException)ex; throw new EndOfStreamException(ex.Message, ex); } } else { OnStatusMessage("Loading database input data..."); const string MeasurementTable = "ActiveMeasurements"; Dictionary<string, string> dataProviderSettings = m_dataProviderString.ParseKeyValuePairs(); Assembly assm = Assembly.Load(dataProviderSettings["AssemblyName"]); Type connectionType = assm.GetType(dataProviderSettings["ConnectionType"]); Dictionary<Guid, MeasurementKey> lookupCache = new Dictionary<Guid, MeasurementKey>(); IDbCommand command; IDataReader dbReader; MeasurementKey key; Guid id; ushort index = 0; connection = (IDbConnection)Activator.CreateInstance(connectionType); connection.ConnectionString = m_dbConnectionString; connection.Open(); command = connection.CreateCommand(); command.CommandText = string.Format("SELECT * FROM {0}", m_dbTableName); using (dbReader = command.ExecuteReader()) { while (dbReader.Read()) { measurement = new CompactMeasurement(signalIndexCache); foreach (string fieldName in m_fieldNames.Keys) { object value = dbReader[fieldName]; string propertyName = m_fieldNames[fieldName]; switch (propertyName) { case "Timestamp": // If the value is a timestamp, use the timestamp format // specified by the user when reading the timestamp. if (m_timestampFormat == null) measurement.Timestamp = long.Parse(value.ToNonNullString()); else measurement.Timestamp = DateTime.ParseExact(value.ToNonNullString(), m_timestampFormat, CultureInfo.CurrentCulture); break; case "ID": if (Guid.TryParse(value.ToString(), out id)) { if (!lookupCache.TryGetValue(id, out key)) { if (DataSource.Tables.Contains(MeasurementTable)) { DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("SignalID = '{0}'", id)); if (filteredRows.Length > 0) key = MeasurementKey.LookUpOrCreate(id, filteredRows[0]["ID"].ToString()); } if (key != MeasurementKey.Undefined) { // Cache measurement key associated with ID lookupCache[id] = key; // Assign a runtime index optimization for distinct measurements signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(id, key.Source, key.ID)); } } measurement.Key = key; } break; case "Key": if (MeasurementKey.TryParse(value.ToString(), out key)) { if (!lookupCache.ContainsKey(key.SignalID)) { // Cache measurement key associated with ID lookupCache[key.SignalID] = key; // Assign a runtime index optimization for distinct measurements signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(key.SignalID, key.Source, key.ID)); } measurement.Key = key; } break; case "Value": measurement.Value = Convert.ToDouble(value); break; default: PropertyInfo property = GetAllProperties(typeof(IMeasurement)).FirstOrDefault(propertyInfo => propertyInfo.Name == propertyName); if (property != null) { Type propertyType = property.PropertyType; Type valueType = value.GetType(); if (property.PropertyType.IsAssignableFrom(value.GetType())) { property.SetValue(measurement, value, null); } else if (property.PropertyType == typeof(string)) { property.SetValue(measurement, value.ToNonNullString(), null); } else if (valueType == typeof(string)) { MethodInfo parseMethod = propertyType.GetMethod("Parse", new[] { typeof(string) }); if (parseMethod != null && parseMethod.IsStatic) property.SetValue(measurement, parseMethod.Invoke(null, new[] { value }), null); } else { string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1}.", fieldName, propertyName); OnProcessException(new InvalidCastException(exceptionMessage)); } } else { string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1} - no property match was found.", fieldName, propertyName); OnProcessException(new InvalidCastException(exceptionMessage)); } break; } m_dbMeasurements.Add(measurement); if (m_dbMeasurements.Count % 50000 == 0) OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count); } } } OnStatusMessage("Sorting data by time..."); m_dbMeasurements = m_dbMeasurements.OrderBy(m => (long)m.Timestamp).ToList(); OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2)); if (m_cacheFileName != null) { OnStatusMessage("Caching data for next initialization..."); using (FileStream data = File.OpenWrite(m_cacheFileName)) { byte[] signalIndexCacheImage = Serialization.Serialize(signalIndexCache, SerializationFormat.Binary); int compactMeasurementSize = (new CompactMeasurement(signalIndexCache)).BinaryLength; // Write the signal index cache image size to the file data.Write(LittleEndian.GetBytes(signalIndexCacheImage.Length), 0, 4); // Write the signal index cache image to the file data.Write(signalIndexCacheImage, 0, signalIndexCacheImage.Length); // Write the size of each compact measurement to the file data.Write(LittleEndian.GetBytes(compactMeasurementSize), 0, 4); // Write the total number of compact measurements to the file data.Write(LittleEndian.GetBytes(m_dbMeasurements.Count), 0, 4); // Write each compact measurement image to the file for (int i = 0; i < m_dbMeasurements.Count; i++) { ((ISupportBinaryImage)m_dbMeasurements[i]).CopyBinaryImageToStream(data); } } } } OnStatusMessage("Entering data read cycle..."); ThreadPool.QueueUserWorkItem(PublishData); } catch (EndOfStreamException ex) { OnProcessException(new EndOfStreamException(string.Format("Failed load cached data from {0} due to file corruption{1} cache will be recreated from database", m_cacheFileName, string.IsNullOrWhiteSpace(ex.Message) ? "," : ": " + ex.Message + " - "))); // If the cached file is corrupt, delete it and load from the database if (File.Exists(m_cacheFileName)) File.Delete(m_cacheFileName); m_dbMeasurements.Clear(); GetDbMeasurements(null); } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed during data load: " + ex.Message, ex)); } finally { if (connection != null) connection.Close(); } }
public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache) : this(measurement, signalIndexCache, true, null, 0, false) { }
/// <summary> /// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer. /// </summary> /// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param> /// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param> /// <param name="index">Index into buffer where compressed payload begins.</param> /// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param> /// <param name="measurementCount">Number of compressed measurements in the payload.</param> /// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param> /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param> /// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns> public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags) { CompactMeasurement[] measurements = new CompactMeasurement[measurementCount]; // Actual data length has to take into account response byte and in-response-to server command byte in the payload header //int dataLength = length - index - 2; int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength); // Copy source data into a decompression buffer byte[] buffer = new byte[bufferLength]; Buffer.BlockCopy(source, index, buffer, 0, dataLength); // Check that OS endian-order matches endian-order of compressed data if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0)) { // TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it // can be modified to decompress a payload that is in a non-native Endian order throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order."); } // Attempt to decompress buffer int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength); if (uncompressedSize == 0) throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption."); index = 0; // Decode ID and state flags for (int i = 0; i < measurementCount; i++) { uint value = NativeEndianOrder.Default.ToUInt32(buffer, index); measurements[i] = new CompactMeasurement(signalIndexCache, includeTime) { CompactStateFlags = (byte)(value >> 16), RuntimeID = (ushort)value }; index += 4; } // Decode values for (int i = 0; i < measurementCount; i++) { measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index); index += 4; } if (includeTime) { // Decode timestamps for (int i = 0; i < measurementCount; i++) { measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index); index += 8; } } return measurements; }
/// <summary> /// Creates a new <see cref="CompactMeasurement"/>. /// </summary> /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param> /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param> /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param> /// <param name="timeIndex">Time index to use for base offset.</param> /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param> public CompactMeasurement(SignalIndexCache signalIndexCache, bool includeTime, long[] baseTimeOffsets, int timeIndex, bool useMillisecondResolution) { m_signalIndexCache = signalIndexCache; m_includeTime = includeTime; // We keep a clone of the base time offsets, if provided, since array contents can change at any time if (baseTimeOffsets == null) m_baseTimeOffsets = s_emptyBaseTimeOffsets; else m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] }; m_timeIndex = timeIndex; m_useMillisecondResolution = useMillisecondResolution; }
/// <summary> /// Handles meta-data synchronization to local system. /// </summary> /// <remarks> /// This function should only be initiated from call to <see cref="SynchronizeMetadata(DataSet)"/> to make /// sure only one meta-data synchronization happens at once. Users can override this method to customize /// process of meta-data synchronization. /// </remarks> protected virtual void SynchronizeMetadata() { bool dataMonitoringEnabled = false; // TODO: This function is complex and very closely tied to the current time-series data schema - perhaps it should be moved outside this class and referenced // TODO: as a delegate that can be assigned and called to allow other schemas as well. DataPublisher is already very flexible in what data it can deliver. try { DataSet metadata = m_receivedMetadata; // Only perform database synchronization if meta-data has changed since last update if (!SynchronizedMetadataChanged(metadata)) return; if ((object)metadata == null) { OnStatusMessage("WARNING: Meta-data synchronization was not performed, deserialized dataset was empty."); return; } // Reset data stream monitor while meta-data synchronization is in progress if ((object)m_dataStreamMonitor != null && m_dataStreamMonitor.Enabled) { m_dataStreamMonitor.Enabled = false; dataMonitoringEnabled = true; } // Track total meta-data synchronization process time Ticks startTime = DateTime.UtcNow.Ticks; DateTime updateTime; DateTime latestUpdateTime = DateTime.MinValue; // Open the configuration database using settings found in the config file using (AdoDataConnection database = new AdoDataConnection("systemSettings")) using (IDbCommand command = database.Connection.CreateCommand()) { IDbTransaction transaction = null; if (m_useTransactionForMetadata) transaction = database.Connection.BeginTransaction(database.DefaultIsloationLevel); try { if ((object)transaction != null) command.Transaction = transaction; // Query the actual record ID based on the known run-time ID for this subscriber device int parentID = Convert.ToInt32(command.ExecuteScalar($"SELECT SourceID FROM Runtime WHERE ID = {ID} AND SourceTable='Device'", m_metadataSynchronizationTimeout)); // Validate that the subscriber device is marked as a concentrator (we are about to associate children devices with it) if (!command.ExecuteScalar($"SELECT IsConcentrator FROM Device WHERE ID = {parentID}", m_metadataSynchronizationTimeout).ToString().ParseBoolean()) command.ExecuteNonQuery($"UPDATE Device SET IsConcentrator = 1 WHERE ID = {parentID}", m_metadataSynchronizationTimeout); // Get any historian associated with the subscriber device object historianID = command.ExecuteScalar($"SELECT HistorianID FROM Device WHERE ID = {parentID}", m_metadataSynchronizationTimeout); // Determine the active node ID - we cache this since this value won't change for the lifetime of this class if (m_nodeID == Guid.Empty) m_nodeID = Guid.Parse(command.ExecuteScalar($"SELECT NodeID FROM IaonInputAdapter WHERE ID = {(int)ID}", m_metadataSynchronizationTimeout).ToString()); // Determine the protocol record auto-inc ID value for the gateway transport protocol (GEP) - this value is also cached since it shouldn't change for the lifetime of this class if (m_gatewayProtocolID == 0) m_gatewayProtocolID = int.Parse(command.ExecuteScalar("SELECT ID FROM Protocol WHERE Acronym='GatewayTransport'", m_metadataSynchronizationTimeout).ToString()); // Ascertain total number of actions required for all meta-data synchronization so some level feed back can be provided on progress InitSyncProgress(metadata.Tables.Cast<DataTable>().Select(dataTable => (long)dataTable.Rows.Count).Sum() + 3); // Prefix all children devices with the name of the parent since the same device names could appear in different connections (helps keep device names unique) string sourcePrefix = Name + "!"; Dictionary<string, int> deviceIDs = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); string deviceAcronym, signalTypeAcronym; decimal longitude, latitude; decimal? location; object originalSource; int deviceID; // Check to see if data for the "DeviceDetail" table was included in the meta-data if (metadata.Tables.Contains("DeviceDetail")) { DataTable deviceDetail = metadata.Tables["DeviceDetail"]; List<Guid> uniqueIDs = new List<Guid>(); DataRow[] deviceRows; // Define SQL statement to query if this device is already defined (this should always be based on the unique guid-based device ID) string deviceExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0}", "uniqueID"); // Define SQL statement to insert new device record string insertDeviceSql = database.ParameterizedQueryString("INSERT INTO Device(NodeID, ParentID, HistorianID, Acronym, Name, ProtocolID, FramesPerSecond, OriginalSource, AccessID, Longitude, Latitude, ContactList, IsConcentrator, Enabled) " + "VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, 0, 1)", "nodeID", "parentID", "historianID", "acronym", "name", "protocolID", "framesPerSecond", "originalSource", "accessID", "longitude", "latitude", "contactList"); // Define SQL statement to update device's guid-based unique ID after insert string updateDeviceUniqueIDSql = database.ParameterizedQueryString("UPDATE Device SET UniqueID = {0} WHERE Acronym = {1}", "uniqueID", "acronym"); // Define SQL statement to query if a device can be safely updated string deviceIsUpdatableSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0} AND (ParentID <> {1} OR ParentID IS NULL)", "uniqueID", "parentID"); // Define SQL statement to update existing device record string updateDeviceSql = database.ParameterizedQueryString("UPDATE Device SET Acronym = {0}, Name = {1}, OriginalSource = {2}, ProtocolID = {3}, FramesPerSecond = {4}, HistorianID = {5}, AccessID = {6}, Longitude = {7}, Latitude = {8}, ContactList = {9} WHERE UniqueID = {10}", "acronym", "name", "originalSource", "protocolID", "framesPerSecond", "historianID", "accessID", "longitude", "latitude", "contactList", "uniqueID"); // Define SQL statement to retrieve device's auto-inc ID based on its unique guid-based ID string queryDeviceIDSql = database.ParameterizedQueryString("SELECT ID FROM Device WHERE UniqueID = {0}", "uniqueID"); // Define SQL statement to retrieve all unique device ID's for the current parent to check for mismatches string queryUniqueDeviceIDsSql = database.ParameterizedQueryString("SELECT UniqueID FROM Device WHERE ParentID = {0}", "parentID"); // Define SQL statement to remove device records that no longer exist in the meta-data string deleteDeviceSql = database.ParameterizedQueryString("DELETE FROM Device WHERE UniqueID = {0}", "uniqueID"); // Determine which device rows should be synchronized based on operational mode flags if (ReceiveInternalMetadata && ReceiveExternalMetadata) deviceRows = deviceDetail.Select(); else if (ReceiveInternalMetadata) deviceRows = deviceDetail.Select("OriginalSource IS NULL"); else if (ReceiveExternalMetadata) deviceRows = deviceDetail.Select("OriginalSource IS NOT NULL"); else deviceRows = new DataRow[0]; // Check existence of optional meta-data fields DataColumnCollection deviceDetailColumns = deviceDetail.Columns; bool accessIDFieldExists = deviceDetailColumns.Contains("AccessID"); bool longitudeFieldExists = deviceDetailColumns.Contains("Longitude"); bool latitudeFieldExists = deviceDetailColumns.Contains("Latitude"); bool companyAcronymFieldExists = deviceDetailColumns.Contains("CompanyAcronym"); bool protocolNameFieldExists = deviceDetailColumns.Contains("ProtocolName"); bool vendorAcronymFieldExists = deviceDetailColumns.Contains("VendorAcronym"); bool vendorDeviceNameFieldExists = deviceDetailColumns.Contains("VendorDeviceName"); bool interconnectionNameFieldExists = deviceDetailColumns.Contains("InterconnectionName"); bool updatedOnFieldExists = deviceDetailColumns.Contains("UpdatedOn"); // Older versions of GEP did not include the AccessID field, so this is treated as optional int accessID = 0; foreach (DataRow row in deviceRows) { Guid uniqueID = Guid.Parse(row.Field<object>("UniqueID").ToString()); bool recordNeedsUpdating; // Track unique device Guids in this meta-data session, we'll need to remove any old associated devices that no longer exist uniqueIDs.Add(uniqueID); // Determine if record has changed since last synchronization if (updatedOnFieldExists) { try { updateTime = Convert.ToDateTime(row["UpdatedOn"]); recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime; if (updateTime > latestUpdateTime) latestUpdateTime = updateTime; } catch { recordNeedsUpdating = true; } } else { recordNeedsUpdating = true; } // We will synchronize meta-data only if the source owns this device and it's not defined as a concentrator (these should normally be filtered by publisher - but we check just in case). if (!row["IsConcentrator"].ToNonNullString("0").ParseBoolean()) { if (accessIDFieldExists) accessID = row.ConvertField<int>("AccessID"); // Get longitude and latitude values if they are defined longitude = 0M; latitude = 0M; if (longitudeFieldExists) { location = row.ConvertNullableField<decimal>("Longitude"); if (location.HasValue) longitude = location.Value; } if (latitudeFieldExists) { location = row.ConvertNullableField<decimal>("Latitude"); if (location.HasValue) latitude = location.Value; } // Save any reported extraneous values from device meta-data in connection string formatted contact list - all fields are considered optional Dictionary<string, string> contactList = new Dictionary<string, string>(); if (companyAcronymFieldExists) contactList["companyAcronym"] = row.Field<string>("CompanyAcronym") ?? string.Empty; if (protocolNameFieldExists) contactList["protocolName"] = row.Field<string>("ProtocolName") ?? string.Empty; if (vendorAcronymFieldExists) contactList["vendorAcronym"] = row.Field<string>("VendorAcronym") ?? string.Empty; if (vendorDeviceNameFieldExists) contactList["vendorDeviceName"] = row.Field<string>("VendorDeviceName") ?? string.Empty; if (interconnectionNameFieldExists) contactList["interconnectionName"] = row.Field<string>("InterconnectionName") ?? string.Empty; // Determine if device record already exists if (Convert.ToInt32(command.ExecuteScalar(deviceExistsSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID))) == 0) { // Insert new device record command.ExecuteNonQuery(insertDeviceSql, m_metadataSynchronizationTimeout, database.Guid(m_nodeID), parentID, historianID, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), m_gatewayProtocolID, row.ConvertField<int>("FramesPerSecond"), m_internal ? (object)DBNull.Value : string.IsNullOrEmpty(row.Field<string>("ParentAcronym")) ? sourcePrefix + row.Field<string>("Acronym") : sourcePrefix + row.Field<string>("ParentAcronym"), accessID, longitude, latitude, contactList.JoinKeyValuePairs()); // Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database // scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned. command.ExecuteNonQuery(updateDeviceUniqueIDSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID), sourcePrefix + row.Field<string>("Acronym")); } else if (recordNeedsUpdating) { // Perform safety check to preserve device records which are not safe to overwrite if (Convert.ToInt32(command.ExecuteScalar(deviceIsUpdatableSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID), parentID)) > 0) continue; // Gateway is assuming ownership of the device records when the "internal" flag is true - this means the device's measurements can be forwarded to another party. From a device record perspective, // ownership is inferred by setting 'OriginalSource' to null. When gateway doesn't own device records (i.e., the "internal" flag is false), this means the device's measurements can only be consumed // locally - from a device record perspective this means the 'OriginalSource' field is set to the acronym of the PDC or PMU that generated the source measurements. This field allows a mirrored source // restriction to be implemented later to ensure all devices in an output protocol came from the same original source connection, if desired. originalSource = m_internal ? (object)DBNull.Value : string.IsNullOrEmpty(row.Field<string>("ParentAcronym")) ? sourcePrefix + row.Field<string>("Acronym") : sourcePrefix + row.Field<string>("ParentAcronym"); // Update existing device record command.ExecuteNonQuery(updateDeviceSql, m_metadataSynchronizationTimeout, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), originalSource, m_gatewayProtocolID, row.ConvertField<int>("FramesPerSecond"), historianID, accessID, longitude, latitude, contactList.JoinKeyValuePairs(), database.Guid(uniqueID)); } } // Capture local device ID auto-inc value for measurement association deviceIDs[row.Field<string>("Acronym")] = Convert.ToInt32(command.ExecuteScalar(queryDeviceIDSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID))); // Periodically notify user about synchronization progress UpdateSyncProgress(); } // Remove any device records associated with this subscriber that no longer exist in the meta-data if (uniqueIDs.Count > 0) { // Sort unique ID list so that binary search can be used for quick lookups uniqueIDs.Sort(); DataTable deviceUniqueIDs = command.RetrieveData(database.AdapterType, queryUniqueDeviceIDsSql, m_metadataSynchronizationTimeout, parentID); Guid uniqueID; foreach (DataRow deviceRow in deviceUniqueIDs.Rows) { uniqueID = database.Guid(deviceRow, "UniqueID"); // Remove any devices in the database that are associated with the parent device and do not exist in the meta-data if (uniqueIDs.BinarySearch(uniqueID) < 0) command.ExecuteNonQuery(deleteDeviceSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID)); } UpdateSyncProgress(); } } // Check to see if data for the "MeasurementDetail" table was included in the meta-data if (metadata.Tables.Contains("MeasurementDetail")) { DataTable measurementDetail = metadata.Tables["MeasurementDetail"]; List<Guid> signalIDs = new List<Guid>(); DataRow[] measurementRows; // Define SQL statement to query if this measurement is already defined (this should always be based on the unique signal ID Guid) string measurementExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Measurement WHERE SignalID = {0}", "signalID"); // Define SQL statement to insert new measurement record string insertMeasurementSql = database.ParameterizedQueryString("INSERT INTO Measurement(DeviceID, HistorianID, PointTag, AlternateTag, SignalTypeID, PhasorSourceIndex, SignalReference, Description, Internal, Subscribed, Enabled) " + "VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, 0, 1)", "deviceID", "historianID", "pointTag", "alternateTag", "signalTypeID", "phasorSourceIndex", "signalReference", "description", "internal"); // Define SQL statement to update measurement's signal ID after insert string updateMeasurementSignalIDSql = database.ParameterizedQueryString("UPDATE Measurement SET SignalID = {0}, AlternateTag = NULL WHERE AlternateTag = {1}", "signalID", "alternateTag"); // Define SQL statement to update existing measurement record string updateMeasurementSql = database.ParameterizedQueryString("UPDATE Measurement SET HistorianID = {0}, PointTag = {1}, SignalTypeID = {2}, PhasorSourceIndex = {3}, SignalReference = {4}, Description = {5}, Internal = {6} WHERE SignalID = {7}", "historianID", "pointTag", "signalTypeID", "phasorSourceIndex", "signalReference", "description", "internal", "signalID"); // Define SQL statement to retrieve all measurement signal ID's for the current parent to check for mismatches - note that we use the ActiveMeasurements view // since it associates measurements with their top-most parent runtime device ID, this allows us to easily query all measurements for the parent device string queryMeasurementSignalIDsSql = database.ParameterizedQueryString("SELECT SignalID FROM ActiveMeasurement WHERE DeviceID = {0}", "deviceID"); // Define SQL statement to retrieve measurement's associated device ID, i.e., actual record ID, based on measurement's signal ID string queryMeasurementDeviceIDSql = database.ParameterizedQueryString("SELECT DeviceID FROM Measurement WHERE SignalID = {0}", "signalID"); // Define SQL statement to remove device records that no longer exist in the meta-data string deleteMeasurementSql = database.ParameterizedQueryString("DELETE FROM Measurement WHERE SignalID = {0}", "signalID"); // Load signal type ID's from local database associated with their acronym for proper signal type translation Dictionary<string, int> signalTypeIDs = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase); foreach (DataRow row in command.RetrieveData(database.AdapterType, "SELECT ID, Acronym FROM SignalType").Rows) { signalTypeAcronym = row.Field<string>("Acronym"); if (!string.IsNullOrWhiteSpace(signalTypeAcronym)) signalTypeIDs[signalTypeAcronym] = row.ConvertField<int>("ID"); } // Determine which measurement rows should be synchronized based on operational mode flags if (ReceiveInternalMetadata && ReceiveExternalMetadata) measurementRows = measurementDetail.Select(); else if (ReceiveInternalMetadata) measurementRows = measurementDetail.Select("Internal <> 0"); else if (ReceiveExternalMetadata) measurementRows = measurementDetail.Select("Internal = 0"); else measurementRows = new DataRow[0]; // Check existence of optional meta-data fields DataColumnCollection measurementDetailColumns = measurementDetail.Columns; bool phasorSourceIndexFieldExists = measurementDetailColumns.Contains("PhasorSourceIndex"); bool updatedOnFieldExists = measurementDetailColumns.Contains("UpdatedOn"); object phasorSourceIndex = DBNull.Value; foreach (DataRow row in measurementRows) { bool recordNeedsUpdating; // Determine if record has changed since last synchronization if (updatedOnFieldExists) { try { updateTime = Convert.ToDateTime(row["UpdatedOn"]); recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime; if (updateTime > latestUpdateTime) latestUpdateTime = updateTime; } catch { recordNeedsUpdating = true; } } else { recordNeedsUpdating = true; } // Get device and signal type acronyms deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty; signalTypeAcronym = row.Field<string>("SignalAcronym") ?? string.Empty; // Get phasor source index if field is defined if (phasorSourceIndexFieldExists) { // Using ConvertNullableField extension since publisher could use SQLite database in which case // all integers would arrive in data set as longs and need to be converted back to integers int? index = row.ConvertNullableField<int>("PhasorSourceIndex"); phasorSourceIndex = index.HasValue ? (object)index.Value : (object)DBNull.Value; } // Make sure we have an associated device and signal type already defined for the measurement if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym) && !string.IsNullOrWhiteSpace(signalTypeAcronym) && signalTypeIDs.ContainsKey(signalTypeAcronym)) { Guid signalID = Guid.Parse(row.Field<object>("SignalID").ToString()); // Track unique measurement signal Guids in this meta-data session, we'll need to remove any old associated measurements that no longer exist signalIDs.Add(signalID); // Prefix the tag name with the "updated" device name string pointTag = sourcePrefix + row.Field<string>("PointTag"); // Look up associated device ID (local DB auto-inc) deviceID = deviceIDs[deviceAcronym]; // Determine if measurement record already exists if (Convert.ToInt32(command.ExecuteScalar(measurementExistsSql, m_metadataSynchronizationTimeout, database.Guid(signalID))) == 0) { string alternateTag = Guid.NewGuid().ToString(); // Insert new measurement record command.ExecuteNonQuery(insertMeasurementSql, m_metadataSynchronizationTimeout, deviceID, historianID, pointTag, alternateTag, signalTypeIDs[signalTypeAcronym], phasorSourceIndex, sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal)); // Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database // scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned. command.ExecuteNonQuery(updateMeasurementSignalIDSql, m_metadataSynchronizationTimeout, database.Guid(signalID), alternateTag); } else if (recordNeedsUpdating) { // Update existing measurement record. Note that this update assumes that measurements will remain associated with a static source device. command.ExecuteNonQuery(updateMeasurementSql, m_metadataSynchronizationTimeout, historianID, pointTag, signalTypeIDs[signalTypeAcronym], phasorSourceIndex, sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal), database.Guid(signalID)); } } // Periodically notify user about synchronization progress UpdateSyncProgress(); } // Remove any measurement records associated with existing devices in this session but no longer exist in the meta-data if (signalIDs.Count > 0) { // Sort signal ID list so that binary search can be used for quick lookups signalIDs.Sort(); // Query all the guid-based signal ID's for all measurement records associated with the parent device using run-time ID DataTable measurementSignalIDs = command.RetrieveData(database.AdapterType, queryMeasurementSignalIDsSql, m_metadataSynchronizationTimeout, (int)ID); Guid signalID; // Walk through each database record and see if the measurement exists in the provided meta-data foreach (DataRow measurementRow in measurementSignalIDs.Rows) { signalID = database.Guid(measurementRow, "SignalID"); // Remove any measurements in the database that are associated with received devices and do not exist in the meta-data if (signalIDs.BinarySearch(signalID) < 0) { // Measurement was not in the meta-data, get the measurement's actual record based ID for its associated device object measurementDeviceID = command.ExecuteScalar(queryMeasurementDeviceIDSql, m_metadataSynchronizationTimeout, database.Guid(signalID)); // If the unknown measurement is directly associated with a device that exists in the meta-data it is assumed that this measurement // was removed from the publishing system and no longer exists therefore we remove it from the local measurement cache. If the user // needs custom local measurements associated with a remote device, they should be associated with the parent device only. if (measurementDeviceID != null && !(measurementDeviceID is DBNull) && deviceIDs.ContainsValue(Convert.ToInt32(measurementDeviceID))) command.ExecuteNonQuery(deleteMeasurementSql, m_metadataSynchronizationTimeout, database.Guid(signalID)); } } UpdateSyncProgress(); } } // Check to see if data for the "PhasorDetail" table was included in the meta-data if (metadata.Tables.Contains("PhasorDetail")) { Dictionary<int, int> maxSourceIndicies = new Dictionary<int, int>(); int sourceIndex; // Phasor data is normally only needed so that the user can properly generate a mirrored IEEE C37.118 output stream from the source data. // This is necessary since, in this protocol, the phasors are described (i.e., labeled) as a unit (i.e., as a complex number) instead of // as two distinct angle and magnitude measurements. // Define SQL statement to query if phasor record is already defined (no Guid is defined for these simple label records) string phasorExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Phasor WHERE DeviceID = {0} AND SourceIndex = {1}", "deviceID", "sourceIndex"); // Define SQL statement to insert new phasor record string insertPhasorSql = database.ParameterizedQueryString("INSERT INTO Phasor(DeviceID, Label, Type, Phase, SourceIndex) VALUES ({0}, {1}, {2}, {3}, {4})", "deviceID", "label", "type", "phase", "sourceIndex"); // Define SQL statement to update existing phasor record string updatePhasorSql = database.ParameterizedQueryString("UPDATE Phasor SET Label = {0}, Type = {1}, Phase = {2} WHERE DeviceID = {3} AND SourceIndex = {4}", "label", "type", "phase", "deviceID", "sourceIndex"); // Define SQL statement to delete a phasor record string deletePhasorSql = database.ParameterizedQueryString("DELETE FROM Phasor WHERE DeviceID = {0} AND SourceIndex > {1}", "deviceID", "sourceIndex"); foreach (DataRow row in metadata.Tables["PhasorDetail"].Rows) { // Get device acronym deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty; // Make sure we have an associated device already defined for the phasor record if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym)) { bool recordNeedsUpdating; // Determine if record has changed since last synchronization try { updateTime = Convert.ToDateTime(row["UpdatedOn"]); recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime; if (updateTime > latestUpdateTime) latestUpdateTime = updateTime; } catch { recordNeedsUpdating = true; } deviceID = deviceIDs[deviceAcronym]; // Determine if phasor record already exists if (Convert.ToInt32(command.ExecuteScalar(phasorExistsSql, m_metadataSynchronizationTimeout, deviceID, row.ConvertField<int>("SourceIndex"))) == 0) { // Insert new phasor record command.ExecuteNonQuery(insertPhasorSql, m_metadataSynchronizationTimeout, deviceID, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), row.ConvertField<int>("SourceIndex")); } else if (recordNeedsUpdating) { // Update existing phasor record command.ExecuteNonQuery(updatePhasorSql, m_metadataSynchronizationTimeout, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), deviceID, row.ConvertField<int>("SourceIndex")); } // Track largest source index for each device maxSourceIndicies.TryGetValue(deviceID, out sourceIndex); if (row.ConvertField<int>("SourceIndex") > sourceIndex) maxSourceIndicies[deviceID] = row.ConvertField<int>("SourceIndex"); } // Periodically notify user about synchronization progress UpdateSyncProgress(); } // Remove any phasor records associated with existing devices in this session but no longer exist in the meta-data if (maxSourceIndicies.Count > 0) { foreach (KeyValuePair<int, int> deviceIndexPair in maxSourceIndicies) { command.ExecuteNonQuery(deletePhasorSql, m_metadataSynchronizationTimeout, deviceIndexPair.Key, deviceIndexPair.Value); } } } if ((object)transaction != null) transaction.Commit(); // Update local in-memory synchronized meta-data cache m_synchronizedMetadata = metadata; } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex)); if ((object)transaction != null) { try { transaction.Rollback(); } catch (Exception rollbackException) { OnProcessException(new InvalidOperationException("Failed to roll back database transaction due to exception: " + rollbackException.Message, rollbackException)); } } return; } finally { if ((object)transaction != null) transaction.Dispose(); } } // New signals may have been defined, take original remote signal index cache and apply changes if (m_remoteSignalIndexCache != null) m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache); m_lastMetaDataRefreshTime = latestUpdateTime > DateTime.MinValue ? latestUpdateTime : DateTime.UtcNow; OnStatusMessage("Meta-data synchronization completed successfully in {0}", (DateTime.UtcNow.Ticks - startTime).ToElapsedTimeString(2)); // Send notification that system configuration has changed OnConfigurationChanged(); } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex)); } finally { // Restart data stream monitor after meta-data synchronization if it was originally enabled if (dataMonitoringEnabled && (object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = true; } }
public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache) : this(measurement, signalIndexCache, true, null, 0 , false) { }
/// <summary> /// Creates a new <see cref="UnsynchronizedClientSubscription"/>. /// </summary> /// <param name="parent">Reference to parent.</param> /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param> /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param> public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID) { // Pass parent reference into base class AssignParentCollection(parent); m_parent = parent; m_clientID = clientID; m_subscriberID = subscriberID; m_signalIndexCache = new SignalIndexCache() { SubscriberID = subscriberID }; m_processQueue = new AsyncQueue<IEnumerable<IMeasurement>>() { ProcessItemFunction = ProcessMeasurements }; m_processQueue.ProcessException += m_processQueue_ProcessException; }
private void ProcessServerResponse(byte[] buffer, int length) { // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread... if (buffer != null && length > 0) { try { Dictionary<Guid, DeviceStatisticsHelper<SubscribedDevice>> subscribedDevicesLookup; DeviceStatisticsHelper<SubscribedDevice> statisticsHelper; ServerResponse responseCode = (ServerResponse)buffer[0]; ServerCommand commandCode = (ServerCommand)buffer[1]; int responseLength = BigEndian.ToInt32(buffer, 2); int responseIndex = DataPublisher.ClientResponseHeaderSize; bool solicited = false; byte[][][] keyIVs; // See if this was a solicited response to a requested server command if (responseCode.IsSolicited()) { lock (m_requests) { int index = m_requests.BinarySearch(commandCode); if (index >= 0) { solicited = true; m_requests.RemoveAt(index); } } // Disconnect any established UDP data channel upon successful unsubscribe if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded) DataChannel = null; } OnReceivedServerResponse(responseCode, commandCode); switch (responseCode) { case ServerResponse.Succeeded: if (solicited) { switch (commandCode) { case ServerCommand.Authenticate: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_authenticated = true; OnConnectionAuthenticated(); break; case ServerCommand.Subscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = true; break; case ServerCommand.Unsubscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = false; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = false; break; case ServerCommand.RotateCipherKeys: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.MetaDataRefresh: OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); m_metadataRefreshPending = false; break; } } else { switch (commandCode) { case ServerCommand.MetaDataRefresh: // Meta-data refresh may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); m_metadataRefreshPending = false; break; case ServerCommand.RotateCipherKeys: // Key rotation may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.Subscribe: OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; default: OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode)); break; } } break; case ServerResponse.Failed: if (solicited) OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); else OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode)); if (commandCode == ServerCommand.MetaDataRefresh) m_metadataRefreshPending = false; break; case ServerResponse.DataPacket: long now = DateTime.UtcNow.Ticks; // Deserialize data packet List<IMeasurement> measurements = new List<IMeasurement>(); DataPacketFlags flags; Ticks timestamp = 0; int count; if (m_totalBytesReceived == 0) { // At the point when data is being received, data monitor should be enabled if ((object)m_dataStreamMonitor != null && !m_dataStreamMonitor.Enabled) m_dataStreamMonitor.Enabled = true; // Establish run-time log for subscriber if (m_autoConnect || m_dataGapRecoveryEnabled) { if ((object)m_runTimeLog == null) { m_runTimeLog = new RunTimeLog(); m_runTimeLog.FileName = GetLoggingPath(Name + "_RunTimeLog.txt"); m_runTimeLog.ProcessException += m_runTimeLog_ProcessException; m_runTimeLog.Initialize(); } else { // Mark the start of any data transmissions m_runTimeLog.StartTime = DateTime.UtcNow; m_runTimeLog.Enabled = true; } } // The duration between last disconnection and start of data transmissions // represents a gap in data - if data gap recovery is enabled, we log // this as a gap for recovery: if (m_dataGapRecoveryEnabled && (object)m_dataGapRecoverer != null) m_dataGapRecoverer.LogDataGap(m_runTimeLog.StopTime, DateTime.UtcNow); } // Track total data packet bytes received from any channel m_totalBytesReceived += m_lastBytesReceived; m_monitoredBytesReceived += m_lastBytesReceived; // Get data packet flags flags = (DataPacketFlags)buffer[responseIndex]; responseIndex++; bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0); bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0); bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0); int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0; // Decrypt data packet payload if keys are available if ((object)m_keyIVs != null) { // Get a local copy of volatile keyIVs reference since this can change at any time keyIVs = m_keyIVs; // Decrypt payload portion of data packet buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]); responseIndex = 0; responseLength = buffer.Length; } // Synchronized packets contain a frame level timestamp if (synchronizedMeasurements) { timestamp = BigEndian.ToInt64(buffer, responseIndex); responseIndex += 8; } // Deserialize number of measurements that follow count = BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; if (compressedPayload) { if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } else { try { if (CompressionModes.HasFlag(CompressionModes.TSSC)) { // Use TSSC compression to decompress measurements if ((object)m_decompressionBlock == null) m_decompressionBlock = new MeasurementDecompressionBlock(); MemoryStream bufferStream = new MemoryStream(buffer, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize); bool eos = false; while (!eos) { Measurement measurement; Tuple<Guid, string, uint> tuple; ushort id; long time; uint quality; float value; byte command; switch (m_decompressionBlock.GetMeasurement(out id, out time, out quality, out value, out command)) { case DecompressionExitCode.EndOfStreamOccured: if (bufferStream.Position != bufferStream.Length) m_decompressionBlock.Fill(bufferStream); else eos = true; break; case DecompressionExitCode.CommandRead: break; case DecompressionExitCode.MeasurementRead: // Attempt to restore signal identification if (m_signalIndexCache.Reference.TryGetValue(id, out tuple)) { measurement = new Measurement(); measurement.Key = MeasurementKey.LookUpOrCreate(tuple.Item1, tuple.Item2, tuple.Item3); measurement.Timestamp = time; measurement.StateFlags = (MeasurementStateFlags)quality; measurement.Value = value; measurements.Add(measurement); } break; } } } else { // Decompress compact measurements from payload measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags)); } } catch (Exception ex) { OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex)); } } } else { // Deserialize measurements for (int i = 0; i < count; i++) { if (!compactMeasurementFormat) { // Deserialize full measurement format SerializableMeasurement measurement = new SerializableMeasurement(m_encoding); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); measurements.Add(measurement); } else if ((object)m_signalIndexCache != null) { // Deserialize compact measurement format CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); // Apply timestamp from frame if not included in transmission if (!measurement.IncludeTime) measurement.Timestamp = timestamp; measurements.Add(measurement); } else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } } } // Calculate statistics subscribedDevicesLookup = m_subscribedDevicesLookup; statisticsHelper = null; if ((object)subscribedDevicesLookup != null) { IEnumerable<IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement>> deviceGroups = measurements .Where(measurement => subscribedDevicesLookup.TryGetValue(measurement.ID, out statisticsHelper)) .Select(measurement => Tuple.Create(statisticsHelper, measurement)) .ToList() .GroupBy(tuple => tuple.Item1, tuple => tuple.Item2); foreach (IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement> deviceGroup in deviceGroups) { statisticsHelper = deviceGroup.Key; foreach (IGrouping<Ticks, IMeasurement> frame in deviceGroup.GroupBy(measurement => measurement.Timestamp)) { // Determine the number of measurements received with valid values int measurementsReceived = frame.Count(measurement => !double.IsNaN(measurement.Value)); IMeasurement statusFlags = null; IMeasurement frequency = null; IMeasurement deltaFrequency = null; // Attempt to update real-time if (!m_useLocalClockAsRealTime && frame.Key > m_realTime) m_realTime = frame.Key; // Search the frame for status flags, frequency, and delta frequency foreach (IMeasurement measurement in frame) { if (measurement.ID == statisticsHelper.Device.StatusFlagsID) statusFlags = measurement; else if (measurement.ID == statisticsHelper.Device.FrequencyID) frequency = measurement; else if (measurement.ID == statisticsHelper.Device.DeltaFrequencyID) deltaFrequency = measurement; } // If we are receiving status flags for this device, // count the data quality, time quality, and device errors if ((object)statusFlags != null) { uint commonStatusFlags = (uint)statusFlags.Value; if ((commonStatusFlags & (uint)Bits.Bit19) > 0) statisticsHelper.Device.DataQualityErrors++; if ((commonStatusFlags & (uint)Bits.Bit18) > 0) statisticsHelper.Device.TimeQualityErrors++; if ((commonStatusFlags & (uint)Bits.Bit16) > 0) statisticsHelper.Device.DeviceErrors++; measurementsReceived--; } // Zero is not a valid value for frequency. // If frequency is zero, invalidate both frequency and delta frequency if ((object)frequency != null && frequency.Value == 0.0D) { if ((object)deltaFrequency != null) measurementsReceived -= 2; else measurementsReceived--; } // Track the number of measurements received statisticsHelper.AddToMeasurementsReceived(measurementsReceived); } } } // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer if ((object)m_localConcentrator != null) m_localConcentrator.SortMeasurements(measurements); else OnNewMeasurements(measurements); // Gather statistics on received data DateTime timeReceived = RealTime; if (!m_useLocalClockAsRealTime && timeReceived.Ticks - m_lastStatisticsHelperUpdate > Ticks.PerSecond) { UpdateStatisticsHelpers(); m_lastStatisticsHelperUpdate = m_realTime; } m_lifetimeMeasurements += measurements.Count; UpdateMeasurementsPerSecond(timeReceived, measurements.Count); for (int x = 0; x < measurements.Count; x++) { long latency = timeReceived.Ticks - (long)measurements[x].Timestamp; // Throw out latencies that exceed one hour as invalid if (Math.Abs(latency) > Time.SecondsPerHour * Ticks.PerSecond) continue; if (m_lifetimeMinimumLatency > latency || m_lifetimeMinimumLatency == 0) m_lifetimeMinimumLatency = latency; if (m_lifetimeMaximumLatency < latency || m_lifetimeMaximumLatency == 0) m_lifetimeMaximumLatency = latency; m_lifetimeTotalLatency += latency; m_lifetimeLatencyMeasurements++; } break; case ServerResponse.BufferBlock: // Buffer block received - wrap as a buffer block measurement and expose back to consumer uint sequenceNumber = BigEndian.ToUInt32(buffer, responseIndex); int cacheIndex = (int)(sequenceNumber - m_expectedBufferBlockSequenceNumber); BufferBlockMeasurement bufferBlockMeasurement; Tuple<Guid, string, uint> measurementKey; ushort signalIndex; // Check if this buffer block has already been processed (e.g., mistaken retransmission due to timeout) if (cacheIndex >= 0 && (cacheIndex >= m_bufferBlockCache.Count || (object)m_bufferBlockCache[cacheIndex] == null)) { // Send confirmation that buffer block is received SendServerCommand(ServerCommand.ConfirmBufferBlock, buffer.BlockCopy(responseIndex, 4)); // Get measurement key from signal index cache signalIndex = BigEndian.ToUInt16(buffer, responseIndex + 4); if (!m_signalIndexCache.Reference.TryGetValue(signalIndex, out measurementKey)) throw new InvalidOperationException("Failed to find associated signal identification for runtime ID " + signalIndex); // Skip the sequence number and signal index when creating the buffer block measurement bufferBlockMeasurement = new BufferBlockMeasurement(buffer, responseIndex + 6, responseLength - 6) { Key = MeasurementKey.LookUpOrCreate(measurementKey.Item1, measurementKey.Item2, measurementKey.Item3) }; // Determine if this is the next buffer block in the sequence if (sequenceNumber == m_expectedBufferBlockSequenceNumber) { List<IMeasurement> bufferBlockMeasurements = new List<IMeasurement>(); int i; // Add the buffer block measurement to the list of measurements to be published bufferBlockMeasurements.Add(bufferBlockMeasurement); m_expectedBufferBlockSequenceNumber++; // Add cached buffer block measurements to the list of measurements to be published for (i = 1; i < m_bufferBlockCache.Count; i++) { if ((object)m_bufferBlockCache[i] == null) break; bufferBlockMeasurements.Add(m_bufferBlockCache[i]); m_expectedBufferBlockSequenceNumber++; } // Remove published measurements from the buffer block queue if (m_bufferBlockCache.Count > 0) m_bufferBlockCache.RemoveRange(0, i); // Publish measurements OnNewMeasurements(bufferBlockMeasurements); } else { // Ensure that the list has at least as many // elements as it needs to cache this measurement for (int i = m_bufferBlockCache.Count; i <= cacheIndex; i++) m_bufferBlockCache.Add(null); // Insert this buffer block into the proper location in the list m_bufferBlockCache[cacheIndex] = bufferBlockMeasurement; } } m_lifetimeMeasurements += 1; UpdateMeasurementsPerSecond(DateTime.UtcNow, 1); break; case ServerResponse.DataStartTime: // Raise data start time event OnDataStartTime(BigEndian.ToInt64(buffer, responseIndex)); break; case ServerResponse.ProcessingComplete: // Raise input processing completed event OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerResponse.UpdateSignalIndexCache: // Deserialize new signal index cache m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength)); m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache); FixExpectedMeasurementCounts(); break; case ServerResponse.UpdateBaseTimes: // Get active time index m_timeIndex = BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; // Deserialize new base time offsets m_baseTimeOffsets = new[] { BigEndian.ToInt64(buffer, responseIndex), BigEndian.ToInt64(buffer, responseIndex + 8) }; break; case ServerResponse.UpdateCipherKeys: // Move past active cipher index (not currently used anywhere else) responseIndex++; // Extract remaining response byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1); // Decrypt response payload if subscription is authenticated if (m_authenticated) bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256); // Deserialize new cipher keys keyIVs = new byte[2][][]; keyIVs[EvenKey] = new byte[2][]; keyIVs[OddKey] = new byte[2][]; int index = 0; int bufferLen; // Read even key size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read even key keyIVs[EvenKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read even initialization vector size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read even initialization vector keyIVs[EvenKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen); index += bufferLen; // Read odd key size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read odd key keyIVs[OddKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read odd initialization vector size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read odd initialization vector keyIVs[OddKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen); //index += bufferLen; // Exchange keys m_keyIVs = keyIVs; OnStatusMessage("Successfully established new cipher keys for data packet transmissions."); break; case ServerResponse.Notify: // Skip the 4-byte hash string message = m_encoding.GetString(buffer, responseIndex + 4, responseLength - 4); // Display notification OnStatusMessage("NOTIFICATION: {0}", message); OnNotificationReceived(message); // Send confirmation of receipt of the notification SendServerCommand(ServerCommand.ConfirmNotification, buffer.BlockCopy(responseIndex, 4)); break; case ServerResponse.ConfigurationChanged: OnStatusMessage("Received notification from publisher that configuration has changed."); OnServerConfigurationChanged(); // Initiate meta-data refresh when publisher configuration has changed - we only do this // for automatic connections since API style connections have to manually initiate a // meta-data refresh. API style connection should attach to server configuration changed // event and request meta-data refresh to complete automated cycle. if (m_autoConnect && m_autoSynchronizeMetadata) SendServerCommand(ServerCommand.MetaDataRefresh, m_metadataFilters); break; } } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex)); } } }
private void ProcessServerResponse(byte[] buffer, int length) { // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread... if (buffer != null && length > 0) { try { ServerResponse responseCode = (ServerResponse)buffer[0]; ServerCommand commandCode = (ServerCommand)buffer[1]; int responseLength = EndianOrder.BigEndian.ToInt32(buffer, 2); int responseIndex = 6; bool solicited = false; byte[][][] keyIVs; // See if this was a solicited response to a requested server command if (responseCode.IsSolicited()) { lock (m_requests) { int index = m_requests.BinarySearch(commandCode); if (index >= 0) { solicited = true; m_requests.RemoveAt(index); } } // Disconnect any established UDP data channel upon successful unsubscribe if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded) DataChannel = null; } switch (responseCode) { case ServerResponse.Succeeded: if (solicited) { switch (commandCode) { case ServerCommand.Authenticate: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_authenticated = true; OnConnectionAuthenticated(); break; case ServerCommand.Subscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = true; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = true; break; case ServerCommand.Unsubscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = false; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = false; break; case ServerCommand.RotateCipherKeys: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.MetaDataRefresh: OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); break; } } else { switch (commandCode) { case ServerCommand.MetaDataRefresh: // Meta-data refresh may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); break; case ServerCommand.RotateCipherKeys: // Key rotation may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.Subscribe: OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; default: OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode)); break; } } break; case ServerResponse.Failed: if (solicited) OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); else OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode)); break; case ServerResponse.DataPacket: long now = DateTime.UtcNow.Ticks; // Deserialize data packet List<IMeasurement> measurements = new List<IMeasurement>(); DataPacketFlags flags; Ticks timestamp = 0; int count; // Track total data packet bytes received from any channel m_totalBytesReceived += m_lastBytesReceived; m_monitoredBytesReceived += m_lastBytesReceived; // Get data packet flags flags = (DataPacketFlags)buffer[responseIndex]; responseIndex++; bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0); bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0); int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0; // Decrypt data packet payload if keys are available // if (m_keyIVs != null) // { // // Get a local copy of volatile keyIVs reference since this can change at any time // keyIVs = m_keyIVs; // // // Decrypt payload portion of data packet // buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]); // responseIndex = 0; // responseLength = buffer.Length; // } // Synchronized packets contain a frame level timestamp if (synchronizedMeasurements) { timestamp = EndianOrder.BigEndian.ToInt64(buffer, responseIndex); responseIndex += 8; } // Deserialize number of measurements that follow count = EndianOrder.BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; // Deserialize measurements for (int i = 0; i < count; i++) { if (!compactMeasurementFormat) { // Deserialize full measurement format SerializableMeasurement measurement = new SerializableMeasurement(m_encoding); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); measurements.Add(measurement); } else if ((object)m_signalIndexCache != null) { // Deserialize compact measurement format CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); // Apply timestamp from frame if not included in transmission if (!measurement.IncludeTime) measurement.Timestamp = timestamp; measurements.Add(measurement); } else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } } // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer if ((object)m_localConcentrator != null) m_localConcentrator.SortMeasurements(measurements); else OnNewMeasurements(measurements); break; case ServerResponse.DataStartTime: // Raise data start time event OnDataStartTime(EndianOrder.BigEndian.ToInt64(buffer, responseIndex)); break; case ServerResponse.ProcessingComplete: // Raise input processing completed event OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerResponse.UpdateSignalIndexCache: // Deserialize new signal index cache m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength)); m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache); break; case ServerResponse.UpdateBaseTimes: // Get active time index m_timeIndex = EndianOrder.BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; // Deserialize new base time offsets m_baseTimeOffsets = new long[] { EndianOrder.BigEndian.ToInt64(buffer, responseIndex), EndianOrder.BigEndian.ToInt64(buffer, responseIndex + 8) }; break; case ServerResponse.UpdateCipherKeys: // Get active cipher index m_cipherIndex = buffer[responseIndex++]; // Extract remaining response byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1); // // Decrypt response payload if subscription is authenticated // if (m_authenticated) // bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256); // Deserialize new cipher keys keyIVs = new byte[2][][]; keyIVs[EvenKey] = new byte[2][]; keyIVs[OddKey] = new byte[2][]; int index = 0; int bufferLen; // Read even key size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read even key keyIVs[EvenKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read even initialization vector size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read even initialization vector keyIVs[EvenKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen); index += bufferLen; // Read odd key size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read odd key keyIVs[OddKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read odd initialization vector size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read odd initialization vector keyIVs[OddKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen); index += bufferLen; // Exchange keys m_keyIVs = keyIVs; OnStatusMessage("Successfully established new cipher keys for data packet transmissions."); break; } } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex)); } } }