Inheritance: ISupportBinaryImage
Example #1
0
        /// <summary>
        /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value.
        /// </summary>
        /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param>
        /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
        /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
        /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
        /// <param name="timeIndex">Time index to use for base offset.</param>
        /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
        public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false)
        {
            Key        = measurement.Key;
            Value      = measurement.Value;
            Adder      = measurement.Adder;
            Multiplier = measurement.Multiplier;
            Timestamp  = measurement.Timestamp;
            StateFlags = measurement.StateFlags;

            m_signalIndexCache = signalIndexCache;
            m_includeTime      = includeTime;

            // We keep a clone of the base time offsets, if provided, since array contents can change at any time
            if ((object)baseTimeOffsets == null)
            {
                m_baseTimeOffsets = s_emptyBaseTimeOffsets;
            }
            else
            {
                m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] }
            };

            m_timeIndex = timeIndex;
            m_useMillisecondResolution = useMillisecondResolution;
        }
Example #2
0
        /// <summary>
        /// Creates a new local system cache from one that was received remotely.
        /// </summary>
        /// <param name="dataSource"><see cref="DataSet"/> based data source used to interpret local measurement keys.</param>
        /// <param name="remoteCache">Deserialized remote signal index cache.</param>
        public SignalIndexCache(DataSet dataSource, SignalIndexCache remoteCache)
        {
            m_subscriberID = remoteCache.SubscriberID;

            // If active measurements are defined, interpret signal cache in context of current measurement key definitions
            if (dataSource != null && dataSource.Tables != null && dataSource.Tables.Contains("ActiveMeasurements"))
            {
                DataTable activeMeasurements = dataSource.Tables["ActiveMeasurements"];
                m_reference = new ConcurrentDictionary<ushort, Tuple<Guid, string, uint>>();

                foreach (KeyValuePair<ushort, Tuple<Guid, string, uint>> signalIndex in remoteCache.Reference)
                {
                    Guid signalID = signalIndex.Value.Item1;
                    DataRow[] filteredRows = activeMeasurements.Select("SignalID = '" + signalID.ToString() + "'");

                    if (filteredRows.Length > 0)
                    {
                        DataRow row = filteredRows[0];
                        MeasurementKey key = MeasurementKey.Parse(row["ID"].ToNonNullString(MeasurementKey.Undefined.ToString()), signalID);
                        m_reference.TryAdd(signalIndex.Key, new Tuple<Guid, string, uint>(signalID, key.Source, key.ID));
                    }
                }

                m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
            }
            else
            {
                // Just use remote signal index cache as-is if no local configuration exists
                m_reference = remoteCache.Reference;
                m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
            }
        }
Example #3
0
        /// <summary>
        /// Creates a new local system cache from one that was received remotely.
        /// </summary>
        /// <param name="dataSource"><see cref="DataSet"/> based data source used to interpret local measurement keys.</param>
        /// <param name="remoteCache">Deserialized remote signal index cache.</param>
        public SignalIndexCache(DataSet dataSource, SignalIndexCache remoteCache)
        {
            m_subscriberID = remoteCache.SubscriberID;

            // If active measurements are defined, interpret signal cache in context of current measurement key definitions
            if (dataSource != null && dataSource.Tables.Contains("ActiveMeasurements"))
            {
                DataTable activeMeasurements = dataSource.Tables["ActiveMeasurements"];
                m_reference = new ConcurrentDictionary <ushort, MeasurementKey>();

                foreach (KeyValuePair <ushort, MeasurementKey> signalIndex in remoteCache.Reference)
                {
                    Guid      signalID     = signalIndex.Value.SignalID;
                    DataRow[] filteredRows = activeMeasurements.Select("SignalID = '" + signalID.ToString() + "'");

                    if (filteredRows.Length > 0)
                    {
                        DataRow        row = filteredRows[0];
                        MeasurementKey key = MeasurementKey.LookUpOrCreate(signalID, row["ID"].ToNonNullString(MeasurementKey.Undefined.ToString()));
                        m_reference.TryAdd(signalIndex.Key, key);
                    }
                }

                m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
            }
            else
            {
                // Just use remote signal index cache as-is if no local configuration exists
                m_reference             = remoteCache.Reference;
                m_unauthorizedSignalIDs = remoteCache.UnauthorizedSignalIDs;
            }
        }
        /// <summary>
        /// Creates a new <see cref="SynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
        {
            // Pass parent reference into base class
            AssignParentCollection(parent);

            m_parent = parent;
            m_clientID = clientID;
            m_subscriberID = subscriberID;
            m_signalIndexCache = new SignalIndexCache()
            {
                SubscriberID = subscriberID
            };
        }
Example #5
0
        /// <summary>
        /// Creates a new <see cref="UnsynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
        {
            m_parent       = parent;
            m_clientID     = clientID;
            m_subscriberID = subscriberID;

            m_signalIndexCache = new SignalIndexCache();
            m_signalIndexCache.SubscriberID = subscriberID;

            m_workingBuffer        = new BlockAllocatedMemoryStream();
            m_bufferBlockCache     = new List <byte[]>();
            m_bufferBlockCacheLock = new object();
        }
Example #6
0
        /// <summary>
        /// Creates a new <see cref="SynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
        {
            // Pass parent reference into base class
            AssignParentCollection(parent);

            m_parent           = parent;
            m_clientID         = clientID;
            m_subscriberID     = subscriberID;
            m_signalIndexCache = new SignalIndexCache()
            {
                SubscriberID = subscriberID
            };
        }
        /// <summary>
        /// Creates a new <see cref="SynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        /// <param name="compressionModes"><see cref="CompressionModes"/> requested by client.</param>
        public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID, CompressionModes compressionModes)
        {
            m_parent           = parent;
            m_clientID         = clientID;
            m_subscriberID     = subscriberID;
            m_compressionModes = compressionModes;

            m_signalIndexCache = new SignalIndexCache();
            m_signalIndexCache.SubscriberID = subscriberID;

            m_bufferBlockCache     = new List <byte[]>();
            m_bufferBlockCacheLock = new object();
        }
Example #8
0
        /// <summary>
        /// Creates a new <see cref="CompactMeasurement"/>.
        /// </summary>
        /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
        /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
        /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
        /// <param name="timeIndex">Time index to use for base offset.</param>
        /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
        public CompactMeasurement(SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false)
        {
            m_signalIndexCache = signalIndexCache;
            m_includeTime      = includeTime;

            // We keep a clone of the base time offsets, if provided, since array contents can change at any time
            if ((object)baseTimeOffsets == null)
            {
                m_baseTimeOffsets = s_emptyBaseTimeOffsets;
            }
            else
            {
                m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] }
            };

            m_timeIndex = timeIndex;
            m_useMillisecondResolution = useMillisecondResolution;
        }
Example #9
0
        /// <summary>
        /// Creates a new <see cref="UnsynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
        {
            // Pass parent reference into base class
            AssignParentCollection(parent);

            m_parent       = parent;
            m_clientID     = clientID;
            m_subscriberID = subscriberID;

            m_signalIndexCache = new SignalIndexCache()
            {
                SubscriberID = subscriberID
            };

            m_processQueue = new AsyncQueue <IEnumerable <IMeasurement> >()
            {
                ProcessItemFunction = ProcessMeasurements
            };

            m_processQueue.ProcessException += m_processQueue_ProcessException;
        }
        /// <summary>
        /// Creates a new <see cref="SynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        /// <param name="compressionModes"><see cref="CompressionModes"/> requested by client.</param>
        public SynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID, CompressionModes compressionModes)
        {
            m_parent = parent;
            m_clientID = clientID;
            m_subscriberID = subscriberID;
            m_compressionModes = compressionModes;

            m_signalIndexCache = new SignalIndexCache();
            m_signalIndexCache.SubscriberID = subscriberID;

            m_bufferBlockCache = new List<byte[]>();
            m_bufferBlockCacheLock = new object();
        }
Example #11
0
        /// <summary>
        /// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
        /// </summary>
        /// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
        /// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
        /// <param name="index">Index into buffer where compressed payload begins.</param>
        /// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
        /// <param name="measurementCount">Number of compressed measurements in the payload.</param>
        /// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
        /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
        /// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
        public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
        {
            CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];

            // Actual data length has to take into account response byte and in-response-to server command byte in the payload header
            //int dataLength = length - index - 2;
            int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);

            // Copy source data into a decompression buffer
            byte[] buffer = new byte[bufferLength];
            Buffer.BlockCopy(source, index, buffer, 0, dataLength);

            // Check that OS endian-order matches endian-order of compressed data
            if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
            {
                // can be modified to decompress a payload that is in a non-native Endian order
                throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
            }

            // Attempt to decompress buffer
            int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);

            if (uncompressedSize == 0)
            {
                throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");
            }

            index = 0;

            // Decode ID and state flags
            for (int i = 0; i < measurementCount; i++)
            {
                uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);

                measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
                {
                    CompactStateFlags = (byte)(value >> 16),
                    RuntimeID         = (ushort)value
                };

                index += 4;
            }

            // Decode values
            for (int i = 0; i < measurementCount; i++)
            {
                measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
                index += 4;
            }

            if (includeTime)
            {
                // Decode timestamps
                for (int i = 0; i < measurementCount; i++)
                {
                    measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
                    index += 8;
                }
            }

            return(measurements);
        }
Example #12
0
        private SignalIndexCache DeserializeSignalIndexCache(byte[] buffer)
        {
            CompressionModes compressionModes = (CompressionModes)(m_operationalModes & OperationalModes.CompressionModeMask);
            bool useCommonSerializationFormat = (m_operationalModes & OperationalModes.UseCommonSerializationFormat) > 0;
            bool compressSignalIndexCache = (m_operationalModes & OperationalModes.CompressSignalIndexCache) > 0;

            SignalIndexCache deserializedCache;

            GZipStream inflater = null;

            if (compressSignalIndexCache && compressionModes.HasFlag(CompressionModes.GZip))
            {
                try
                {
                    using (MemoryStream compressedData = new MemoryStream(buffer))
                    {
                        inflater = new GZipStream(compressedData, CompressionMode.Decompress, true);
                        buffer = inflater.ReadStream();
                    }
                }
                finally
                {
                    if ((object)inflater != null)
                        inflater.Close();
                }
            }

            if (useCommonSerializationFormat)
            {
                deserializedCache = new SignalIndexCache();
                deserializedCache.Encoding = m_encoding;
                deserializedCache.ParseBinaryImage(buffer, 0, buffer.Length);
            }
            else
            {
                deserializedCache = Serialization.Deserialize<SignalIndexCache>(buffer, SerializationFormat.Binary);
            }

            return deserializedCache;
        }
        /// <summary>
        /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value.
        /// </summary>
        /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param>
        /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
        /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
        /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
        /// <param name="timeIndex">Time index to use for base offset.</param>
        /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
        public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime = true, long[] baseTimeOffsets = null, int timeIndex = 0, bool useMillisecondResolution = false)
        {
            Metadata = measurement.Metadata;
            Value = measurement.Value;
            Timestamp = measurement.Timestamp;
            StateFlags = measurement.StateFlags;

            m_signalIndexCache = signalIndexCache;
            m_includeTime = includeTime;

            // We keep a clone of the base time offsets, if provided, since array contents can change at any time
            if ((object)baseTimeOffsets == null)
                m_baseTimeOffsets = s_emptyBaseTimeOffsets;
            else
                m_baseTimeOffsets = new[] { baseTimeOffsets[0], baseTimeOffsets[1] };

            m_timeIndex = timeIndex;
            m_useMillisecondResolution = useMillisecondResolution;
        }
Example #14
0
        private SignalIndexCache DeserializeSignalIndexCache(byte[] buffer)
        {
            GatewayCompressionMode gatewayCompressionMode = (GatewayCompressionMode)(m_operationalModes & OperationalModes.CompressionModeMask);
            bool useCommonSerializationFormat = (m_operationalModes & OperationalModes.UseCommonSerializationFormat) > 0;
            bool compressSignalIndexCache = (m_operationalModes & OperationalModes.CompressSignalIndexCache) > 0;

            SignalIndexCache deserializedCache;

            MemoryStream compressedData = null;
            GZipStream inflater = null;

            if (compressSignalIndexCache && gatewayCompressionMode == GatewayCompressionMode.GZip)
            {
                try
                {
                    compressedData = new MemoryStream(buffer);
                    inflater = new GZipStream(compressedData, CompressionMode.Decompress);
                    buffer = inflater.ReadStream();
                }
                finally
                {
                    if ((object)inflater != null)
                        inflater.Close();

                    if ((object)compressedData != null)
                        compressedData.Close();
                }
            }

            if (useCommonSerializationFormat)
            {
                deserializedCache = new SignalIndexCache();
                deserializedCache.Encoding = m_encoding;
                deserializedCache.ParseBinaryImage(buffer, 0, buffer.Length);
            }
            else
            {
                deserializedCache = Serialization.Deserialize<SignalIndexCache>(buffer, GSF.SerializationFormat.Binary);
            }

            return deserializedCache;
        }
Example #15
0
        /// <summary>
        /// Creates a new <see cref="CompactMeasurement"/> from an existing <see cref="IMeasurement"/> value.
        /// </summary>
        /// <param name="measurement">Source <see cref="IMeasurement"/> value.</param>
        /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
        /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
        /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
        /// <param name="timeIndex">Time index to use for base offset.</param>
        /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
        public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache, bool includeTime, long[] baseTimeOffsets, int timeIndex, bool useMillisecondResolution)
        {
            ID = measurement.ID;
            Key = measurement.Key;
            Value = measurement.Value;
            Adder = measurement.Adder;
            Multiplier = measurement.Multiplier;
            Timestamp = measurement.Timestamp;
            StateFlags = measurement.StateFlags;

            m_signalIndexCache = signalIndexCache;
            m_includeTime = includeTime;

            // We keep a clone of the base time offsets, if provided, since array contents can change at any time
            if (baseTimeOffsets == null)
                m_baseTimeOffsets = s_emptyBaseTimeOffsets;
            else
                m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] };

            m_timeIndex = timeIndex;
            m_useMillisecondResolution = useMillisecondResolution;
        }
Example #16
0
        // Retrieves the measurements from the database.
        private void GetDbMeasurements(object state)
        {
            IDbConnection connection = null;

            // Get measurements from the database.
            try
            {
                SignalIndexCache signalIndexCache = new SignalIndexCache();
                CompactMeasurement measurement;
                long startTime = DateTime.UtcNow.Ticks;

                if (m_cacheFileName != null && File.Exists(m_cacheFileName))
                {
                    OnStatusMessage("Loading cached input data...");

                    try
                    {
                        using (FileStream data = File.OpenRead(m_cacheFileName))
                        {
                            byte[] buffer = new byte[4];
                            int signalIndexCacheImageSize;
                            int compactMeasurementSize;
                            int totalMeasurements;

                            // Read the signal index cache image size from the file
                            if (data.Read(buffer, 0, 4) != 4)
                                throw new EndOfStreamException();

                            signalIndexCacheImageSize = LittleEndian.ToInt32(buffer, 0);

                            // Resize buffer to accomodate exact signal index cache
                            buffer = new byte[signalIndexCacheImageSize];

                            // Read the signal index cache image from the file
                            if (data.Read(buffer, 0, signalIndexCacheImageSize) != signalIndexCacheImageSize)
                                throw new EndOfStreamException();

                            // Deserialize the signal index cache
                            signalIndexCache = Serialization.Deserialize<SignalIndexCache>(buffer, SerializationFormat.Binary);

                            // Read the size of each compact measurement from the file
                            if (data.Read(buffer, 0, 4) != 4)
                                throw new EndOfStreamException();

                            compactMeasurementSize = LittleEndian.ToInt32(buffer, 0);

                            // Read the total number of compact measurements from the file
                            if (data.Read(buffer, 0, 4) != 4)
                                throw new EndOfStreamException();

                            totalMeasurements = LittleEndian.ToInt32(buffer, 0);

                            // Resize buffer to accomodate compact measurement if needed (not likely)
                            if (buffer.Length < compactMeasurementSize)
                                buffer = new byte[compactMeasurementSize];

                            // Read each compact measurement image from the file
                            for (int i = 0; i < totalMeasurements; i++)
                            {
                                if (data.Read(buffer, 0, compactMeasurementSize) != compactMeasurementSize)
                                    throw new EndOfStreamException();

                                // Parse compact measurement
                                measurement = new CompactMeasurement(signalIndexCache);
                                measurement.ParseBinaryImage(buffer, 0, compactMeasurementSize);

                                m_dbMeasurements.Add(measurement);

                                if (m_dbMeasurements.Count % 50000 == 0)
                                    OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
                            }

                            OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2));
                        }
                    }
                    catch (Exception ex)
                    {
                        if (ex is EndOfStreamException)
                            throw (EndOfStreamException)ex;

                        throw new EndOfStreamException(ex.Message, ex);
                    }
                }
                else
                {
                    OnStatusMessage("Loading database input data...");

                    const string MeasurementTable = "ActiveMeasurements";

                    Dictionary<string, string> dataProviderSettings = m_dataProviderString.ParseKeyValuePairs();
                    Assembly assm = Assembly.Load(dataProviderSettings["AssemblyName"]);
                    Type connectionType = assm.GetType(dataProviderSettings["ConnectionType"]);

                    Dictionary<Guid, MeasurementKey> lookupCache = new Dictionary<Guid, MeasurementKey>();
                    IDbCommand command;
                    IDataReader dbReader;
                    MeasurementKey key;
                    Guid id;
                    ushort index = 0;

                    connection = (IDbConnection)Activator.CreateInstance(connectionType);
                    connection.ConnectionString = m_dbConnectionString;
                    connection.Open();

                    command = connection.CreateCommand();
                    command.CommandText = string.Format("SELECT * FROM {0}", m_dbTableName);

                    using (dbReader = command.ExecuteReader())
                    {
                        while (dbReader.Read())
                        {
                            measurement = new CompactMeasurement(signalIndexCache);

                            foreach (string fieldName in m_fieldNames.Keys)
                            {
                                object value = dbReader[fieldName];
                                string propertyName = m_fieldNames[fieldName];

                                switch (propertyName)
                                {
                                    case "Timestamp":
                                        // If the value is a timestamp, use the timestamp format
                                        // specified by the user when reading the timestamp.
                                        if (m_timestampFormat == null)
                                            measurement.Timestamp = long.Parse(value.ToNonNullString());
                                        else
                                            measurement.Timestamp = DateTime.ParseExact(value.ToNonNullString(), m_timestampFormat, CultureInfo.CurrentCulture);
                                        break;
                                    case "ID":
                                        if (Guid.TryParse(value.ToString(), out id))
                                        {
                                            if (!lookupCache.TryGetValue(id, out key))
                                            {
                                                if (DataSource.Tables.Contains(MeasurementTable))
                                                {
                                                    DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("SignalID = '{0}'", id));

                                                    if (filteredRows.Length > 0)
                                                        key = MeasurementKey.LookUpOrCreate(id, filteredRows[0]["ID"].ToString());
                                                }

                                                if (key != MeasurementKey.Undefined)
                                                {
                                                    // Cache measurement key associated with ID
                                                    lookupCache[id] = key;

                                                    // Assign a runtime index optimization for distinct measurements
                                                    signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(id, key.Source, key.ID));
                                                }
                                            }

                                            measurement.Key = key;
                                        }
                                        break;
                                    case "Key":
                                        if (MeasurementKey.TryParse(value.ToString(), out key))
                                        {
                                            if (!lookupCache.ContainsKey(key.SignalID))
                                            {
                                                // Cache measurement key associated with ID
                                                lookupCache[key.SignalID] = key;

                                                // Assign a runtime index optimization for distinct measurements
                                                signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(key.SignalID, key.Source, key.ID));
                                            }

                                            measurement.Key = key;
                                        }
                                        break;
                                    case "Value":
                                        measurement.Value = Convert.ToDouble(value);
                                        break;
                                    default:
                                        PropertyInfo property = GetAllProperties(typeof(IMeasurement)).FirstOrDefault(propertyInfo => propertyInfo.Name == propertyName);

                                        if (property != null)
                                        {
                                            Type propertyType = property.PropertyType;
                                            Type valueType = value.GetType();

                                            if (property.PropertyType.IsAssignableFrom(value.GetType()))
                                            {
                                                property.SetValue(measurement, value, null);
                                            }
                                            else if (property.PropertyType == typeof(string))
                                            {
                                                property.SetValue(measurement, value.ToNonNullString(), null);
                                            }
                                            else if (valueType == typeof(string))
                                            {
                                                MethodInfo parseMethod = propertyType.GetMethod("Parse", new[] { typeof(string) });

                                                if (parseMethod != null && parseMethod.IsStatic)
                                                    property.SetValue(measurement, parseMethod.Invoke(null, new[] { value }), null);
                                            }
                                            else
                                            {
                                                string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1}.", fieldName, propertyName);
                                                OnProcessException(new InvalidCastException(exceptionMessage));
                                            }
                                        }
                                        else
                                        {
                                            string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1} - no property match was found.", fieldName, propertyName);
                                            OnProcessException(new InvalidCastException(exceptionMessage));
                                        }
                                        break;
                                }

                                m_dbMeasurements.Add(measurement);

                                if (m_dbMeasurements.Count % 50000 == 0)
                                    OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count);
                            }
                        }
                    }

                    OnStatusMessage("Sorting data by time...");

                    m_dbMeasurements = m_dbMeasurements.OrderBy(m => (long)m.Timestamp).ToList();

                    OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(2));

                    if (m_cacheFileName != null)
                    {
                        OnStatusMessage("Caching data for next initialization...");

                        using (FileStream data = File.OpenWrite(m_cacheFileName))
                        {
                            byte[] signalIndexCacheImage = Serialization.Serialize(signalIndexCache, SerializationFormat.Binary);
                            int compactMeasurementSize = (new CompactMeasurement(signalIndexCache)).BinaryLength;

                            // Write the signal index cache image size to the file
                            data.Write(LittleEndian.GetBytes(signalIndexCacheImage.Length), 0, 4);

                            // Write the signal index cache image to the file
                            data.Write(signalIndexCacheImage, 0, signalIndexCacheImage.Length);

                            // Write the size of each compact measurement to the file
                            data.Write(LittleEndian.GetBytes(compactMeasurementSize), 0, 4);

                            // Write the total number of compact measurements to the file
                            data.Write(LittleEndian.GetBytes(m_dbMeasurements.Count), 0, 4);

                            // Write each compact measurement image to the file
                            for (int i = 0; i < m_dbMeasurements.Count; i++)
                            {
                                ((ISupportBinaryImage)m_dbMeasurements[i]).CopyBinaryImageToStream(data);
                            }
                        }
                    }
                }

                OnStatusMessage("Entering data read cycle...");
                ThreadPool.QueueUserWorkItem(PublishData);
            }
            catch (EndOfStreamException ex)
            {
                OnProcessException(new EndOfStreamException(string.Format("Failed load cached data from {0} due to file corruption{1} cache will be recreated from database", m_cacheFileName, string.IsNullOrWhiteSpace(ex.Message) ? "," : ": " + ex.Message + " - ")));

                // If the cached file is corrupt, delete it and load from the database
                if (File.Exists(m_cacheFileName))
                    File.Delete(m_cacheFileName);

                m_dbMeasurements.Clear();
                GetDbMeasurements(null);
            }
            catch (Exception ex)
            {
                OnProcessException(new InvalidOperationException("Failed during data load: " + ex.Message, ex));
            }
            finally
            {

                if (connection != null)
                    connection.Close();
            }
        }
Example #17
0
 public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache)
     : this(measurement, signalIndexCache, true, null, 0, false)
 {
 }
Example #18
0
        /// <summary>
        /// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.
        /// </summary>
        /// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param>
        /// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param>
        /// <param name="index">Index into buffer where compressed payload begins.</param>
        /// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param>
        /// <param name="measurementCount">Number of compressed measurements in the payload.</param>
        /// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param>
        /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param>
        /// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns>
        public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags)
        {
            CompactMeasurement[] measurements = new CompactMeasurement[measurementCount];

            // Actual data length has to take into account response byte and in-response-to server command byte in the payload header
            //int dataLength = length - index - 2;
            int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength);

            // Copy source data into a decompression buffer
            byte[] buffer = new byte[bufferLength];
            Buffer.BlockCopy(source, index, buffer, 0, dataLength);

            // Check that OS endian-order matches endian-order of compressed data
            if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0))
            {
                // TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it
                // can be modified to decompress a payload that is in a non-native Endian order
                throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order.");
            }

            // Attempt to decompress buffer
            int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength);

            if (uncompressedSize == 0)
                throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption.");

            index = 0;

            // Decode ID and state flags
            for (int i = 0; i < measurementCount; i++)
            {
                uint value = NativeEndianOrder.Default.ToUInt32(buffer, index);

                measurements[i] = new CompactMeasurement(signalIndexCache, includeTime)
                {
                    CompactStateFlags = (byte)(value >> 16),
                    RuntimeID = (ushort)value
                };

                index += 4;
            }

            // Decode values
            for (int i = 0; i < measurementCount; i++)
            {
                measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index);
                index += 4;
            }

            if (includeTime)
            {
                // Decode timestamps
                for (int i = 0; i < measurementCount; i++)
                {
                    measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index);
                    index += 8;
                }
            }

            return measurements;
        }
Example #19
0
        /// <summary>
        /// Creates a new <see cref="CompactMeasurement"/>.
        /// </summary>
        /// <param name="signalIndexCache">Signal index cache used to serialize or deserialize runtime information.</param>
        /// <param name="includeTime">Set to <c>true</c> to include time in serialized packet; otherwise <c>false</c>.</param>
        /// <param name="baseTimeOffsets">Base time offset array - set to <c>null</c> to use full fidelity measurement time.</param>
        /// <param name="timeIndex">Time index to use for base offset.</param>
        /// <param name="useMillisecondResolution">Flag that determines if millisecond resolution is in use for this serialization.</param>
        public CompactMeasurement(SignalIndexCache signalIndexCache, bool includeTime, long[] baseTimeOffsets, int timeIndex, bool useMillisecondResolution)
        {
            m_signalIndexCache = signalIndexCache;
            m_includeTime = includeTime;

            // We keep a clone of the base time offsets, if provided, since array contents can change at any time
            if (baseTimeOffsets == null)
                m_baseTimeOffsets = s_emptyBaseTimeOffsets;
            else
                m_baseTimeOffsets = new long[] { baseTimeOffsets[0], baseTimeOffsets[1] };

            m_timeIndex = timeIndex;
            m_useMillisecondResolution = useMillisecondResolution;
        }
Example #20
0
        /// <summary>
        /// Handles meta-data synchronization to local system.
        /// </summary>
        /// <remarks>
        /// This function should only be initiated from call to <see cref="SynchronizeMetadata(DataSet)"/> to make
        /// sure only one meta-data synchronization happens at once. Users can override this method to customize
        /// process of meta-data synchronization.
        /// </remarks>
        protected virtual void SynchronizeMetadata()
        {
            bool dataMonitoringEnabled = false;

            // TODO: This function is complex and very closely tied to the current time-series data schema - perhaps it should be moved outside this class and referenced
            // TODO: as a delegate that can be assigned and called to allow other schemas as well. DataPublisher is already very flexible in what data it can deliver.
            try
            {
                DataSet metadata = m_receivedMetadata;

                // Only perform database synchronization if meta-data has changed since last update
                if (!SynchronizedMetadataChanged(metadata))
                    return;

                if ((object)metadata == null)
                {
                    OnStatusMessage("WARNING: Meta-data synchronization was not performed, deserialized dataset was empty.");
                    return;
                }

                // Reset data stream monitor while meta-data synchronization is in progress
                if ((object)m_dataStreamMonitor != null && m_dataStreamMonitor.Enabled)
                {
                    m_dataStreamMonitor.Enabled = false;
                    dataMonitoringEnabled = true;
                }

                // Track total meta-data synchronization process time
                Ticks startTime = DateTime.UtcNow.Ticks;
                DateTime updateTime;
                DateTime latestUpdateTime = DateTime.MinValue;

                // Open the configuration database using settings found in the config file
                using (AdoDataConnection database = new AdoDataConnection("systemSettings"))
                using (IDbCommand command = database.Connection.CreateCommand())
                {
                    IDbTransaction transaction = null;

                    if (m_useTransactionForMetadata)
                        transaction = database.Connection.BeginTransaction(database.DefaultIsloationLevel);

                    try
                    {
                        if ((object)transaction != null)
                            command.Transaction = transaction;

                        // Query the actual record ID based on the known run-time ID for this subscriber device
                        int parentID = Convert.ToInt32(command.ExecuteScalar($"SELECT SourceID FROM Runtime WHERE ID = {ID} AND SourceTable='Device'", m_metadataSynchronizationTimeout));

                        // Validate that the subscriber device is marked as a concentrator (we are about to associate children devices with it)
                        if (!command.ExecuteScalar($"SELECT IsConcentrator FROM Device WHERE ID = {parentID}", m_metadataSynchronizationTimeout).ToString().ParseBoolean())
                            command.ExecuteNonQuery($"UPDATE Device SET IsConcentrator = 1 WHERE ID = {parentID}", m_metadataSynchronizationTimeout);

                        // Get any historian associated with the subscriber device
                        object historianID = command.ExecuteScalar($"SELECT HistorianID FROM Device WHERE ID = {parentID}", m_metadataSynchronizationTimeout);

                        // Determine the active node ID - we cache this since this value won't change for the lifetime of this class
                        if (m_nodeID == Guid.Empty)
                            m_nodeID = Guid.Parse(command.ExecuteScalar($"SELECT NodeID FROM IaonInputAdapter WHERE ID = {(int)ID}", m_metadataSynchronizationTimeout).ToString());

                        // Determine the protocol record auto-inc ID value for the gateway transport protocol (GEP) - this value is also cached since it shouldn't change for the lifetime of this class
                        if (m_gatewayProtocolID == 0)
                            m_gatewayProtocolID = int.Parse(command.ExecuteScalar("SELECT ID FROM Protocol WHERE Acronym='GatewayTransport'", m_metadataSynchronizationTimeout).ToString());

                        // Ascertain total number of actions required for all meta-data synchronization so some level feed back can be provided on progress
                        InitSyncProgress(metadata.Tables.Cast<DataTable>().Select(dataTable => (long)dataTable.Rows.Count).Sum() + 3);

                        // Prefix all children devices with the name of the parent since the same device names could appear in different connections (helps keep device names unique)
                        string sourcePrefix = Name + "!";
                        Dictionary<string, int> deviceIDs = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
                        string deviceAcronym, signalTypeAcronym;
                        decimal longitude, latitude;
                        decimal? location;
                        object originalSource;
                        int deviceID;

                        // Check to see if data for the "DeviceDetail" table was included in the meta-data
                        if (metadata.Tables.Contains("DeviceDetail"))
                        {
                            DataTable deviceDetail = metadata.Tables["DeviceDetail"];
                            List<Guid> uniqueIDs = new List<Guid>();
                            DataRow[] deviceRows;

                            // Define SQL statement to query if this device is already defined (this should always be based on the unique guid-based device ID)
                            string deviceExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0}", "uniqueID");

                            // Define SQL statement to insert new device record
                            string insertDeviceSql = database.ParameterizedQueryString("INSERT INTO Device(NodeID, ParentID, HistorianID, Acronym, Name, ProtocolID, FramesPerSecond, OriginalSource, AccessID, Longitude, Latitude, ContactList, IsConcentrator, Enabled) " +
                                                                                       "VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}, 0, 1)", "nodeID", "parentID", "historianID", "acronym", "name", "protocolID", "framesPerSecond", "originalSource", "accessID", "longitude", "latitude", "contactList");

                            // Define SQL statement to update device's guid-based unique ID after insert
                            string updateDeviceUniqueIDSql = database.ParameterizedQueryString("UPDATE Device SET UniqueID = {0} WHERE Acronym = {1}", "uniqueID", "acronym");

                            // Define SQL statement to query if a device can be safely updated
                            string deviceIsUpdatableSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Device WHERE UniqueID = {0} AND (ParentID <> {1} OR ParentID IS NULL)", "uniqueID", "parentID");

                            // Define SQL statement to update existing device record
                            string updateDeviceSql = database.ParameterizedQueryString("UPDATE Device SET Acronym = {0}, Name = {1}, OriginalSource = {2}, ProtocolID = {3}, FramesPerSecond = {4}, HistorianID = {5}, AccessID = {6}, Longitude = {7}, Latitude = {8}, ContactList = {9} WHERE UniqueID = {10}",
                                                                                       "acronym", "name", "originalSource", "protocolID", "framesPerSecond", "historianID", "accessID", "longitude", "latitude", "contactList", "uniqueID");

                            // Define SQL statement to retrieve device's auto-inc ID based on its unique guid-based ID
                            string queryDeviceIDSql = database.ParameterizedQueryString("SELECT ID FROM Device WHERE UniqueID = {0}", "uniqueID");

                            // Define SQL statement to retrieve all unique device ID's for the current parent to check for mismatches
                            string queryUniqueDeviceIDsSql = database.ParameterizedQueryString("SELECT UniqueID FROM Device WHERE ParentID = {0}", "parentID");

                            // Define SQL statement to remove device records that no longer exist in the meta-data
                            string deleteDeviceSql = database.ParameterizedQueryString("DELETE FROM Device WHERE UniqueID = {0}", "uniqueID");

                            // Determine which device rows should be synchronized based on operational mode flags
                            if (ReceiveInternalMetadata && ReceiveExternalMetadata)
                                deviceRows = deviceDetail.Select();
                            else if (ReceiveInternalMetadata)
                                deviceRows = deviceDetail.Select("OriginalSource IS NULL");
                            else if (ReceiveExternalMetadata)
                                deviceRows = deviceDetail.Select("OriginalSource IS NOT NULL");
                            else
                                deviceRows = new DataRow[0];

                            // Check existence of optional meta-data fields
                            DataColumnCollection deviceDetailColumns = deviceDetail.Columns;
                            bool accessIDFieldExists = deviceDetailColumns.Contains("AccessID");
                            bool longitudeFieldExists = deviceDetailColumns.Contains("Longitude");
                            bool latitudeFieldExists = deviceDetailColumns.Contains("Latitude");
                            bool companyAcronymFieldExists = deviceDetailColumns.Contains("CompanyAcronym");
                            bool protocolNameFieldExists = deviceDetailColumns.Contains("ProtocolName");
                            bool vendorAcronymFieldExists = deviceDetailColumns.Contains("VendorAcronym");
                            bool vendorDeviceNameFieldExists = deviceDetailColumns.Contains("VendorDeviceName");
                            bool interconnectionNameFieldExists = deviceDetailColumns.Contains("InterconnectionName");
                            bool updatedOnFieldExists = deviceDetailColumns.Contains("UpdatedOn");

                            // Older versions of GEP did not include the AccessID field, so this is treated as optional
                            int accessID = 0;

                            foreach (DataRow row in deviceRows)
                            {
                                Guid uniqueID = Guid.Parse(row.Field<object>("UniqueID").ToString());
                                bool recordNeedsUpdating;

                                // Track unique device Guids in this meta-data session, we'll need to remove any old associated devices that no longer exist
                                uniqueIDs.Add(uniqueID);

                                // Determine if record has changed since last synchronization
                                if (updatedOnFieldExists)
                                {
                                    try
                                    {
                                        updateTime = Convert.ToDateTime(row["UpdatedOn"]);
                                        recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime;

                                        if (updateTime > latestUpdateTime)
                                            latestUpdateTime = updateTime;
                                    }
                                    catch
                                    {
                                        recordNeedsUpdating = true;
                                    }
                                }
                                else
                                {
                                    recordNeedsUpdating = true;
                                }

                                // We will synchronize meta-data only if the source owns this device and it's not defined as a concentrator (these should normally be filtered by publisher - but we check just in case).
                                if (!row["IsConcentrator"].ToNonNullString("0").ParseBoolean())
                                {
                                    if (accessIDFieldExists)
                                        accessID = row.ConvertField<int>("AccessID");

                                    // Get longitude and latitude values if they are defined
                                    longitude = 0M;
                                    latitude = 0M;

                                    if (longitudeFieldExists)
                                    {
                                        location = row.ConvertNullableField<decimal>("Longitude");

                                        if (location.HasValue)
                                            longitude = location.Value;
                                    }

                                    if (latitudeFieldExists)
                                    {
                                        location = row.ConvertNullableField<decimal>("Latitude");

                                        if (location.HasValue)
                                            latitude = location.Value;
                                    }

                                    // Save any reported extraneous values from device meta-data in connection string formatted contact list - all fields are considered optional
                                    Dictionary<string, string> contactList = new Dictionary<string, string>();

                                    if (companyAcronymFieldExists)
                                        contactList["companyAcronym"] = row.Field<string>("CompanyAcronym") ?? string.Empty;

                                    if (protocolNameFieldExists)
                                        contactList["protocolName"] = row.Field<string>("ProtocolName") ?? string.Empty;

                                    if (vendorAcronymFieldExists)
                                        contactList["vendorAcronym"] = row.Field<string>("VendorAcronym") ?? string.Empty;

                                    if (vendorDeviceNameFieldExists)
                                        contactList["vendorDeviceName"] = row.Field<string>("VendorDeviceName") ?? string.Empty;

                                    if (interconnectionNameFieldExists)
                                        contactList["interconnectionName"] = row.Field<string>("InterconnectionName") ?? string.Empty;

                                    // Determine if device record already exists
                                    if (Convert.ToInt32(command.ExecuteScalar(deviceExistsSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID))) == 0)
                                    {
                                        // Insert new device record
                                        command.ExecuteNonQuery(insertDeviceSql, m_metadataSynchronizationTimeout, database.Guid(m_nodeID), parentID, historianID, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), m_gatewayProtocolID, row.ConvertField<int>("FramesPerSecond"),
                                                                m_internal ? (object)DBNull.Value : string.IsNullOrEmpty(row.Field<string>("ParentAcronym")) ? sourcePrefix + row.Field<string>("Acronym") : sourcePrefix + row.Field<string>("ParentAcronym"), accessID, longitude, latitude, contactList.JoinKeyValuePairs());

                                        // Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database
                                        // scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned.
                                        command.ExecuteNonQuery(updateDeviceUniqueIDSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID), sourcePrefix + row.Field<string>("Acronym"));
                                    }
                                    else if (recordNeedsUpdating)
                                    {
                                        // Perform safety check to preserve device records which are not safe to overwrite
                                        if (Convert.ToInt32(command.ExecuteScalar(deviceIsUpdatableSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID), parentID)) > 0)
                                            continue;

                                        // Gateway is assuming ownership of the device records when the "internal" flag is true - this means the device's measurements can be forwarded to another party. From a device record perspective,
                                        // ownership is inferred by setting 'OriginalSource' to null. When gateway doesn't own device records (i.e., the "internal" flag is false), this means the device's measurements can only be consumed
                                        // locally - from a device record perspective this means the 'OriginalSource' field is set to the acronym of the PDC or PMU that generated the source measurements. This field allows a mirrored source
                                        // restriction to be implemented later to ensure all devices in an output protocol came from the same original source connection, if desired.
                                        originalSource = m_internal ? (object)DBNull.Value : string.IsNullOrEmpty(row.Field<string>("ParentAcronym")) ? sourcePrefix + row.Field<string>("Acronym") : sourcePrefix + row.Field<string>("ParentAcronym");

                                        // Update existing device record
                                        command.ExecuteNonQuery(updateDeviceSql, m_metadataSynchronizationTimeout, sourcePrefix + row.Field<string>("Acronym"), row.Field<string>("Name"), originalSource, m_gatewayProtocolID, row.ConvertField<int>("FramesPerSecond"), historianID, accessID, longitude, latitude, contactList.JoinKeyValuePairs(), database.Guid(uniqueID));
                                    }
                                }

                                // Capture local device ID auto-inc value for measurement association
                                deviceIDs[row.Field<string>("Acronym")] = Convert.ToInt32(command.ExecuteScalar(queryDeviceIDSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID)));

                                // Periodically notify user about synchronization progress
                                UpdateSyncProgress();
                            }

                            // Remove any device records associated with this subscriber that no longer exist in the meta-data
                            if (uniqueIDs.Count > 0)
                            {
                                // Sort unique ID list so that binary search can be used for quick lookups
                                uniqueIDs.Sort();

                                DataTable deviceUniqueIDs = command.RetrieveData(database.AdapterType, queryUniqueDeviceIDsSql, m_metadataSynchronizationTimeout, parentID);
                                Guid uniqueID;

                                foreach (DataRow deviceRow in deviceUniqueIDs.Rows)
                                {
                                    uniqueID = database.Guid(deviceRow, "UniqueID");

                                    // Remove any devices in the database that are associated with the parent device and do not exist in the meta-data
                                    if (uniqueIDs.BinarySearch(uniqueID) < 0)
                                        command.ExecuteNonQuery(deleteDeviceSql, m_metadataSynchronizationTimeout, database.Guid(uniqueID));
                                }
                                UpdateSyncProgress();
                            }
                        }

                        // Check to see if data for the "MeasurementDetail" table was included in the meta-data
                        if (metadata.Tables.Contains("MeasurementDetail"))
                        {
                            DataTable measurementDetail = metadata.Tables["MeasurementDetail"];
                            List<Guid> signalIDs = new List<Guid>();
                            DataRow[] measurementRows;

                            // Define SQL statement to query if this measurement is already defined (this should always be based on the unique signal ID Guid)
                            string measurementExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Measurement WHERE SignalID = {0}", "signalID");

                            // Define SQL statement to insert new measurement record
                            string insertMeasurementSql = database.ParameterizedQueryString("INSERT INTO Measurement(DeviceID, HistorianID, PointTag, AlternateTag, SignalTypeID, PhasorSourceIndex, SignalReference, Description, Internal, Subscribed, Enabled) " +
                                                                                            "VALUES ({0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, 0, 1)", "deviceID", "historianID", "pointTag", "alternateTag", "signalTypeID", "phasorSourceIndex", "signalReference", "description", "internal");

                            // Define SQL statement to update measurement's signal ID after insert
                            string updateMeasurementSignalIDSql = database.ParameterizedQueryString("UPDATE Measurement SET SignalID = {0}, AlternateTag = NULL WHERE AlternateTag = {1}", "signalID", "alternateTag");

                            // Define SQL statement to update existing measurement record
                            string updateMeasurementSql = database.ParameterizedQueryString("UPDATE Measurement SET HistorianID = {0}, PointTag = {1}, SignalTypeID = {2}, PhasorSourceIndex = {3}, SignalReference = {4}, Description = {5}, Internal = {6} WHERE SignalID = {7}",
                                                                                            "historianID", "pointTag", "signalTypeID", "phasorSourceIndex", "signalReference", "description", "internal", "signalID");

                            // Define SQL statement to retrieve all measurement signal ID's for the current parent to check for mismatches - note that we use the ActiveMeasurements view
                            // since it associates measurements with their top-most parent runtime device ID, this allows us to easily query all measurements for the parent device
                            string queryMeasurementSignalIDsSql = database.ParameterizedQueryString("SELECT SignalID FROM ActiveMeasurement WHERE DeviceID = {0}", "deviceID");

                            // Define SQL statement to retrieve measurement's associated device ID, i.e., actual record ID, based on measurement's signal ID
                            string queryMeasurementDeviceIDSql = database.ParameterizedQueryString("SELECT DeviceID FROM Measurement WHERE SignalID = {0}", "signalID");

                            // Define SQL statement to remove device records that no longer exist in the meta-data
                            string deleteMeasurementSql = database.ParameterizedQueryString("DELETE FROM Measurement WHERE SignalID = {0}", "signalID");

                            // Load signal type ID's from local database associated with their acronym for proper signal type translation
                            Dictionary<string, int> signalTypeIDs = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);

                            foreach (DataRow row in command.RetrieveData(database.AdapterType, "SELECT ID, Acronym FROM SignalType").Rows)
                            {
                                signalTypeAcronym = row.Field<string>("Acronym");

                                if (!string.IsNullOrWhiteSpace(signalTypeAcronym))
                                    signalTypeIDs[signalTypeAcronym] = row.ConvertField<int>("ID");
                            }

                            // Determine which measurement rows should be synchronized based on operational mode flags
                            if (ReceiveInternalMetadata && ReceiveExternalMetadata)
                                measurementRows = measurementDetail.Select();
                            else if (ReceiveInternalMetadata)
                                measurementRows = measurementDetail.Select("Internal <> 0");
                            else if (ReceiveExternalMetadata)
                                measurementRows = measurementDetail.Select("Internal = 0");
                            else
                                measurementRows = new DataRow[0];

                            // Check existence of optional meta-data fields
                            DataColumnCollection measurementDetailColumns = measurementDetail.Columns;
                            bool phasorSourceIndexFieldExists = measurementDetailColumns.Contains("PhasorSourceIndex");
                            bool updatedOnFieldExists = measurementDetailColumns.Contains("UpdatedOn");

                            object phasorSourceIndex = DBNull.Value;

                            foreach (DataRow row in measurementRows)
                            {
                                bool recordNeedsUpdating;

                                // Determine if record has changed since last synchronization
                                if (updatedOnFieldExists)
                                {
                                    try
                                    {
                                        updateTime = Convert.ToDateTime(row["UpdatedOn"]);
                                        recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime;

                                        if (updateTime > latestUpdateTime)
                                            latestUpdateTime = updateTime;
                                    }
                                    catch
                                    {
                                        recordNeedsUpdating = true;
                                    }
                                }
                                else
                                {
                                    recordNeedsUpdating = true;
                                }

                                // Get device and signal type acronyms
                                deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty;
                                signalTypeAcronym = row.Field<string>("SignalAcronym") ?? string.Empty;

                                // Get phasor source index if field is defined
                                if (phasorSourceIndexFieldExists)
                                {
                                    // Using ConvertNullableField extension since publisher could use SQLite database in which case
                                    // all integers would arrive in data set as longs and need to be converted back to integers
                                    int? index = row.ConvertNullableField<int>("PhasorSourceIndex");
                                    phasorSourceIndex = index.HasValue ? (object)index.Value : (object)DBNull.Value;
                                }

                                // Make sure we have an associated device and signal type already defined for the measurement
                                if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym) && !string.IsNullOrWhiteSpace(signalTypeAcronym) && signalTypeIDs.ContainsKey(signalTypeAcronym))
                                {
                                    Guid signalID = Guid.Parse(row.Field<object>("SignalID").ToString());

                                    // Track unique measurement signal Guids in this meta-data session, we'll need to remove any old associated measurements that no longer exist
                                    signalIDs.Add(signalID);


                                    // Prefix the tag name with the "updated" device name
                                    string pointTag = sourcePrefix + row.Field<string>("PointTag");

                                    // Look up associated device ID (local DB auto-inc)
                                    deviceID = deviceIDs[deviceAcronym];

                                    // Determine if measurement record already exists
                                    if (Convert.ToInt32(command.ExecuteScalar(measurementExistsSql, m_metadataSynchronizationTimeout, database.Guid(signalID))) == 0)
                                    {
                                        string alternateTag = Guid.NewGuid().ToString();

                                        // Insert new measurement record
                                        command.ExecuteNonQuery(insertMeasurementSql, m_metadataSynchronizationTimeout, deviceID, historianID, pointTag, alternateTag, signalTypeIDs[signalTypeAcronym], phasorSourceIndex, sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal));

                                        // Guids are normally auto-generated during insert - after insertion update the Guid so that it matches the source data. Most of the database
                                        // scripts have triggers that support properly assigning the Guid during an insert, but this code ensures the Guid will always get assigned.
                                        command.ExecuteNonQuery(updateMeasurementSignalIDSql, m_metadataSynchronizationTimeout, database.Guid(signalID), alternateTag);
                                    }
                                    else if (recordNeedsUpdating)
                                    {
                                        // Update existing measurement record. Note that this update assumes that measurements will remain associated with a static source device.
                                        command.ExecuteNonQuery(updateMeasurementSql, m_metadataSynchronizationTimeout, historianID, pointTag, signalTypeIDs[signalTypeAcronym], phasorSourceIndex, sourcePrefix + row.Field<string>("SignalReference"), row.Field<string>("Description") ?? string.Empty, database.Bool(m_internal), database.Guid(signalID));
                                    }
                                }

                                // Periodically notify user about synchronization progress
                                UpdateSyncProgress();
                            }

                            // Remove any measurement records associated with existing devices in this session but no longer exist in the meta-data
                            if (signalIDs.Count > 0)
                            {
                                // Sort signal ID list so that binary search can be used for quick lookups
                                signalIDs.Sort();

                                // Query all the guid-based signal ID's for all measurement records associated with the parent device using run-time ID
                                DataTable measurementSignalIDs = command.RetrieveData(database.AdapterType, queryMeasurementSignalIDsSql, m_metadataSynchronizationTimeout, (int)ID);
                                Guid signalID;

                                // Walk through each database record and see if the measurement exists in the provided meta-data
                                foreach (DataRow measurementRow in measurementSignalIDs.Rows)
                                {
                                    signalID = database.Guid(measurementRow, "SignalID");

                                    // Remove any measurements in the database that are associated with received devices and do not exist in the meta-data
                                    if (signalIDs.BinarySearch(signalID) < 0)
                                    {
                                        // Measurement was not in the meta-data, get the measurement's actual record based ID for its associated device
                                        object measurementDeviceID = command.ExecuteScalar(queryMeasurementDeviceIDSql, m_metadataSynchronizationTimeout, database.Guid(signalID));

                                        // If the unknown measurement is directly associated with a device that exists in the meta-data it is assumed that this measurement
                                        // was removed from the publishing system and no longer exists therefore we remove it from the local measurement cache. If the user
                                        // needs custom local measurements associated with a remote device, they should be associated with the parent device only.
                                        if (measurementDeviceID != null && !(measurementDeviceID is DBNull) && deviceIDs.ContainsValue(Convert.ToInt32(measurementDeviceID)))
                                            command.ExecuteNonQuery(deleteMeasurementSql, m_metadataSynchronizationTimeout, database.Guid(signalID));
                                    }
                                }

                                UpdateSyncProgress();
                            }
                        }

                        // Check to see if data for the "PhasorDetail" table was included in the meta-data
                        if (metadata.Tables.Contains("PhasorDetail"))
                        {
                            Dictionary<int, int> maxSourceIndicies = new Dictionary<int, int>();
                            int sourceIndex;

                            // Phasor data is normally only needed so that the user can properly generate a mirrored IEEE C37.118 output stream from the source data.
                            // This is necessary since, in this protocol, the phasors are described (i.e., labeled) as a unit (i.e., as a complex number) instead of
                            // as two distinct angle and magnitude measurements.

                            // Define SQL statement to query if phasor record is already defined (no Guid is defined for these simple label records)
                            string phasorExistsSql = database.ParameterizedQueryString("SELECT COUNT(*) FROM Phasor WHERE DeviceID = {0} AND SourceIndex = {1}", "deviceID", "sourceIndex");

                            // Define SQL statement to insert new phasor record
                            string insertPhasorSql = database.ParameterizedQueryString("INSERT INTO Phasor(DeviceID, Label, Type, Phase, SourceIndex) VALUES ({0}, {1}, {2}, {3}, {4})", "deviceID", "label", "type", "phase", "sourceIndex");

                            // Define SQL statement to update existing phasor record
                            string updatePhasorSql = database.ParameterizedQueryString("UPDATE Phasor SET Label = {0}, Type = {1}, Phase = {2} WHERE DeviceID = {3} AND SourceIndex = {4}", "label", "type", "phase", "deviceID", "sourceIndex");

                            // Define SQL statement to delete a phasor record
                            string deletePhasorSql = database.ParameterizedQueryString("DELETE FROM Phasor WHERE DeviceID = {0} AND SourceIndex > {1}", "deviceID", "sourceIndex");

                            foreach (DataRow row in metadata.Tables["PhasorDetail"].Rows)
                            {
                                // Get device acronym
                                deviceAcronym = row.Field<string>("DeviceAcronym") ?? string.Empty;

                                // Make sure we have an associated device already defined for the phasor record
                                if (!string.IsNullOrWhiteSpace(deviceAcronym) && deviceIDs.ContainsKey(deviceAcronym))
                                {
                                    bool recordNeedsUpdating;

                                    // Determine if record has changed since last synchronization
                                    try
                                    {
                                        updateTime = Convert.ToDateTime(row["UpdatedOn"]);
                                        recordNeedsUpdating = updateTime > m_lastMetaDataRefreshTime;

                                        if (updateTime > latestUpdateTime)
                                            latestUpdateTime = updateTime;
                                    }
                                    catch
                                    {
                                        recordNeedsUpdating = true;
                                    }

                                    deviceID = deviceIDs[deviceAcronym];

                                    // Determine if phasor record already exists
                                    if (Convert.ToInt32(command.ExecuteScalar(phasorExistsSql, m_metadataSynchronizationTimeout, deviceID, row.ConvertField<int>("SourceIndex"))) == 0)
                                    {
                                        // Insert new phasor record
                                        command.ExecuteNonQuery(insertPhasorSql, m_metadataSynchronizationTimeout, deviceID, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), row.ConvertField<int>("SourceIndex"));
                                    }
                                    else if (recordNeedsUpdating)
                                    {
                                        // Update existing phasor record
                                        command.ExecuteNonQuery(updatePhasorSql, m_metadataSynchronizationTimeout, row.Field<string>("Label") ?? "undefined", (row.Field<string>("Type") ?? "V").TruncateLeft(1), (row.Field<string>("Phase") ?? "+").TruncateLeft(1), deviceID, row.ConvertField<int>("SourceIndex"));
                                    }

                                    // Track largest source index for each device
                                    maxSourceIndicies.TryGetValue(deviceID, out sourceIndex);

                                    if (row.ConvertField<int>("SourceIndex") > sourceIndex)
                                        maxSourceIndicies[deviceID] = row.ConvertField<int>("SourceIndex");
                                }

                                // Periodically notify user about synchronization progress
                                UpdateSyncProgress();
                            }

                            // Remove any phasor records associated with existing devices in this session but no longer exist in the meta-data
                            if (maxSourceIndicies.Count > 0)
                            {
                                foreach (KeyValuePair<int, int> deviceIndexPair in maxSourceIndicies)
                                {
                                    command.ExecuteNonQuery(deletePhasorSql, m_metadataSynchronizationTimeout, deviceIndexPair.Key, deviceIndexPair.Value);
                                }
                            }
                        }

                        if ((object)transaction != null)
                            transaction.Commit();

                        // Update local in-memory synchronized meta-data cache
                        m_synchronizedMetadata = metadata;
                    }
                    catch (Exception ex)
                    {
                        OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex));

                        if ((object)transaction != null)
                        {
                            try
                            {
                                transaction.Rollback();
                            }
                            catch (Exception rollbackException)
                            {
                                OnProcessException(new InvalidOperationException("Failed to roll back database transaction due to exception: " + rollbackException.Message, rollbackException));
                            }
                        }

                        return;
                    }
                    finally
                    {
                        if ((object)transaction != null)
                            transaction.Dispose();
                    }
                }

                // New signals may have been defined, take original remote signal index cache and apply changes
                if (m_remoteSignalIndexCache != null)
                    m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);

                m_lastMetaDataRefreshTime = latestUpdateTime > DateTime.MinValue ? latestUpdateTime : DateTime.UtcNow;

                OnStatusMessage("Meta-data synchronization completed successfully in {0}", (DateTime.UtcNow.Ticks - startTime).ToElapsedTimeString(2));

                // Send notification that system configuration has changed
                OnConfigurationChanged();
            }
            catch (Exception ex)
            {
                OnProcessException(new InvalidOperationException("Failed to synchronize meta-data to local cache: " + ex.Message, ex));
            }
            finally
            {
                // Restart data stream monitor after meta-data synchronization if it was originally enabled
                if (dataMonitoringEnabled && (object)m_dataStreamMonitor != null)
                    m_dataStreamMonitor.Enabled = true;
            }
        }
Example #21
0
 public CompactMeasurement(IMeasurement measurement, SignalIndexCache signalIndexCache)
     : this(measurement, signalIndexCache, true, null, 0 , false)
 {
     
 }
        /// <summary>
        /// Creates a new <see cref="UnsynchronizedClientSubscription"/>.
        /// </summary>
        /// <param name="parent">Reference to parent.</param>
        /// <param name="clientID"><see cref="Guid"/> based client connection ID.</param>
        /// <param name="subscriberID"><see cref="Guid"/> based subscriber ID.</param>
        public UnsynchronizedClientSubscription(DataPublisher parent, Guid clientID, Guid subscriberID)
        {
            // Pass parent reference into base class
            AssignParentCollection(parent);

            m_parent = parent;
            m_clientID = clientID;
            m_subscriberID = subscriberID;

            m_signalIndexCache = new SignalIndexCache()
            {
                SubscriberID = subscriberID
            };

            m_processQueue = new AsyncQueue<IEnumerable<IMeasurement>>()
            {
                ProcessItemFunction = ProcessMeasurements
            };

            m_processQueue.ProcessException += m_processQueue_ProcessException;
        }
Example #23
0
        private void ProcessServerResponse(byte[] buffer, int length)
        {
            // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing
            // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread...
            if (buffer != null && length > 0)
            {
                try
                {
                    Dictionary<Guid, DeviceStatisticsHelper<SubscribedDevice>> subscribedDevicesLookup;
                    DeviceStatisticsHelper<SubscribedDevice> statisticsHelper;

                    ServerResponse responseCode = (ServerResponse)buffer[0];
                    ServerCommand commandCode = (ServerCommand)buffer[1];
                    int responseLength = BigEndian.ToInt32(buffer, 2);
                    int responseIndex = DataPublisher.ClientResponseHeaderSize;
                    bool solicited = false;
                    byte[][][] keyIVs;

                    // See if this was a solicited response to a requested server command
                    if (responseCode.IsSolicited())
                    {
                        lock (m_requests)
                        {
                            int index = m_requests.BinarySearch(commandCode);

                            if (index >= 0)
                            {
                                solicited = true;
                                m_requests.RemoveAt(index);
                            }
                        }

                        // Disconnect any established UDP data channel upon successful unsubscribe
                        if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded)
                            DataChannel = null;
                    }

                    OnReceivedServerResponse(responseCode, commandCode);

                    switch (responseCode)
                    {
                        case ServerResponse.Succeeded:
                            if (solicited)
                            {
                                switch (commandCode)
                                {
                                    case ServerCommand.Authenticate:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_authenticated = true;
                                        OnConnectionAuthenticated();
                                        break;
                                    case ServerCommand.Subscribe:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_subscribed = true;
                                        break;
                                    case ServerCommand.Unsubscribe:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_subscribed = false;
                                        if ((object)m_dataStreamMonitor != null)
                                            m_dataStreamMonitor.Enabled = false;
                                        break;
                                    case ServerCommand.RotateCipherKeys:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    case ServerCommand.MetaDataRefresh:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode);
                                        OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
                                        m_metadataRefreshPending = false;
                                        break;
                                }
                            }
                            else
                            {
                                switch (commandCode)
                                {
                                    case ServerCommand.MetaDataRefresh:
                                        // Meta-data refresh may be unsolicited
                                        OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode);
                                        OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
                                        m_metadataRefreshPending = false;
                                        break;
                                    case ServerCommand.RotateCipherKeys:
                                        // Key rotation may be unsolicited
                                        OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    case ServerCommand.Subscribe:
                                        OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    default:
                                        OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode));
                                        break;
                                }
                            }
                            break;
                        case ServerResponse.Failed:
                            if (solicited)
                                OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                            else
                                OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode));

                            if (commandCode == ServerCommand.MetaDataRefresh)
                                m_metadataRefreshPending = false;
                            break;
                        case ServerResponse.DataPacket:
                            long now = DateTime.UtcNow.Ticks;

                            // Deserialize data packet
                            List<IMeasurement> measurements = new List<IMeasurement>();
                            DataPacketFlags flags;
                            Ticks timestamp = 0;
                            int count;

                            if (m_totalBytesReceived == 0)
                            {
                                // At the point when data is being received, data monitor should be enabled
                                if ((object)m_dataStreamMonitor != null && !m_dataStreamMonitor.Enabled)
                                    m_dataStreamMonitor.Enabled = true;

                                // Establish run-time log for subscriber
                                if (m_autoConnect || m_dataGapRecoveryEnabled)
                                {
                                    if ((object)m_runTimeLog == null)
                                    {
                                        m_runTimeLog = new RunTimeLog();
                                        m_runTimeLog.FileName = GetLoggingPath(Name + "_RunTimeLog.txt");
                                        m_runTimeLog.ProcessException += m_runTimeLog_ProcessException;
                                        m_runTimeLog.Initialize();
                                    }
                                    else
                                    {
                                        // Mark the start of any data transmissions
                                        m_runTimeLog.StartTime = DateTime.UtcNow;
                                        m_runTimeLog.Enabled = true;
                                    }
                                }

                                // The duration between last disconnection and start of data transmissions
                                // represents a gap in data - if data gap recovery is enabled, we log
                                // this as a gap for recovery:
                                if (m_dataGapRecoveryEnabled && (object)m_dataGapRecoverer != null)
                                    m_dataGapRecoverer.LogDataGap(m_runTimeLog.StopTime, DateTime.UtcNow);
                            }

                            // Track total data packet bytes received from any channel
                            m_totalBytesReceived += m_lastBytesReceived;
                            m_monitoredBytesReceived += m_lastBytesReceived;

                            // Get data packet flags
                            flags = (DataPacketFlags)buffer[responseIndex];
                            responseIndex++;

                            bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0);
                            bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0);
                            bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0);
                            int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0;

                            // Decrypt data packet payload if keys are available
                            if ((object)m_keyIVs != null)
                            {
                                // Get a local copy of volatile keyIVs reference since this can change at any time
                                keyIVs = m_keyIVs;

                                // Decrypt payload portion of data packet
                                buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]);
                                responseIndex = 0;
                                responseLength = buffer.Length;
                            }

                            // Synchronized packets contain a frame level timestamp
                            if (synchronizedMeasurements)
                            {
                                timestamp = BigEndian.ToInt64(buffer, responseIndex);
                                responseIndex += 8;
                            }

                            // Deserialize number of measurements that follow
                            count = BigEndian.ToInt32(buffer, responseIndex);
                            responseIndex += 4;

                            if (compressedPayload)
                            {
                                if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
                                {
                                    if (m_lastMissingCacheWarning != 0L)
                                    {
                                        // Warning message for missing signal index cache
                                        OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
                                    }

                                    m_lastMissingCacheWarning = now;
                                }
                                else
                                {
                                    try
                                    {
                                        if (CompressionModes.HasFlag(CompressionModes.TSSC))
                                        {
                                            // Use TSSC compression to decompress measurements                                            
                                            if ((object)m_decompressionBlock == null)
                                                m_decompressionBlock = new MeasurementDecompressionBlock();

                                            MemoryStream bufferStream = new MemoryStream(buffer, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize);
                                            bool eos = false;

                                            while (!eos)
                                            {
                                                Measurement measurement;
                                                Tuple<Guid, string, uint> tuple;
                                                ushort id;
                                                long time;
                                                uint quality;
                                                float value;
                                                byte command;

                                                switch (m_decompressionBlock.GetMeasurement(out id, out time, out quality, out value, out command))
                                                {
                                                    case DecompressionExitCode.EndOfStreamOccured:
                                                        if (bufferStream.Position != bufferStream.Length)
                                                            m_decompressionBlock.Fill(bufferStream);
                                                        else
                                                            eos = true;
                                                        break;
                                                    case DecompressionExitCode.CommandRead:
                                                        break;
                                                    case DecompressionExitCode.MeasurementRead:                                                        
                                                        // Attempt to restore signal identification
                                                        if (m_signalIndexCache.Reference.TryGetValue(id, out tuple))
                                                        {
                                                            measurement = new Measurement();
                                                            measurement.Key = MeasurementKey.LookUpOrCreate(tuple.Item1, tuple.Item2, tuple.Item3);
                                                            measurement.Timestamp = time;
                                                            measurement.StateFlags = (MeasurementStateFlags)quality;
                                                            measurement.Value = value;
                                                            measurements.Add(measurement);
                                                        }
                                                        break;
                                                }
                                            }
                                        }
                                        else
                                        {
                                            // Decompress compact measurements from payload
                                            measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags));
                                        }
                                    }
                                    catch (Exception ex)
                                    {
                                        OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex));
                                    }
                                }
                            }
                            else
                            {
                                // Deserialize measurements
                                for (int i = 0; i < count; i++)
                                {
                                    if (!compactMeasurementFormat)
                                    {
                                        // Deserialize full measurement format
                                        SerializableMeasurement measurement = new SerializableMeasurement(m_encoding);
                                        responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
                                        measurements.Add(measurement);
                                    }
                                    else if ((object)m_signalIndexCache != null)
                                    {
                                        // Deserialize compact measurement format
                                        CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
                                        responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);

                                        // Apply timestamp from frame if not included in transmission
                                        if (!measurement.IncludeTime)
                                            measurement.Timestamp = timestamp;

                                        measurements.Add(measurement);
                                    }
                                    else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
                                    {
                                        if (m_lastMissingCacheWarning != 0L)
                                        {
                                            // Warning message for missing signal index cache
                                            OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
                                        }

                                        m_lastMissingCacheWarning = now;
                                    }
                                }
                            }

                            // Calculate statistics
                            subscribedDevicesLookup = m_subscribedDevicesLookup;
                            statisticsHelper = null;

                            if ((object)subscribedDevicesLookup != null)
                            {
                                IEnumerable<IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement>> deviceGroups = measurements
                                    .Where(measurement => subscribedDevicesLookup.TryGetValue(measurement.ID, out statisticsHelper))
                                    .Select(measurement => Tuple.Create(statisticsHelper, measurement))
                                    .ToList()
                                    .GroupBy(tuple => tuple.Item1, tuple => tuple.Item2);

                                foreach (IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement> deviceGroup in deviceGroups)
                                {
                                    statisticsHelper = deviceGroup.Key;

                                    foreach (IGrouping<Ticks, IMeasurement> frame in deviceGroup.GroupBy(measurement => measurement.Timestamp))
                                    {
                                        // Determine the number of measurements received with valid values
                                        int measurementsReceived = frame.Count(measurement => !double.IsNaN(measurement.Value));

                                        IMeasurement statusFlags = null;
                                        IMeasurement frequency = null;
                                        IMeasurement deltaFrequency = null;

                                        // Attempt to update real-time
                                        if (!m_useLocalClockAsRealTime && frame.Key > m_realTime)
                                            m_realTime = frame.Key;

                                        // Search the frame for status flags, frequency, and delta frequency
                                        foreach (IMeasurement measurement in frame)
                                        {
                                            if (measurement.ID == statisticsHelper.Device.StatusFlagsID)
                                                statusFlags = measurement;
                                            else if (measurement.ID == statisticsHelper.Device.FrequencyID)
                                                frequency = measurement;
                                            else if (measurement.ID == statisticsHelper.Device.DeltaFrequencyID)
                                                deltaFrequency = measurement;
                                        }

                                        // If we are receiving status flags for this device,
                                        // count the data quality, time quality, and device errors
                                        if ((object)statusFlags != null)
                                        {
                                            uint commonStatusFlags = (uint)statusFlags.Value;

                                            if ((commonStatusFlags & (uint)Bits.Bit19) > 0)
                                                statisticsHelper.Device.DataQualityErrors++;

                                            if ((commonStatusFlags & (uint)Bits.Bit18) > 0)
                                                statisticsHelper.Device.TimeQualityErrors++;

                                            if ((commonStatusFlags & (uint)Bits.Bit16) > 0)
                                                statisticsHelper.Device.DeviceErrors++;

                                            measurementsReceived--;
                                        }

                                        // Zero is not a valid value for frequency.
                                        // If frequency is zero, invalidate both frequency and delta frequency
                                        if ((object)frequency != null && frequency.Value == 0.0D)
                                        {
                                            if ((object)deltaFrequency != null)
                                                measurementsReceived -= 2;
                                            else
                                                measurementsReceived--;
                                        }

                                        // Track the number of measurements received
                                        statisticsHelper.AddToMeasurementsReceived(measurementsReceived);
                                    }
                                }
                            }

                            // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer
                            if ((object)m_localConcentrator != null)
                                m_localConcentrator.SortMeasurements(measurements);
                            else
                                OnNewMeasurements(measurements);

                            // Gather statistics on received data
                            DateTime timeReceived = RealTime;

                            if (!m_useLocalClockAsRealTime && timeReceived.Ticks - m_lastStatisticsHelperUpdate > Ticks.PerSecond)
                            {
                                UpdateStatisticsHelpers();
                                m_lastStatisticsHelperUpdate = m_realTime;
                            }

                            m_lifetimeMeasurements += measurements.Count;
                            UpdateMeasurementsPerSecond(timeReceived, measurements.Count);

                            for (int x = 0; x < measurements.Count; x++)
                            {
                                long latency = timeReceived.Ticks - (long)measurements[x].Timestamp;

                                // Throw out latencies that exceed one hour as invalid
                                if (Math.Abs(latency) > Time.SecondsPerHour * Ticks.PerSecond)
                                    continue;

                                if (m_lifetimeMinimumLatency > latency || m_lifetimeMinimumLatency == 0)
                                    m_lifetimeMinimumLatency = latency;

                                if (m_lifetimeMaximumLatency < latency || m_lifetimeMaximumLatency == 0)
                                    m_lifetimeMaximumLatency = latency;

                                m_lifetimeTotalLatency += latency;
                                m_lifetimeLatencyMeasurements++;
                            }
                            break;
                        case ServerResponse.BufferBlock:
                            // Buffer block received - wrap as a buffer block measurement and expose back to consumer
                            uint sequenceNumber = BigEndian.ToUInt32(buffer, responseIndex);
                            int cacheIndex = (int)(sequenceNumber - m_expectedBufferBlockSequenceNumber);
                            BufferBlockMeasurement bufferBlockMeasurement;
                            Tuple<Guid, string, uint> measurementKey;
                            ushort signalIndex;

                            // Check if this buffer block has already been processed (e.g., mistaken retransmission due to timeout)
                            if (cacheIndex >= 0 && (cacheIndex >= m_bufferBlockCache.Count || (object)m_bufferBlockCache[cacheIndex] == null))
                            {
                                // Send confirmation that buffer block is received
                                SendServerCommand(ServerCommand.ConfirmBufferBlock, buffer.BlockCopy(responseIndex, 4));

                                // Get measurement key from signal index cache
                                signalIndex = BigEndian.ToUInt16(buffer, responseIndex + 4);

                                if (!m_signalIndexCache.Reference.TryGetValue(signalIndex, out measurementKey))
                                    throw new InvalidOperationException("Failed to find associated signal identification for runtime ID " + signalIndex);

                                // Skip the sequence number and signal index when creating the buffer block measurement
                                bufferBlockMeasurement = new BufferBlockMeasurement(buffer, responseIndex + 6, responseLength - 6)
                                {
                                    Key = MeasurementKey.LookUpOrCreate(measurementKey.Item1, measurementKey.Item2, measurementKey.Item3)
                                };

                                // Determine if this is the next buffer block in the sequence
                                if (sequenceNumber == m_expectedBufferBlockSequenceNumber)
                                {
                                    List<IMeasurement> bufferBlockMeasurements = new List<IMeasurement>();
                                    int i;

                                    // Add the buffer block measurement to the list of measurements to be published
                                    bufferBlockMeasurements.Add(bufferBlockMeasurement);
                                    m_expectedBufferBlockSequenceNumber++;

                                    // Add cached buffer block measurements to the list of measurements to be published
                                    for (i = 1; i < m_bufferBlockCache.Count; i++)
                                    {
                                        if ((object)m_bufferBlockCache[i] == null)
                                            break;

                                        bufferBlockMeasurements.Add(m_bufferBlockCache[i]);
                                        m_expectedBufferBlockSequenceNumber++;
                                    }

                                    // Remove published measurements from the buffer block queue
                                    if (m_bufferBlockCache.Count > 0)
                                        m_bufferBlockCache.RemoveRange(0, i);

                                    // Publish measurements
                                    OnNewMeasurements(bufferBlockMeasurements);
                                }
                                else
                                {
                                    // Ensure that the list has at least as many
                                    // elements as it needs to cache this measurement
                                    for (int i = m_bufferBlockCache.Count; i <= cacheIndex; i++)
                                        m_bufferBlockCache.Add(null);

                                    // Insert this buffer block into the proper location in the list
                                    m_bufferBlockCache[cacheIndex] = bufferBlockMeasurement;
                                }
                            }

                            m_lifetimeMeasurements += 1;
                            UpdateMeasurementsPerSecond(DateTime.UtcNow, 1);
                            break;
                        case ServerResponse.DataStartTime:
                            // Raise data start time event
                            OnDataStartTime(BigEndian.ToInt64(buffer, responseIndex));
                            break;
                        case ServerResponse.ProcessingComplete:
                            // Raise input processing completed event
                            OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength));
                            break;
                        case ServerResponse.UpdateSignalIndexCache:
                            // Deserialize new signal index cache
                            m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength));
                            m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
                            FixExpectedMeasurementCounts();
                            break;
                        case ServerResponse.UpdateBaseTimes:
                            // Get active time index
                            m_timeIndex = BigEndian.ToInt32(buffer, responseIndex);
                            responseIndex += 4;

                            // Deserialize new base time offsets
                            m_baseTimeOffsets = new[] { BigEndian.ToInt64(buffer, responseIndex), BigEndian.ToInt64(buffer, responseIndex + 8) };
                            break;
                        case ServerResponse.UpdateCipherKeys:
                            // Move past active cipher index (not currently used anywhere else)
                            responseIndex++;

                            // Extract remaining response
                            byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1);

                            // Decrypt response payload if subscription is authenticated
                            if (m_authenticated)
                                bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256);

                            // Deserialize new cipher keys
                            keyIVs = new byte[2][][];
                            keyIVs[EvenKey] = new byte[2][];
                            keyIVs[OddKey] = new byte[2][];

                            int index = 0;
                            int bufferLen;

                            // Read even key size
                            bufferLen = BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read even key
                            keyIVs[EvenKey][KeyIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read even initialization vector size
                            bufferLen = BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read even initialization vector
                            keyIVs[EvenKey][IVIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read odd key size
                            bufferLen = BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read odd key
                            keyIVs[OddKey][KeyIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read odd initialization vector size
                            bufferLen = BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read odd initialization vector
                            keyIVs[OddKey][IVIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen);
                            //index += bufferLen;

                            // Exchange keys
                            m_keyIVs = keyIVs;

                            OnStatusMessage("Successfully established new cipher keys for data packet transmissions.");
                            break;
                        case ServerResponse.Notify:
                            // Skip the 4-byte hash
                            string message = m_encoding.GetString(buffer, responseIndex + 4, responseLength - 4);

                            // Display notification
                            OnStatusMessage("NOTIFICATION: {0}", message);
                            OnNotificationReceived(message);

                            // Send confirmation of receipt of the notification
                            SendServerCommand(ServerCommand.ConfirmNotification, buffer.BlockCopy(responseIndex, 4));
                            break;
                        case ServerResponse.ConfigurationChanged:
                            OnStatusMessage("Received notification from publisher that configuration has changed.");
                            OnServerConfigurationChanged();

                            // Initiate meta-data refresh when publisher configuration has changed - we only do this
                            // for automatic connections since API style connections have to manually initiate a
                            // meta-data refresh. API style connection should attach to server configuration changed
                            // event and request meta-data refresh to complete automated cycle.
                            if (m_autoConnect && m_autoSynchronizeMetadata)
                                SendServerCommand(ServerCommand.MetaDataRefresh, m_metadataFilters);
                            break;
                    }
                }
                catch (Exception ex)
                {
                    OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex));
                }
            }
        }
Example #24
0
        private void ProcessServerResponse(byte[] buffer, int length)
        {
            // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing
            // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread...
            if (buffer != null && length > 0)
            {
                try
                {
                    ServerResponse responseCode = (ServerResponse)buffer[0];
                    ServerCommand commandCode = (ServerCommand)buffer[1];
                    int responseLength = EndianOrder.BigEndian.ToInt32(buffer, 2);
                    int responseIndex = 6;
                    bool solicited = false;
                    byte[][][] keyIVs;

                    // See if this was a solicited response to a requested server command
                    if (responseCode.IsSolicited())
                    {
                        lock (m_requests)
                        {
                            int index = m_requests.BinarySearch(commandCode);

                            if (index >= 0)
                            {
                                solicited = true;
                                m_requests.RemoveAt(index);
                            }
                        }

                        // Disconnect any established UDP data channel upon successful unsubscribe
                        if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded)
                            DataChannel = null;
                    }

                    switch (responseCode)
                    {
                        case ServerResponse.Succeeded:
                            if (solicited)
                            {
                                switch (commandCode)
                                {
                                    case ServerCommand.Authenticate:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_authenticated = true;
                                        OnConnectionAuthenticated();
                                        break;
                                    case ServerCommand.Subscribe:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_subscribed = true;
                                        if ((object)m_dataStreamMonitor != null)
                                            m_dataStreamMonitor.Enabled = true;
                                        break;
                                    case ServerCommand.Unsubscribe:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        m_subscribed = false;
                                        if ((object)m_dataStreamMonitor != null)
                                            m_dataStreamMonitor.Enabled = false;
                                        break;
                                    case ServerCommand.RotateCipherKeys:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    case ServerCommand.MetaDataRefresh:
                                        OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode);
                                        OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
                                        break;
                                }
                            }
                            else
                            {
                                switch (commandCode)
                                {
                                    case ServerCommand.MetaDataRefresh:
                                        // Meta-data refresh may be unsolicited
                                        OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode);
                                        OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength)));
                                        break;
                                    case ServerCommand.RotateCipherKeys:
                                        // Key rotation may be unsolicited
                                        OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    case ServerCommand.Subscribe:
                                        OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                                        break;
                                    default:
                                        OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode));
                                        break;
                                }
                            }
                            break;
                        case ServerResponse.Failed:
                            if (solicited)
                                OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength));
                            else
                                OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode));
                            break;
                        case ServerResponse.DataPacket:
                            long now = DateTime.UtcNow.Ticks;

                            // Deserialize data packet
                            List<IMeasurement> measurements = new List<IMeasurement>();
                            DataPacketFlags flags;
                            Ticks timestamp = 0;
                            int count;

                            // Track total data packet bytes received from any channel
                            m_totalBytesReceived += m_lastBytesReceived;
                            m_monitoredBytesReceived += m_lastBytesReceived;

                            // Get data packet flags
                            flags = (DataPacketFlags)buffer[responseIndex];
                            responseIndex++;

                            bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0);
                            bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0);
                            int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0;

                            // Decrypt data packet payload if keys are available
//                            if (m_keyIVs != null)
//                            {
//                                // Get a local copy of volatile keyIVs reference since this can change at any time
//                                keyIVs = m_keyIVs;
//
//                                // Decrypt payload portion of data packet
//                                buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]);
//                                responseIndex = 0;
//                                responseLength = buffer.Length;
//                            }

                            // Synchronized packets contain a frame level timestamp
                            if (synchronizedMeasurements)
                            {
                                timestamp = EndianOrder.BigEndian.ToInt64(buffer, responseIndex);
                                responseIndex += 8;
                            }

                            // Deserialize number of measurements that follow
                            count = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
                            responseIndex += 4;

                            // Deserialize measurements
                            for (int i = 0; i < count; i++)
                            {
                                if (!compactMeasurementFormat)
                                {
                                    // Deserialize full measurement format
                                    SerializableMeasurement measurement = new SerializableMeasurement(m_encoding);
                                    responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);
                                    measurements.Add(measurement);
                                }
                                else if ((object)m_signalIndexCache != null)
                                {
                                    // Deserialize compact measurement format
                                    CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution);
                                    responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex);

                                    // Apply timestamp from frame if not included in transmission
                                    if (!measurement.IncludeTime)
                                        measurement.Timestamp = timestamp;

                                    measurements.Add(measurement);
                                }
                                else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now)
                                {
                                    if (m_lastMissingCacheWarning != 0L)
                                    {
                                        // Warning message for missing signal index cache
                                        OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed.");
                                    }

                                    m_lastMissingCacheWarning = now;
                                }
                            }

                            // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer
                            if ((object)m_localConcentrator != null)
                                m_localConcentrator.SortMeasurements(measurements);
                            else
                                OnNewMeasurements(measurements);
                            break;
                        case ServerResponse.DataStartTime:
                            // Raise data start time event
                            OnDataStartTime(EndianOrder.BigEndian.ToInt64(buffer, responseIndex));
                            break;
                        case ServerResponse.ProcessingComplete:
                            // Raise input processing completed event
                            OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength));
                            break;
                        case ServerResponse.UpdateSignalIndexCache:
                            // Deserialize new signal index cache
                            m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength));
                            m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache);
                            break;
                        case ServerResponse.UpdateBaseTimes:
                            // Get active time index
                            m_timeIndex = EndianOrder.BigEndian.ToInt32(buffer, responseIndex);
                            responseIndex += 4;

                            // Deserialize new base time offsets
                            m_baseTimeOffsets = new long[] { EndianOrder.BigEndian.ToInt64(buffer, responseIndex), EndianOrder.BigEndian.ToInt64(buffer, responseIndex + 8) };
                            break;
                        case ServerResponse.UpdateCipherKeys:
                            // Get active cipher index
                            m_cipherIndex = buffer[responseIndex++];

                            // Extract remaining response
                            byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1);

//                            // Decrypt response payload if subscription is authenticated
//                            if (m_authenticated)
//                                bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256);

                            // Deserialize new cipher keys
                            keyIVs = new byte[2][][];
                            keyIVs[EvenKey] = new byte[2][];
                            keyIVs[OddKey] = new byte[2][];

                            int index = 0;
                            int bufferLen;

                            // Read even key size
                            bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read even key
                            keyIVs[EvenKey][KeyIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read even initialization vector size
                            bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read even initialization vector
                            keyIVs[EvenKey][IVIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read odd key size
                            bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read odd key
                            keyIVs[OddKey][KeyIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen);
                            index += bufferLen;

                            // Read odd initialization vector size
                            bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index);
                            index += 4;

                            // Read odd initialization vector
                            keyIVs[OddKey][IVIndex] = new byte[bufferLen];
                            Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen);
                            index += bufferLen;

                            // Exchange keys
                            m_keyIVs = keyIVs;

                            OnStatusMessage("Successfully established new cipher keys for data packet transmissions.");
                            break;
                    }
                }
                catch (Exception ex)
                {
                    OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex));
                }
            }
        }