private void ProcessServerResponse(byte[] buffer, int length) { // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread... if (buffer != null && length > 0) { try { ServerResponse responseCode = (ServerResponse)buffer[0]; ServerCommand commandCode = (ServerCommand)buffer[1]; int responseLength = EndianOrder.BigEndian.ToInt32(buffer, 2); int responseIndex = DataPublisher.ClientResponseHeaderSize; bool solicited = false; byte[][][] keyIVs; // See if this was a solicited response to a requested server command if (responseCode.IsSolicited()) { lock (m_requests) { int index = m_requests.BinarySearch(commandCode); if (index >= 0) { solicited = true; m_requests.RemoveAt(index); } } // Disconnect any established UDP data channel upon successful unsubscribe if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded) DataChannel = null; } switch (responseCode) { case ServerResponse.Succeeded: if (solicited) { switch (commandCode) { case ServerCommand.Authenticate: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_authenticated = true; OnConnectionAuthenticated(); break; case ServerCommand.Subscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = true; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = true; break; case ServerCommand.Unsubscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = false; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = false; break; case ServerCommand.RotateCipherKeys: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.MetaDataRefresh: OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); break; } } else { switch (commandCode) { case ServerCommand.MetaDataRefresh: // Meta-data refresh may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); break; case ServerCommand.RotateCipherKeys: // Key rotation may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.Subscribe: OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; default: OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode)); break; } } break; case ServerResponse.Failed: if (solicited) OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); else OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode)); break; case ServerResponse.DataPacket: long now = DateTime.UtcNow.Ticks; // Deserialize data packet List<IMeasurement> measurements = new List<IMeasurement>(); DataPacketFlags flags; Ticks timestamp = 0; int count; // Track total data packet bytes received from any channel m_totalBytesReceived += m_lastBytesReceived; m_monitoredBytesReceived += m_lastBytesReceived; // Get data packet flags flags = (DataPacketFlags)buffer[responseIndex]; responseIndex++; bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0); bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0); bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0); int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0; // Decrypt data packet payload if keys are available if (m_keyIVs != null) { // Get a local copy of volatile keyIVs reference since this can change at any time keyIVs = m_keyIVs; // Decrypt payload portion of data packet buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]); responseIndex = 0; responseLength = buffer.Length; } // Synchronized packets contain a frame level timestamp if (synchronizedMeasurements) { timestamp = EndianOrder.BigEndian.ToInt64(buffer, responseIndex); responseIndex += 8; } // Deserialize number of measurements that follow count = EndianOrder.BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; if (compressedPayload) { if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } else { try { // Decompress compact measurements from payload measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags)); } catch (Exception ex) { OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex)); } } } else { // Deserialize measurements for (int i = 0; i < count; i++) { if (!compactMeasurementFormat) { // Deserialize full measurement format SerializableMeasurement measurement = new SerializableMeasurement(m_encoding); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); measurements.Add(measurement); } else if ((object)m_signalIndexCache != null) { // Deserialize compact measurement format CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); // Apply timestamp from frame if not included in transmission if (!measurement.IncludeTime) measurement.Timestamp = timestamp; measurements.Add(measurement); } else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } } } // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer if ((object)m_localConcentrator != null) m_localConcentrator.SortMeasurements(measurements); else OnNewMeasurements(measurements); break; case ServerResponse.DataStartTime: // Raise data start time event OnDataStartTime(EndianOrder.BigEndian.ToInt64(buffer, responseIndex)); break; case ServerResponse.ProcessingComplete: // Raise input processing completed event OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerResponse.UpdateSignalIndexCache: // Deserialize new signal index cache m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength)); m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache); break; case ServerResponse.UpdateBaseTimes: // Get active time index m_timeIndex = EndianOrder.BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; // Deserialize new base time offsets m_baseTimeOffsets = new long[] { EndianOrder.BigEndian.ToInt64(buffer, responseIndex), EndianOrder.BigEndian.ToInt64(buffer, responseIndex + 8) }; break; case ServerResponse.UpdateCipherKeys: // Get active cipher index m_cipherIndex = buffer[responseIndex++]; // Extract remaining response byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1); // Decrypt response payload if subscription is authenticated if (m_authenticated) bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256); // Deserialize new cipher keys keyIVs = new byte[2][][]; keyIVs[EvenKey] = new byte[2][]; keyIVs[OddKey] = new byte[2][]; int index = 0; int bufferLen; // Read even key size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read even key keyIVs[EvenKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read even initialization vector size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read even initialization vector keyIVs[EvenKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen); index += bufferLen; // Read odd key size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read odd key keyIVs[OddKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read odd initialization vector size bufferLen = EndianOrder.BigEndian.ToInt32(bytes, index); index += 4; // Read odd initialization vector keyIVs[OddKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen); index += bufferLen; // Exchange keys m_keyIVs = keyIVs; OnStatusMessage("Successfully established new cipher keys for data packet transmissions."); break; } } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex)); } } }
// Retrieves the measurements from the database. private void GetDbMeasurements(object state) { IDbConnection connection = null; // Get measurements from the database. try { SignalIndexCache signalIndexCache = new SignalIndexCache(); CompactMeasurement measurement; long startTime = DateTime.UtcNow.Ticks; if (m_cacheFileName != null && File.Exists(m_cacheFileName)) { OnStatusMessage("Loading cached input data..."); try { using (FileStream data = File.OpenRead(m_cacheFileName)) { byte[] buffer = new byte[4]; int signalIndexCacheImageSize; int compactMeasurementSize; int totalMeasurements; // Read the signal index cache image size from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); signalIndexCacheImageSize = EndianOrder.LittleEndian.ToInt32(buffer, 0); // Resize buffer to accomodate exact signal index cache buffer = new byte[signalIndexCacheImageSize]; // Read the signal index cache image from the file if (data.Read(buffer, 0, signalIndexCacheImageSize) != signalIndexCacheImageSize) throw new EndOfStreamException(); // Deserialize the signal index cache signalIndexCache = Serialization.Deserialize<SignalIndexCache>(buffer, TVA.SerializationFormat.Binary); // Read the size of each compact measurement from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); compactMeasurementSize = EndianOrder.LittleEndian.ToInt32(buffer, 0); // Read the total number of compact measurements from the file if (data.Read(buffer, 0, 4) != 4) throw new EndOfStreamException(); totalMeasurements = EndianOrder.LittleEndian.ToInt32(buffer, 0); // Resize buffer to accomodate compact measurement if needed (not likely) if (buffer.Length < compactMeasurementSize) buffer = new byte[compactMeasurementSize]; // Read each compact measurement image from the file for (int i = 0; i < totalMeasurements; i++) { if (data.Read(buffer, 0, compactMeasurementSize) != compactMeasurementSize) throw new EndOfStreamException(); // Parse compact measurement measurement = new CompactMeasurement(signalIndexCache); measurement.ParseBinaryImage(buffer, 0, compactMeasurementSize); m_dbMeasurements.Add(measurement); if (m_dbMeasurements.Count % 50000 == 0) OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count); } OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(4)); } } catch (Exception ex) { if (ex is EndOfStreamException) throw (EndOfStreamException)ex; throw new EndOfStreamException(ex.Message, ex); } } else { OnStatusMessage("Loading database input data..."); const string MeasurementTable = "ActiveMeasurements"; Dictionary<string, string> dataProviderSettings = m_dataProviderString.ParseKeyValuePairs(); Assembly assm = Assembly.Load(dataProviderSettings["AssemblyName"]); Type connectionType = assm.GetType(dataProviderSettings["ConnectionType"]); Dictionary<Guid, MeasurementKey> lookupCache = new Dictionary<Guid, MeasurementKey>(); IDbCommand command; IDataReader dbReader; MeasurementKey key; Guid id; ushort index = 0; connection = (IDbConnection)Activator.CreateInstance(connectionType); connection.ConnectionString = m_dbConnectionString; connection.Open(); command = connection.CreateCommand(); command.CommandText = string.Format("SELECT * FROM {0}", m_dbTableName); using (dbReader = command.ExecuteReader()) { while (dbReader.Read()) { measurement = new CompactMeasurement(signalIndexCache); foreach (string fieldName in m_fieldNames.Keys) { object value = dbReader[fieldName]; string propertyName = m_fieldNames[fieldName]; switch (propertyName) { case "Timestamp": // If the value is a timestamp, use the timestamp format // specified by the user when reading the timestamp. if (m_timestampFormat == null) measurement.Timestamp = long.Parse(value.ToNonNullString()); else measurement.Timestamp = DateTime.ParseExact(value.ToNonNullString(), m_timestampFormat, CultureInfo.CurrentCulture); break; case "ID": if (Guid.TryParse(value.ToString(), out id)) { if (!lookupCache.TryGetValue(id, out key)) { if (DataSource.Tables.Contains(MeasurementTable)) { DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("SignalID = '{0}'", id)); if (filteredRows.Length > 0) MeasurementKey.TryParse(filteredRows[0]["ID"].ToString(), id, out key); } if (key != default(MeasurementKey)) { // Cache measurement key associated with ID lookupCache[id] = key; // Assign a runtime index optimization for distinct measurements signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(id, key.Source, key.ID)); } } measurement.ID = id; measurement.Key = key; } break; case "Key": if (MeasurementKey.TryParse(value.ToString(), Guid.Empty, out key)) { // Attempt to update empty signal ID if available if (key.SignalID == Guid.Empty) { if (DataSource.Tables.Contains(MeasurementTable)) { DataRow[] filteredRows = DataSource.Tables[MeasurementTable].Select(string.Format("ID = '{0}'", key.ToString())); if (filteredRows.Length > 0) key.SignalID = filteredRows[0]["SignalID"].ToNonNullString(Guid.Empty.ToString()).ConvertToType<Guid>(); } } if (key.SignalID != Guid.Empty) { measurement.ID = key.SignalID; if (!lookupCache.ContainsKey(measurement.ID)) { // Cache measurement key associated with ID lookupCache[measurement.ID] = key; // Assign a runtime index optimization for distinct measurements signalIndexCache.Reference.TryAdd(index++, new Tuple<Guid, string, uint>(measurement.ID, key.Source, key.ID)); } } measurement.Key = key; } break; case "Value": measurement.Value = Convert.ToDouble(value); break; default: PropertyInfo property = GetAllProperties(typeof(IMeasurement)).FirstOrDefault(propertyInfo => propertyInfo.Name == propertyName); if (property != null) { Type propertyType = property.PropertyType; Type valueType = value.GetType(); if (property.PropertyType.IsAssignableFrom(value.GetType())) { property.SetValue(measurement, value, null); } else if (property.PropertyType == typeof(string)) { property.SetValue(measurement, value.ToNonNullString(), null); } else if (valueType == typeof(string)) { MethodInfo parseMethod = propertyType.GetMethod("Parse", new Type[] { typeof(string) }); if (parseMethod != null && parseMethod.IsStatic) property.SetValue(measurement, parseMethod.Invoke(null, new object[] { value }), null); } else { string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1}.", fieldName, propertyName); OnProcessException(new InvalidCastException(exceptionMessage)); } } else { string exceptionMessage = string.Format("The type of field {0} could not be converted to the type of property {1} - no property match was found.", fieldName, propertyName); OnProcessException(new InvalidCastException(exceptionMessage)); } break; } m_dbMeasurements.Add(measurement); if (m_dbMeasurements.Count % 50000 == 0) OnStatusMessage("Loaded {0} records so far...", m_dbMeasurements.Count); } } } OnStatusMessage("Sorting data by time..."); m_dbMeasurements = m_dbMeasurements.OrderBy(m => (long)m.Timestamp).ToList(); OnStatusMessage("Completed data load in {0}", ((Ticks)(DateTime.UtcNow.Ticks - startTime)).ToElapsedTimeString(4)); if (m_cacheFileName != null) { OnStatusMessage("Caching data for next initialization..."); using (FileStream data = File.OpenWrite(m_cacheFileName)) { byte[] signalIndexCacheImage = Serialization.Serialize(signalIndexCache, TVA.SerializationFormat.Binary); int compactMeasurementSize = (new CompactMeasurement(signalIndexCache)).BinaryLength; // Write the signal index cache image size to the file data.Write(EndianOrder.LittleEndian.GetBytes(signalIndexCacheImage.Length), 0, 4); // Write the signal index cache image to the file data.Write(signalIndexCacheImage, 0, signalIndexCacheImage.Length); // Write the size of each compact measurement to the file data.Write(EndianOrder.LittleEndian.GetBytes(compactMeasurementSize), 0, 4); // Write the total number of compact measurements to the file data.Write(EndianOrder.LittleEndian.GetBytes(m_dbMeasurements.Count), 0, 4); // Write each compact measurement image to the file for (int i = 0; i < m_dbMeasurements.Count; i++) { ((ISupportBinaryImage)m_dbMeasurements[i]).CopyBinaryImageToStream(data); } } } } OnStatusMessage("Entering data read cycle..."); ThreadPool.QueueUserWorkItem(PublishData); } catch (EndOfStreamException ex) { OnProcessException(new EndOfStreamException(string.Format("Failed load cached data from {0} due to file corruption{1} cache will be recreated from database", m_cacheFileName, string.IsNullOrWhiteSpace(ex.Message) ? "," : ": " + ex.Message + " - "))); // If the cached file is corrupt, delete it and load from the database if (File.Exists(m_cacheFileName)) File.Delete(m_cacheFileName); m_dbMeasurements.Clear(); GetDbMeasurements(null); } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed during data load: " + ex.Message, ex)); } finally { if (connection != null) connection.Close(); } }
/// <summary> /// Decompresses <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer. /// </summary> /// <param name="source">Buffer with compressed <see cref="CompactMeasurement"/> payload.</param> /// <param name="signalIndexCache">Current <see cref="SignalIndexCache"/>.</param> /// <param name="index">Index into buffer where compressed payload begins.</param> /// <param name="dataLength">Length of all data within <paramref name="source"/> buffer.</param> /// <param name="measurementCount">Number of compressed measurements in the payload.</param> /// <param name="includeTime">Flag that determines if timestamps as included in the payload.</param> /// <param name="flags">Current <see cref="DataPacketFlags"/>.</param> /// <returns>Decompressed <see cref="CompactMeasurement"/> values from the given <paramref name="source"/> buffer.</returns> public static CompactMeasurement[] DecompressPayload(this byte[] source, SignalIndexCache signalIndexCache, int index, int dataLength, int measurementCount, bool includeTime, DataPacketFlags flags) { CompactMeasurement[] measurements = new CompactMeasurement[measurementCount]; byte[] buffer = null; try { // Actual data length has to take into account response byte and in-response-to server command byte in the payload header //int dataLength = length - index - 2; int bufferLength = PatternDecompressor.MaximumSizeDecompressed(dataLength); // Copy source data into a decompression buffer buffer = BufferPool.TakeBuffer(bufferLength); Buffer.BlockCopy(source, index, buffer, 0, dataLength); // Check that OS endian-order matches endian-order of compressed data if (!(BitConverter.IsLittleEndian && (flags & DataPacketFlags.LittleEndianCompression) > 0)) { // TODO: Set a flag, e.g., Endianness decompressAs, to pass into pattern decompressor so it // can be modified to decompress a payload that is non-native Endian order throw new NotImplementedException("Cannot currently decompress payload that is not in native endian-order."); } // Attempt to decompress buffer int uncompressedSize = PatternDecompressor.DecompressBuffer(buffer, 0, dataLength, bufferLength); if (uncompressedSize == 0) throw new InvalidOperationException("Failed to decompress payload buffer - possible data corruption."); index = 0; // Decode ID and state flags for (int i = 0; i < measurementCount; i++) { uint value = NativeEndianOrder.Default.ToUInt32(buffer, index); measurements[i] = new CompactMeasurement(signalIndexCache, includeTime) { CompactStateFlags = (byte)(value >> 16), RuntimeID = (ushort)value }; index += 4; } // Decode values for (int i = 0; i < measurementCount; i++) { measurements[i].Value = NativeEndianOrder.Default.ToSingle(buffer, index); index += 4; } if (includeTime) { // Decode timestamps for (int i = 0; i < measurementCount; i++) { measurements[i].Timestamp = NativeEndianOrder.Default.ToInt64(buffer, index); index += 8; } } } finally { if ((object)buffer != null) BufferPool.ReturnBuffer(buffer); } return measurements; }
/// <summary> /// Publish <see cref="IFrame"/> of time-aligned collection of <see cref="IMeasurement"/> values that arrived within the /// concentrator's defined <see cref="ConcentratorBase.LagTime"/>. /// </summary> /// <param name="frame"><see cref="IFrame"/> of measurements with the same timestamp that arrived within <see cref="ConcentratorBase.LagTime"/> that are ready for processing.</param> /// <param name="index">Index of <see cref="IFrame"/> within a second ranging from zero to <c><see cref="ConcentratorBase.FramesPerSecond"/> - 1</c>.</param> protected override void PublishFrame(IFrame frame, int index) { if ((object)m_parent == null || m_disposed) return; // Includes data packet flags, frame level timestamp and measurement count const int PacketHeaderSize = DataPublisher.ClientResponseHeaderSize + 13; List<IBinaryMeasurement> packet = new List<IBinaryMeasurement>(); bool usePayloadCompression = m_usePayloadCompression; bool useCompactMeasurementFormat = m_useCompactMeasurementFormat || usePayloadCompression; long frameLevelTimestamp = frame.Timestamp; IBinaryMeasurement binaryMeasurement; int packetSize = PacketHeaderSize; int binaryLength; foreach (IMeasurement measurement in frame.Measurements.Values) { // Serialize the current measurement. if (useCompactMeasurementFormat) binaryMeasurement = new CompactMeasurement(measurement, m_signalIndexCache, false); else binaryMeasurement = new SerializableMeasurement(measurement, m_parent.GetClientEncoding(ClientID)); // Determine the size of the measurement in bytes. binaryLength = binaryMeasurement.BinaryLength; // If the current measurement will not fit in the packet based on // the max packet size, process the packet and start a new one. if (packetSize + binaryLength > DataPublisher.MaxPacketSize) { ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression); packet.Clear(); packetSize = PacketHeaderSize; } // Add the measurement to the packet. packet.Add(binaryMeasurement); packetSize += binaryLength; } // Process the remaining measurements. ProcessBinaryMeasurements(packet, frameLevelTimestamp, useCompactMeasurementFormat, usePayloadCompression); }