private void ProcessBufferBlockMeasurement(BufferBlockMeasurement measurement) { byte[] bufferBlock = measurement.Buffer; int index = 1; if (bufferBlock[0] != 0) { // Start of a new file - read file info int fileNameByteLength = BigEndian.ToInt32(bufferBlock, 1); string fileName = Encoding.Unicode.GetString(bufferBlock, 5, fileNameByteLength); long fileSize = BigEndian.ToInt64(bufferBlock, 5 + fileNameByteLength); // Notify of new file creation OnStatusMessage(MessageLevel.Info, "Now writing to file {0}...", fileName); // ReSharper disable once UnusedVariable > Justification: Implementation pattern closes any existing stream // Create new file using (FileStream activeFileStream = m_activeFileStream) m_activeFileStream = File.Create(Path.Combine(m_outputDirectory, fileName)); m_activeFileStream.SetLength(fileSize); m_activeFileSize = fileSize; m_bytesWritten = 0L; // Advance buffer pointer to file data index = 1 + 4 + fileNameByteLength + 8; } if ((object)m_activeFileStream != null) { // Write data into the file int bytesOfData = measurement.Length - index; m_activeFileStream.Write(bufferBlock, index, bytesOfData); m_bytesWritten += bytesOfData; // Close the file when we detect that // we've written all bytes to the file if (m_bytesWritten >= m_activeFileSize) { OnStatusMessage(MessageLevel.Info, "Finished writing to file {0}.", Path.GetFileName(m_activeFileStream.Name)); m_activeFileStream.Dispose(); m_activeFileStream = null; m_bytesWritten = 0L; } } }
private void ProcessServerResponse(byte[] buffer, int length) { // Currently this work is done on the async socket completion thread, make sure work to be done is timely and if the response processing // is coming in via the command channel and needs to send a command back to the server, it should be done on a separate thread... if (buffer != null && length > 0) { try { Dictionary<Guid, DeviceStatisticsHelper<SubscribedDevice>> subscribedDevicesLookup; DeviceStatisticsHelper<SubscribedDevice> statisticsHelper; ServerResponse responseCode = (ServerResponse)buffer[0]; ServerCommand commandCode = (ServerCommand)buffer[1]; int responseLength = BigEndian.ToInt32(buffer, 2); int responseIndex = DataPublisher.ClientResponseHeaderSize; bool solicited = false; byte[][][] keyIVs; // See if this was a solicited response to a requested server command if (responseCode.IsSolicited()) { lock (m_requests) { int index = m_requests.BinarySearch(commandCode); if (index >= 0) { solicited = true; m_requests.RemoveAt(index); } } // Disconnect any established UDP data channel upon successful unsubscribe if (solicited && commandCode == ServerCommand.Unsubscribe && responseCode == ServerResponse.Succeeded) DataChannel = null; } OnReceivedServerResponse(responseCode, commandCode); switch (responseCode) { case ServerResponse.Succeeded: if (solicited) { switch (commandCode) { case ServerCommand.Authenticate: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_authenticated = true; OnConnectionAuthenticated(); break; case ServerCommand.Subscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = true; break; case ServerCommand.Unsubscribe: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); m_subscribed = false; if ((object)m_dataStreamMonitor != null) m_dataStreamMonitor.Enabled = false; break; case ServerCommand.RotateCipherKeys: OnStatusMessage("Success code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.MetaDataRefresh: OnStatusMessage("Success code received in response to server command \"{0}\": latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); m_metadataRefreshPending = false; break; } } else { switch (commandCode) { case ServerCommand.MetaDataRefresh: // Meta-data refresh may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: latest meta-data received.", commandCode); OnMetaDataReceived(DeserializeMetadata(buffer.BlockCopy(responseIndex, responseLength))); m_metadataRefreshPending = false; break; case ServerCommand.RotateCipherKeys: // Key rotation may be unsolicited OnStatusMessage("Received server confirmation for unsolicited request to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerCommand.Subscribe: OnStatusMessage("Received unsolicited response to \"{0}\" command: {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); break; default: OnProcessException(new InvalidOperationException("Publisher sent a success code for an unsolicited server command: " + commandCode)); break; } } break; case ServerResponse.Failed: if (solicited) OnStatusMessage("Failure code received in response to server command \"{0}\": {1}", commandCode, InterpretResponseMessage(buffer, responseIndex, responseLength)); else OnProcessException(new InvalidOperationException("Publisher sent a failed code for an unsolicited server command: " + commandCode)); if (commandCode == ServerCommand.MetaDataRefresh) m_metadataRefreshPending = false; break; case ServerResponse.DataPacket: long now = DateTime.UtcNow.Ticks; // Deserialize data packet List<IMeasurement> measurements = new List<IMeasurement>(); DataPacketFlags flags; Ticks timestamp = 0; int count; if (m_totalBytesReceived == 0) { // At the point when data is being received, data monitor should be enabled if ((object)m_dataStreamMonitor != null && !m_dataStreamMonitor.Enabled) m_dataStreamMonitor.Enabled = true; // Establish run-time log for subscriber if (m_autoConnect || m_dataGapRecoveryEnabled) { if ((object)m_runTimeLog == null) { m_runTimeLog = new RunTimeLog(); m_runTimeLog.FileName = GetLoggingPath(Name + "_RunTimeLog.txt"); m_runTimeLog.ProcessException += m_runTimeLog_ProcessException; m_runTimeLog.Initialize(); } else { // Mark the start of any data transmissions m_runTimeLog.StartTime = DateTime.UtcNow; m_runTimeLog.Enabled = true; } } // The duration between last disconnection and start of data transmissions // represents a gap in data - if data gap recovery is enabled, we log // this as a gap for recovery: if (m_dataGapRecoveryEnabled && (object)m_dataGapRecoverer != null) m_dataGapRecoverer.LogDataGap(m_runTimeLog.StopTime, DateTime.UtcNow); } // Track total data packet bytes received from any channel m_totalBytesReceived += m_lastBytesReceived; m_monitoredBytesReceived += m_lastBytesReceived; // Get data packet flags flags = (DataPacketFlags)buffer[responseIndex]; responseIndex++; bool synchronizedMeasurements = ((byte)(flags & DataPacketFlags.Synchronized) > 0); bool compactMeasurementFormat = ((byte)(flags & DataPacketFlags.Compact) > 0); bool compressedPayload = ((byte)(flags & DataPacketFlags.Compressed) > 0); int cipherIndex = (flags & DataPacketFlags.CipherIndex) > 0 ? 1 : 0; // Decrypt data packet payload if keys are available if ((object)m_keyIVs != null) { // Get a local copy of volatile keyIVs reference since this can change at any time keyIVs = m_keyIVs; // Decrypt payload portion of data packet buffer = Common.SymmetricAlgorithm.Decrypt(buffer, responseIndex, responseLength - 1, keyIVs[cipherIndex][0], keyIVs[cipherIndex][1]); responseIndex = 0; responseLength = buffer.Length; } // Synchronized packets contain a frame level timestamp if (synchronizedMeasurements) { timestamp = BigEndian.ToInt64(buffer, responseIndex); responseIndex += 8; } // Deserialize number of measurements that follow count = BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; if (compressedPayload) { if ((object)m_signalIndexCache == null && m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } else { try { if (CompressionModes.HasFlag(CompressionModes.TSSC)) { // Use TSSC compression to decompress measurements if ((object)m_decompressionBlock == null) m_decompressionBlock = new MeasurementDecompressionBlock(); MemoryStream bufferStream = new MemoryStream(buffer, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize); bool eos = false; while (!eos) { Measurement measurement; Tuple<Guid, string, uint> tuple; ushort id; long time; uint quality; float value; byte command; switch (m_decompressionBlock.GetMeasurement(out id, out time, out quality, out value, out command)) { case DecompressionExitCode.EndOfStreamOccured: if (bufferStream.Position != bufferStream.Length) m_decompressionBlock.Fill(bufferStream); else eos = true; break; case DecompressionExitCode.CommandRead: break; case DecompressionExitCode.MeasurementRead: // Attempt to restore signal identification if (m_signalIndexCache.Reference.TryGetValue(id, out tuple)) { measurement = new Measurement(); measurement.Key = MeasurementKey.LookUpOrCreate(tuple.Item1, tuple.Item2, tuple.Item3); measurement.Timestamp = time; measurement.StateFlags = (MeasurementStateFlags)quality; measurement.Value = value; measurements.Add(measurement); } break; } } } else { // Decompress compact measurements from payload measurements.AddRange(buffer.DecompressPayload(m_signalIndexCache, responseIndex, responseLength - responseIndex + DataPublisher.ClientResponseHeaderSize, count, m_includeTime, flags)); } } catch (Exception ex) { OnProcessException(new InvalidOperationException("WARNING: Decompression failure: " + ex.Message, ex)); } } } else { // Deserialize measurements for (int i = 0; i < count; i++) { if (!compactMeasurementFormat) { // Deserialize full measurement format SerializableMeasurement measurement = new SerializableMeasurement(m_encoding); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); measurements.Add(measurement); } else if ((object)m_signalIndexCache != null) { // Deserialize compact measurement format CompactMeasurement measurement = new CompactMeasurement(m_signalIndexCache, m_includeTime, m_baseTimeOffsets, m_timeIndex, m_useMillisecondResolution); responseIndex += measurement.ParseBinaryImage(buffer, responseIndex, responseLength - responseIndex); // Apply timestamp from frame if not included in transmission if (!measurement.IncludeTime) measurement.Timestamp = timestamp; measurements.Add(measurement); } else if (m_lastMissingCacheWarning + MissingCacheWarningInterval < now) { if (m_lastMissingCacheWarning != 0L) { // Warning message for missing signal index cache OnStatusMessage("WARNING: Signal index cache has not arrived. No compact measurements can be parsed."); } m_lastMissingCacheWarning = now; } } } // Calculate statistics subscribedDevicesLookup = m_subscribedDevicesLookup; statisticsHelper = null; if ((object)subscribedDevicesLookup != null) { IEnumerable<IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement>> deviceGroups = measurements .Where(measurement => subscribedDevicesLookup.TryGetValue(measurement.ID, out statisticsHelper)) .Select(measurement => Tuple.Create(statisticsHelper, measurement)) .ToList() .GroupBy(tuple => tuple.Item1, tuple => tuple.Item2); foreach (IGrouping<DeviceStatisticsHelper<SubscribedDevice>, IMeasurement> deviceGroup in deviceGroups) { statisticsHelper = deviceGroup.Key; foreach (IGrouping<Ticks, IMeasurement> frame in deviceGroup.GroupBy(measurement => measurement.Timestamp)) { // Determine the number of measurements received with valid values int measurementsReceived = frame.Count(measurement => !double.IsNaN(measurement.Value)); IMeasurement statusFlags = null; IMeasurement frequency = null; IMeasurement deltaFrequency = null; // Attempt to update real-time if (!m_useLocalClockAsRealTime && frame.Key > m_realTime) m_realTime = frame.Key; // Search the frame for status flags, frequency, and delta frequency foreach (IMeasurement measurement in frame) { if (measurement.ID == statisticsHelper.Device.StatusFlagsID) statusFlags = measurement; else if (measurement.ID == statisticsHelper.Device.FrequencyID) frequency = measurement; else if (measurement.ID == statisticsHelper.Device.DeltaFrequencyID) deltaFrequency = measurement; } // If we are receiving status flags for this device, // count the data quality, time quality, and device errors if ((object)statusFlags != null) { uint commonStatusFlags = (uint)statusFlags.Value; if ((commonStatusFlags & (uint)Bits.Bit19) > 0) statisticsHelper.Device.DataQualityErrors++; if ((commonStatusFlags & (uint)Bits.Bit18) > 0) statisticsHelper.Device.TimeQualityErrors++; if ((commonStatusFlags & (uint)Bits.Bit16) > 0) statisticsHelper.Device.DeviceErrors++; measurementsReceived--; } // Zero is not a valid value for frequency. // If frequency is zero, invalidate both frequency and delta frequency if ((object)frequency != null && frequency.Value == 0.0D) { if ((object)deltaFrequency != null) measurementsReceived -= 2; else measurementsReceived--; } // Track the number of measurements received statisticsHelper.AddToMeasurementsReceived(measurementsReceived); } } } // Provide new measurements to local concentrator, if defined, otherwise directly expose them to the consumer if ((object)m_localConcentrator != null) m_localConcentrator.SortMeasurements(measurements); else OnNewMeasurements(measurements); // Gather statistics on received data DateTime timeReceived = RealTime; if (!m_useLocalClockAsRealTime && timeReceived.Ticks - m_lastStatisticsHelperUpdate > Ticks.PerSecond) { UpdateStatisticsHelpers(); m_lastStatisticsHelperUpdate = m_realTime; } m_lifetimeMeasurements += measurements.Count; UpdateMeasurementsPerSecond(timeReceived, measurements.Count); for (int x = 0; x < measurements.Count; x++) { long latency = timeReceived.Ticks - (long)measurements[x].Timestamp; // Throw out latencies that exceed one hour as invalid if (Math.Abs(latency) > Time.SecondsPerHour * Ticks.PerSecond) continue; if (m_lifetimeMinimumLatency > latency || m_lifetimeMinimumLatency == 0) m_lifetimeMinimumLatency = latency; if (m_lifetimeMaximumLatency < latency || m_lifetimeMaximumLatency == 0) m_lifetimeMaximumLatency = latency; m_lifetimeTotalLatency += latency; m_lifetimeLatencyMeasurements++; } break; case ServerResponse.BufferBlock: // Buffer block received - wrap as a buffer block measurement and expose back to consumer uint sequenceNumber = BigEndian.ToUInt32(buffer, responseIndex); int cacheIndex = (int)(sequenceNumber - m_expectedBufferBlockSequenceNumber); BufferBlockMeasurement bufferBlockMeasurement; Tuple<Guid, string, uint> measurementKey; ushort signalIndex; // Check if this buffer block has already been processed (e.g., mistaken retransmission due to timeout) if (cacheIndex >= 0 && (cacheIndex >= m_bufferBlockCache.Count || (object)m_bufferBlockCache[cacheIndex] == null)) { // Send confirmation that buffer block is received SendServerCommand(ServerCommand.ConfirmBufferBlock, buffer.BlockCopy(responseIndex, 4)); // Get measurement key from signal index cache signalIndex = BigEndian.ToUInt16(buffer, responseIndex + 4); if (!m_signalIndexCache.Reference.TryGetValue(signalIndex, out measurementKey)) throw new InvalidOperationException("Failed to find associated signal identification for runtime ID " + signalIndex); // Skip the sequence number and signal index when creating the buffer block measurement bufferBlockMeasurement = new BufferBlockMeasurement(buffer, responseIndex + 6, responseLength - 6) { Key = MeasurementKey.LookUpOrCreate(measurementKey.Item1, measurementKey.Item2, measurementKey.Item3) }; // Determine if this is the next buffer block in the sequence if (sequenceNumber == m_expectedBufferBlockSequenceNumber) { List<IMeasurement> bufferBlockMeasurements = new List<IMeasurement>(); int i; // Add the buffer block measurement to the list of measurements to be published bufferBlockMeasurements.Add(bufferBlockMeasurement); m_expectedBufferBlockSequenceNumber++; // Add cached buffer block measurements to the list of measurements to be published for (i = 1; i < m_bufferBlockCache.Count; i++) { if ((object)m_bufferBlockCache[i] == null) break; bufferBlockMeasurements.Add(m_bufferBlockCache[i]); m_expectedBufferBlockSequenceNumber++; } // Remove published measurements from the buffer block queue if (m_bufferBlockCache.Count > 0) m_bufferBlockCache.RemoveRange(0, i); // Publish measurements OnNewMeasurements(bufferBlockMeasurements); } else { // Ensure that the list has at least as many // elements as it needs to cache this measurement for (int i = m_bufferBlockCache.Count; i <= cacheIndex; i++) m_bufferBlockCache.Add(null); // Insert this buffer block into the proper location in the list m_bufferBlockCache[cacheIndex] = bufferBlockMeasurement; } } m_lifetimeMeasurements += 1; UpdateMeasurementsPerSecond(DateTime.UtcNow, 1); break; case ServerResponse.DataStartTime: // Raise data start time event OnDataStartTime(BigEndian.ToInt64(buffer, responseIndex)); break; case ServerResponse.ProcessingComplete: // Raise input processing completed event OnProcessingComplete(InterpretResponseMessage(buffer, responseIndex, responseLength)); break; case ServerResponse.UpdateSignalIndexCache: // Deserialize new signal index cache m_remoteSignalIndexCache = DeserializeSignalIndexCache(buffer.BlockCopy(responseIndex, responseLength)); m_signalIndexCache = new SignalIndexCache(DataSource, m_remoteSignalIndexCache); FixExpectedMeasurementCounts(); break; case ServerResponse.UpdateBaseTimes: // Get active time index m_timeIndex = BigEndian.ToInt32(buffer, responseIndex); responseIndex += 4; // Deserialize new base time offsets m_baseTimeOffsets = new[] { BigEndian.ToInt64(buffer, responseIndex), BigEndian.ToInt64(buffer, responseIndex + 8) }; break; case ServerResponse.UpdateCipherKeys: // Move past active cipher index (not currently used anywhere else) responseIndex++; // Extract remaining response byte[] bytes = buffer.BlockCopy(responseIndex, responseLength - 1); // Decrypt response payload if subscription is authenticated if (m_authenticated) bytes = bytes.Decrypt(m_sharedSecret, CipherStrength.Aes256); // Deserialize new cipher keys keyIVs = new byte[2][][]; keyIVs[EvenKey] = new byte[2][]; keyIVs[OddKey] = new byte[2][]; int index = 0; int bufferLen; // Read even key size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read even key keyIVs[EvenKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read even initialization vector size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read even initialization vector keyIVs[EvenKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[EvenKey][IVIndex], 0, bufferLen); index += bufferLen; // Read odd key size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read odd key keyIVs[OddKey][KeyIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][KeyIndex], 0, bufferLen); index += bufferLen; // Read odd initialization vector size bufferLen = BigEndian.ToInt32(bytes, index); index += 4; // Read odd initialization vector keyIVs[OddKey][IVIndex] = new byte[bufferLen]; Buffer.BlockCopy(bytes, index, keyIVs[OddKey][IVIndex], 0, bufferLen); //index += bufferLen; // Exchange keys m_keyIVs = keyIVs; OnStatusMessage("Successfully established new cipher keys for data packet transmissions."); break; case ServerResponse.Notify: // Skip the 4-byte hash string message = m_encoding.GetString(buffer, responseIndex + 4, responseLength - 4); // Display notification OnStatusMessage("NOTIFICATION: {0}", message); OnNotificationReceived(message); // Send confirmation of receipt of the notification SendServerCommand(ServerCommand.ConfirmNotification, buffer.BlockCopy(responseIndex, 4)); break; case ServerResponse.ConfigurationChanged: OnStatusMessage("Received notification from publisher that configuration has changed."); OnServerConfigurationChanged(); // Initiate meta-data refresh when publisher configuration has changed - we only do this // for automatic connections since API style connections have to manually initiate a // meta-data refresh. API style connection should attach to server configuration changed // event and request meta-data refresh to complete automated cycle. if (m_autoConnect && m_autoSynchronizeMetadata) SendServerCommand(ServerCommand.MetaDataRefresh, m_metadataFilters); break; } } catch (Exception ex) { OnProcessException(new InvalidOperationException("Failed to process publisher response packet due to exception: " + ex.Message, ex)); } } }