Data gaps will be recovered using an unsynchronized temporal subscription.
This class expects that source historian that feeds temporal subscription will recover data in time-sorted order.
/// <summary> /// Initializes <see cref="DataSubscriber"/>. /// </summary> public override void Initialize() { base.Initialize(); Dictionary<string, string> settings = Settings; string setting; OperationalModes operationalModes; CompressionModes compressionModes; int metadataSynchronizationTimeout; double interval; int bufferSize; // Setup connection to data publishing server with or without authentication required if (settings.TryGetValue("requireAuthentication", out setting)) RequireAuthentication = setting.ParseBoolean(); // See if user has opted for different operational modes if (settings.TryGetValue("operationalModes", out setting) && Enum.TryParse(setting, true, out operationalModes)) OperationalModes = operationalModes; // Set the security mode if explicitly defined if (!settings.TryGetValue("securityMode", out setting) || !Enum.TryParse(setting, true, out m_securityMode)) m_securityMode = SecurityMode.None; // Apply gateway compression mode to operational mode flags if (settings.TryGetValue("compressionModes", out setting) && Enum.TryParse(setting, true, out compressionModes)) CompressionModes = compressionModes; if (settings.TryGetValue("useZeroMQChannel", out setting)) m_useZeroMQChannel = setting.ParseBoolean(); // TODO: Remove this exception when CURVE is enabled in GSF ZeroMQ library if (m_useZeroMQChannel && m_securityMode == SecurityMode.TLS) throw new ArgumentException("CURVE security settings are not yet available for GSF ZeroMQ client channel."); // Settings specific to Gateway security if (m_securityMode == SecurityMode.Gateway) { if (!settings.TryGetValue("sharedSecret", out m_sharedSecret) || string.IsNullOrWhiteSpace(m_sharedSecret)) throw new ArgumentException("The \"sharedSecret\" setting must be defined when using Gateway security mode."); if (!settings.TryGetValue("authenticationID", out m_authenticationID) || string.IsNullOrWhiteSpace(m_authenticationID)) throw new ArgumentException("The \"authenticationID\" setting must be defined when using Gateway security mode."); } // Settings specific to Transport Layer Security if (m_securityMode == SecurityMode.TLS) { if (!settings.TryGetValue("localCertificate", out m_localCertificate) || !File.Exists(m_localCertificate)) m_localCertificate = GetLocalCertificate(); if (!settings.TryGetValue("remoteCertificate", out m_remoteCertificate) || !RemoteCertificateExists()) throw new ArgumentException("The \"remoteCertificate\" setting must be defined and certificate file must exist when using TLS security mode."); if (!settings.TryGetValue("validPolicyErrors", out setting) || !Enum.TryParse(setting, out m_validPolicyErrors)) m_validPolicyErrors = SslPolicyErrors.None; if (!settings.TryGetValue("validChainFlags", out setting) || !Enum.TryParse(setting, out m_validChainFlags)) m_validChainFlags = X509ChainStatusFlags.NoError; if (settings.TryGetValue("checkCertificateRevocation", out setting) && !string.IsNullOrWhiteSpace(setting)) m_checkCertificateRevocation = setting.ParseBoolean(); else m_checkCertificateRevocation = true; } // Check if measurements for this connection should be marked as "internal" - i.e., owned and allowed for proxy if (settings.TryGetValue("internal", out setting)) m_internal = setting.ParseBoolean(); // Check if user has explicitly defined the ReceiveInternalMetadata flag if (settings.TryGetValue("receiveInternalMetadata", out setting)) ReceiveInternalMetadata = setting.ParseBoolean(); // Check if user has explicitly defined the ReceiveExternalMetadata flag if (settings.TryGetValue("receiveExternalMetadata", out setting)) ReceiveExternalMetadata = setting.ParseBoolean(); // Check if user has defined a meta-data synchronization timeout if (settings.TryGetValue("metadataSynchronizationTimeout", out setting) && int.TryParse(setting, out metadataSynchronizationTimeout)) m_metadataSynchronizationTimeout = metadataSynchronizationTimeout; // Check if user has defined a flag for using a transaction during meta-data synchronization if (settings.TryGetValue("useTransactionForMetadata", out setting)) m_useTransactionForMetadata = setting.ParseBoolean(); // Check if user wants to request that publisher use millisecond resolution to conserve bandwidth if (settings.TryGetValue("useMillisecondResolution", out setting)) m_useMillisecondResolution = setting.ParseBoolean(); // Check if user wants to request that publisher remove NaN from the data stream to conserve bandwidth if (settings.TryGetValue("requestNaNValueFilter", out setting)) m_requestNaNValueFilter = setting.ParseBoolean(); // Check if user has defined any meta-data filter expressions if (settings.TryGetValue("metadataFilters", out setting)) m_metadataFilters = setting; // Define auto connect setting if (settings.TryGetValue("autoConnect", out setting)) { m_autoConnect = setting.ParseBoolean(); if (m_autoConnect) m_autoSynchronizeMetadata = true; } // Define the maximum allowed exceptions before resetting the connection if (settings.TryGetValue("allowedParsingExceptions", out setting)) m_allowedParsingExceptions = int.Parse(setting); // Define the window of time over which parsing exceptions are tolerated if (settings.TryGetValue("parsingExceptionWindow", out setting)) m_parsingExceptionWindow = Ticks.FromSeconds(double.Parse(setting)); // Check if synchronize meta-data is explicitly enabled or disabled if (settings.TryGetValue("synchronizeMetadata", out setting)) m_autoSynchronizeMetadata = setting.ParseBoolean(); // Define data loss interval if (settings.TryGetValue("dataLossInterval", out setting) && double.TryParse(setting, out interval)) DataLossInterval = interval; // Define buffer size if (!settings.TryGetValue("bufferSize", out setting) || !int.TryParse(setting, out bufferSize)) bufferSize = ClientBase.DefaultReceiveBufferSize; if (settings.TryGetValue("useLocalClockAsRealTime", out setting)) m_useLocalClockAsRealTime = setting.ParseBoolean(); if (m_autoConnect) { // Connect to local events when automatically engaging connection cycle ConnectionAuthenticated += DataSubscriber_ConnectionAuthenticated; MetaDataReceived += DataSubscriber_MetaDataReceived; // Update output measurements to include "subscribed" points UpdateOutputMeasurements(true); } else if (m_autoSynchronizeMetadata) { // Output measurements do not include "subscribed" points, // but should still be filtered if applicable TryFilterOutputMeasurements(); } if (m_securityMode != SecurityMode.TLS) { if (m_useZeroMQChannel) { // Create a new ZeroMQ Dealer ZeroMQClient commandChannel = new ZeroMQClient(); // Initialize default settings commandChannel.PersistSettings = false; commandChannel.MaxConnectionAttempts = 1; commandChannel.ReceiveBufferSize = bufferSize; commandChannel.SendBufferSize = bufferSize; // Assign command channel client reference and attach to needed events CommandChannel = commandChannel; } else { // Create a new TCP client TcpClient commandChannel = new TcpClient(); // Initialize default settings commandChannel.PayloadAware = true; commandChannel.PersistSettings = false; commandChannel.MaxConnectionAttempts = 1; commandChannel.ReceiveBufferSize = bufferSize; commandChannel.SendBufferSize = bufferSize; // Assign command channel client reference and attach to needed events CommandChannel = commandChannel; } } else { if (m_useZeroMQChannel) { // Create a new ZeroMQ Dealer with CURVE security enabled ZeroMQClient commandChannel = new ZeroMQClient(); // Initialize default settings commandChannel.PersistSettings = false; commandChannel.MaxConnectionAttempts = 1; commandChannel.ReceiveBufferSize = bufferSize; commandChannel.SendBufferSize = bufferSize; // TODO: Parse certificate and pass keys to ZeroMQClient for CURVE security // Assign command channel client reference and attach to needed events CommandChannel = commandChannel; } else { // Create a new TLS client and certificate checker TlsClient commandChannel = new TlsClient(); SimpleCertificateChecker certificateChecker = new SimpleCertificateChecker(); // Set up certificate checker certificateChecker.TrustedCertificates.Add(new X509Certificate2(FilePath.GetAbsolutePath(m_remoteCertificate))); certificateChecker.ValidPolicyErrors = m_validPolicyErrors; certificateChecker.ValidChainFlags = m_validChainFlags; // Initialize default settings commandChannel.PayloadAware = true; commandChannel.PersistSettings = false; commandChannel.MaxConnectionAttempts = 1; commandChannel.CertificateFile = FilePath.GetAbsolutePath(m_localCertificate); commandChannel.CheckCertificateRevocation = m_checkCertificateRevocation; commandChannel.CertificateChecker = certificateChecker; commandChannel.ReceiveBufferSize = bufferSize; commandChannel.SendBufferSize = bufferSize; // Assign command channel client reference and attach to needed events CommandChannel = commandChannel; } } // Get proper connection string - either from specified command channel or from base connection string if (settings.TryGetValue("commandChannel", out setting)) m_commandChannel.ConnectionString = setting; else m_commandChannel.ConnectionString = ConnectionString; // Get logging path, if any has been defined if (settings.TryGetValue("loggingPath", out setting)) { setting = FilePath.GetDirectoryName(FilePath.GetAbsolutePath(setting)); if (Directory.Exists(setting)) m_loggingPath = setting; else OnStatusMessage("WARNING: Logging path \"{0}\" not found, defaulting to \"{1}\"...", setting, FilePath.GetAbsolutePath("")); } // Initialize data gap recovery processing, if requested if (settings.TryGetValue("dataGapRecovery", out setting)) { // Make sure setting exists to allow user to by-pass phasor data source validation at startup ConfigurationFile configFile = ConfigurationFile.Current; CategorizedSettingsElementCollection systemSettings = configFile.Settings["systemSettings"]; CategorizedSettingsElement dataGapRecoveryEnabledSetting = systemSettings["DataGapRecoveryEnabled"]; // See if this node should process phasor source validation if ((object)dataGapRecoveryEnabledSetting == null || dataGapRecoveryEnabledSetting.ValueAsBoolean()) { // Example connection string for data gap recovery: // dataGapRecovery={enabled=true; recoveryStartDelay=10.0; minimumRecoverySpan=0.0; maximumRecoverySpan=3600.0} Dictionary<string, string> dataGapSettings = setting.ParseKeyValuePairs(); if (dataGapSettings.TryGetValue("enabled", out setting) && setting.ParseBoolean()) { // Remove dataGapRecovery connection setting from command channel connection string, if defined there. // This will prevent any recursive data gap recovery operations from being established: Dictionary<string, string> connectionSettings = m_commandChannel.ConnectionString.ParseKeyValuePairs(); connectionSettings.Remove("dataGapRecovery"); connectionSettings.Remove("autoConnect"); connectionSettings.Remove("synchronizeMetadata"); connectionSettings.Remove("outputMeasurements"); // Note that the data gap recoverer will connect on the same command channel port as // the real-time subscriber (TCP only) m_dataGapRecoveryEnabled = true; m_dataGapRecoverer = new DataGapRecoverer(); m_dataGapRecoverer.SourceConnectionName = Name; m_dataGapRecoverer.DataSource = DataSource; m_dataGapRecoverer.ConnectionString = string.Join("; ", $"autoConnect=false; synchronizeMetadata=false{(string.IsNullOrWhiteSpace(m_loggingPath) ? "" : "; loggingPath=" + m_loggingPath)}", dataGapSettings.JoinKeyValuePairs(), connectionSettings.JoinKeyValuePairs()); m_dataGapRecoverer.FilterExpression = this.OutputMeasurementKeys().Select(key => key.SignalID.ToString()).ToDelimitedString(';'); m_dataGapRecoverer.RecoveredMeasurements += m_dataGapRecoverer_RecoveredMeasurements; m_dataGapRecoverer.StatusMessage += m_dataGapRecoverer_StatusMessage; m_dataGapRecoverer.ProcessException += m_dataGapRecoverer_ProcessException; m_dataGapRecoverer.Initialize(); } else { m_dataGapRecoveryEnabled = false; } } } else { m_dataGapRecoveryEnabled = false; } // Register subscriber with the statistics engine StatisticsEngine.Register(this, "Subscriber", "SUB"); StatisticsEngine.Calculated += (sender, args) => ResetMeasurementsPerSecondCounters(); Initialized = true; }
/// <summary> /// Releases the unmanaged resources used by the <see cref="DataSubscriber"/> object and optionally releases the managed resources. /// </summary> /// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param> protected override void Dispose(bool disposing) { if (!m_disposed) { try { if (disposing) { DataLossInterval = 0.0D; CommandChannel = null; DataChannel = null; DisposeLocalConcentrator(); if ((object)m_dataGapRecoverer != null) { m_dataGapRecoverer.RecoveredMeasurements -= m_dataGapRecoverer_RecoveredMeasurements; m_dataGapRecoverer.StatusMessage -= m_dataGapRecoverer_StatusMessage; m_dataGapRecoverer.ProcessException -= m_dataGapRecoverer_ProcessException; m_dataGapRecoverer.Dispose(); m_dataGapRecoverer = null; } if ((object)m_runTimeLog != null) { m_runTimeLog.ProcessException -= m_runTimeLog_ProcessException; m_runTimeLog.Dispose(); m_runTimeLog = null; } if ((object)m_subscribedDevicesTimer != null) { m_subscribedDevicesTimer.Elapsed -= SubscribedDevicesTimer_Elapsed; m_subscribedDevicesTimer.Dispose(); m_subscribedDevicesTimer = null; } } } finally { m_disposed = true; // Prevent duplicate dispose. base.Dispose(disposing); // Call base class Dispose(). } } }