/// <summary> /// handle subscription change messages /// </summary> /// <param name="sender"></param> /// <param name="sequenceNumber"></param> /// <param name="notification"></param> private void CallMessageReceiverDelegates(object sender, uint sequenceNumber, SubscriptionNotificationModel notification) { try { var message = new DataSetMessageModel { // TODO: Filter changes on the monitored items contained in the template Notifications = notification.Notifications.ToList(), ServiceMessageContext = notification.ServiceMessageContext, SubscriptionId = notification.SubscriptionId, SequenceNumber = sequenceNumber, ApplicationUri = notification.ApplicationUri, EndpointUrl = notification.EndpointUrl, TimeStamp = DateTime.UtcNow, PublisherId = _outer._publisherId, Writer = _dataSetWriter, WriterGroup = _outer._writerGroup }; lock (_lock) { if (_outer.NumberOfInvokedMessages >= kNumberOfInvokedMessagesResetThreshold) { _outer._logger.Debug("Message counter has been reset to prevent overflow. " + "So far, {NumberOfInvokedMessages} messages has been invoked by message source.", _outer.NumberOfInvokedMessages); _outer.NumberOfInvokedMessages = 0; } _outer.NumberOfInvokedMessages += message.Notifications.Count(); _outer.OnMessage?.Invoke(sender, message); } } catch (Exception ex) { _outer._logger.Debug(ex, "Failed to produce message"); } }
/// <summary> /// Perform event to message encoding /// </summary> /// <param name="message"></param> /// <returns></returns> public IEnumerable <NetworkMessageModel> Encode(DataSetMessageModel message) { foreach (var notification in message.Notifications) { var value = new MonitoredItemMessage { MessageContentMask = (message.Writer?.MessageSettings? .DataSetMessageContentMask).ToMonitoredItemMessageMask( message.Writer?.DataSetFieldContentMask), ApplicationUri = message.ApplicationUri, EndpointUrl = message.EndpointUrl, ExtensionFields = message.Writer?.DataSet?.ExtensionFields, NodeId = notification.NodeId, Value = notification.Value, DisplayName = notification.DisplayName }; using (var writer = new StringWriter()) { using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext) { // TODO: Configure encoding further UseUriEncoding = true }) { value.Encode(encoder); } var encoded = new NetworkMessageModel { Body = Encoding.UTF8.GetBytes(writer.ToString()), ContentEncoding = "utf-8", Timestamp = DateTime.UtcNow, ContentType = ContentMimeType.Json, MessageId = message.SequenceNumber.ToString(), MessageSchema = MessageSchemaTypes.MonitoredItemMessageJson }; yield return(encoded); } } }
private bool GetEntityData(DataSetMessageModel dataSet, out string partitionLocation, out string partitionDelimitor) { partitionLocation = null; partitionDelimitor = kCsvPartitionsDelimiter; var key = GetNormalizedEntityName(dataSet.PublisherId, dataSet.DataSetWriterId, dataSet.MetaDataVersion); var entityDeclaration = Manifest.Entities.Item(key); if (entityDeclaration == null) { return(false); } var partition = entityDeclaration?.DataPartitions[0]; if (partition == null) { return(false); } var csvTrait = partition.ExhibitsTraits.Item("is.partition.format.CSV"); partitionLocation = _cdmCorpus.Storage.CorpusPathToAdapterPath(partition.Location); partitionDelimitor = csvTrait?.Arguments?.FetchValue("delimiter") ?? kCsvPartitionsDelimiter; return(true); }
/// <inheritdoc/> public async Task HandleMessageAsync(DataSetMessageModel message) { foreach (var datapoint in message.Payload) { var arguments = new object[] { new MonitoredItemMessageApiModel() { Value = datapoint.Value.GetType().IsPrimitive == true ? datapoint.Value.Value : datapoint.Value.Value?.ToString(), Status = datapoint.Value.Status, Timestamp = message.Timestamp, DataSetWriterId = message.DataSetWriterId, PublisherId = message.PublisherId, NodeId = datapoint.Key, DisplayName = datapoint.Key, SourceTimestamp = datapoint.Value.SourceTimestamp, ServerTimestamp = datapoint.Value.ServerTimestamp } }; if (!string.IsNullOrEmpty(message.DataSetWriterId)) { // Send to endpoint listeners await _callback.MulticastAsync(message.DataSetWriterId, EventTargets.PublisherSampleTarget, arguments); } } }
/// <inheritdoc/> public async Task HandleMessageAsync(DataSetMessageModel message) { foreach (var datapoint in message.Payload) { var arguments = new object[] { new MonitoredItemMessageApiModel { Timestamp = message.Timestamp, DataSetWriterId = message.DataSetWriterId, PublisherId = message.PublisherId, NodeId = datapoint.Key, DisplayName = datapoint.Key, Value = datapoint.Value?.Value?.Copy(), Status = datapoint.Value?.Status, SourceTimestamp = datapoint.Value?.SourceTimestamp, SourcePicoseconds = datapoint.Value?.SourcePicoseconds, ServerTimestamp = datapoint.Value?.ServerTimestamp, ServerPicoseconds = datapoint.Value?.ServerPicoseconds, DataType = datapoint.Value?.DataType, EndpointId = null // TODO Remove } }; if (!string.IsNullOrEmpty(message.DataSetWriterId)) { // Send to endpoint listeners await _callback.MulticastAsync(message.DataSetWriterId, EventTargets.PublisherSampleTarget, arguments); } } }
/// <summary> /// Perform json encoding /// </summary> /// <param name="message"></param> /// <returns></returns> public IEnumerable <NetworkMessageModel> EncodeAsJson(DataSetMessageModel message) { foreach (var networkMessage in GetNetworkMessages(message.YieldReturn())) { using (var writer = new StringWriter()) { using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext) { UseAdvancedEncoding = true, UseUriEncoding = true, UseReversibleEncoding = false }) { networkMessage.Encode(encoder); } var json = writer.ToString(); var encoded = new NetworkMessageModel { Body = Encoding.UTF8.GetBytes(json), ContentEncoding = "utf-8", Timestamp = DateTime.UtcNow, ContentType = ContentMimeType.Json, MessageId = message.SequenceNumber.ToString(), MessageSchema = MessageSchemaTypes.NetworkMessageJson }; yield return(encoded); } } }
/// <summary> /// Perform event to message encoding /// </summary> /// <param name="message"></param> /// <returns></returns> public IEnumerable <NetworkMessageModel> EncodeAsUadp(DataSetMessageModel message) { foreach (var notification in message.Notifications) { using (var encoder = new BinaryEncoder(message.ServiceMessageContext)) { var value = new MonitoredItemMessage { MessageContentMask = (message.Writer?.MessageSettings? .DataSetMessageContentMask).ToMonitoredItemMessageMask( message.Writer?.DataSetFieldContentMask), ApplicationUri = message.ApplicationUri, SubscriptionId = message.SubscriptionId, EndpointUrl = message.EndpointUrl, ExtensionFields = message.Writer?.DataSet?.ExtensionFields, NodeId = notification.NodeId.ToExpandedNodeId(message.ServiceMessageContext.NamespaceUris), Timestamp = message.TimeStamp ?? DateTime.UtcNow, Value = notification.Value, DisplayName = notification.DisplayName }; value.Encode(encoder); var encoded = new NetworkMessageModel { Body = encoder.CloseAndReturnBuffer(), ContentEncoding = "utf-8", Timestamp = DateTime.UtcNow, ContentType = ContentMimeType.UaBinary, MessageId = message.SequenceNumber.ToString(), MessageSchema = MessageSchemaTypes.MonitoredItemMessageBinary }; yield return(encoded); } } }
/// <summary> /// Message received handler /// </summary> private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args) { _logger.Debug("Message trigger for {Name} received message with sequenceNumber {SequenceNumber}", Name, args.SequenceNumber); if (_diagnosticStart == DateTime.MinValue) { if (_batchTriggerInterval > TimeSpan.Zero) { _batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan); } // reset diagnostic counter, to be aligned with publishing if (_diagnosticInterval > TimeSpan.Zero) { _diagnosticsOutputTimer.Change(_diagnosticInterval, _diagnosticInterval); } _diagnosticStart = DateTime.UtcNow; } if (_sinkBlock.InputCount >= _maxOutgressMessages) { _sinkBlockInputDroppedCount++; } else { _batchDataSetMessageBlock.Post(args); } }
/// <summary> /// Message received handler /// </summary> /// <param name="sender"></param> /// <param name="args"></param> private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args) { if (_diagnosticStart == DateTime.MinValue) { _diagnosticStart = DateTime.UtcNow; } _batchDataSetMessageBlock.Post(args); }
/// <inheritdoc/> public async Task HandleAsync(string deviceId, string moduleId, byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint) { try { var context = new ServiceMessageContext(); var decoder = new BinaryDecoder(new MemoryStream(payload), context); var messages = decoder.ReadBoolean(null) // is Batch? ? decoder.ReadEncodeableArray(null, typeof(NetworkMessage)) as NetworkMessage[] : (decoder.ReadEncodeable(null, typeof(NetworkMessage)) as NetworkMessage).YieldReturn(); foreach (var message in messages) { foreach (var dataSetMessage in message.Messages) { var dataset = new DataSetMessageModel { PublisherId = message.PublisherId, MessageId = message.MessageId, DataSetClassId = message.DataSetClassId, DataSetWriterId = dataSetMessage.DataSetWriterId, SequenceNumber = dataSetMessage.SequenceNumber, Status = StatusCode.LookupSymbolicId(dataSetMessage.Status.Code), MetaDataVersion = $"{dataSetMessage.MetaDataVersion.MajorVersion}" + $".{dataSetMessage.MetaDataVersion.MinorVersion}", Timestamp = dataSetMessage.Timestamp, Payload = new Dictionary <string, DataValueModel>() }; foreach (var datapoint in dataSetMessage.Payload) { var codec = _encoder.Create(context); var type = BuiltInType.Null; dataset.Payload[datapoint.Key] = new DataValueModel { Value = datapoint.Value == null ? null : codec.Encode(datapoint.Value.WrappedValue, out type), DataType = type == BuiltInType.Null ? null : type.ToString(), Status = (datapoint.Value?.StatusCode.Code == StatusCodes.Good) ? null : StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code), SourceTimestamp = (datapoint.Value?.SourceTimestamp == DateTime.MinValue) ? null : datapoint.Value?.SourceTimestamp, SourcePicoseconds = (datapoint.Value?.SourcePicoseconds == 0) ? null : datapoint.Value?.SourcePicoseconds, ServerTimestamp = (datapoint.Value?.ServerTimestamp == DateTime.MinValue) ? null : datapoint.Value?.ServerTimestamp, ServerPicoseconds = (datapoint.Value?.ServerPicoseconds == 0) ? null : datapoint.Value?.ServerPicoseconds }; } await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset))); } } } catch (Exception ex) { _logger.Error(ex, "Subscriber binary network message handling failed - skip"); } }
/// <inheritdoc/> private string GetNormalizedEntityName(DataSetMessageModel dataSet) { if (string.IsNullOrEmpty(dataSet.PublisherId) || string.IsNullOrEmpty(dataSet.DataSetWriterId)) { return(null); } return(GetNormalizedKey($"{dataSet.PublisherId}_{dataSet.DataSetWriterId}" + $"_{dataSet.DataSetClassId}_{dataSet.MetaDataVersion}")); }
/// <inheritdoc/> public Task HandleMessageAsync(DataSetMessageModel message) { var properties = new Dictionary <string, string>() { [CommonProperties.EventSchemaType] = MessageSchemaTypes.NetworkMessageModelJson }; return(_client.SendAsync(Encoding.UTF8.GetBytes( JsonConvertEx.SerializeObject(message)), properties)); }
/// <inheritdoc/> public Task HandleMessageAsync(DataSetMessageModel message) { var properties = new Dictionary <string, string>() { [CommonProperties.EventSchemaType] = Core.MessageSchemaTypes.NetworkMessageModelJson }; return(_client.SendAsync(_serializer.SerializeToBytes(message).ToArray(), properties, message.DataSetWriterId)); }
/// <summary> /// Message received handler /// </summary> /// <param name="sender"></param> /// <param name="args"></param> private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args) { if (_diagnosticStart == DateTime.MinValue) { _diagnosticStart = DateTime.UtcNow; if (_batchTriggerInterval > TimeSpan.Zero) { _batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan); } } _batchDataSetMessageBlock.Post(args); }
/// <inheritdoc/> public Task <IEnumerable <NetworkMessageModel> > EncodeAsync(DataSetMessageModel message) { switch (message.WriterGroup?.MessageType ?? MessageEncoding.Json) { case MessageEncoding.Json: return(Task.FromResult(EncodeAsJson(message))); case MessageEncoding.Uadp: return(Task.FromResult(EncodeAsUadp(message))); default: return(Task.FromException <IEnumerable <NetworkMessageModel> >( new NotSupportedException("Type not supported"))); } }
/// <summary> /// Perform uadp encoding /// </summary> /// <param name="message"></param> /// <returns></returns> public IEnumerable <NetworkMessageModel> EncodeAsUadp(DataSetMessageModel message) { foreach (var networkMessage in GetNetworkMessages(message.YieldReturn())) { using (var encoder = new BinaryEncoder(message.ServiceMessageContext)) { networkMessage.Encode(encoder); var encoded = new NetworkMessageModel { Body = encoder.CloseAndReturnBuffer(), Timestamp = DateTime.UtcNow, ContentType = ContentMimeType.Uadp, MessageId = message.SequenceNumber.ToString(), MessageSchema = MessageSchemaTypes.NetworkMessageUadp }; yield return(encoded); } } }
/// <inheritdoc/> public async Task HandleAsync(string deviceId, string moduleId, byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint) { var json = Encoding.UTF8.GetString(payload); using (var stream = new MemoryStream(payload)) { var context = new ServiceMessageContext(); try { using (var decoder = new JsonDecoderEx(stream, context)) { var networkMessage = decoder.ReadEncodeable(null, typeof(NetworkMessage)) as NetworkMessage; foreach (var dataSetMessage in networkMessage.Messages) { var dataset = new DataSetMessageModel { PublisherId = networkMessage.PublisherId, MessageId = networkMessage.MessageId, DataSetClassId = networkMessage.DataSetClassId, DataSetWriterId = dataSetMessage.DataSetWriterId, SequenceNumber = dataSetMessage.SequenceNumber, Status = StatusCode.LookupSymbolicId(dataSetMessage.Status.Code), MetaDataVersion = $"{dataSetMessage.MetaDataVersion.MajorVersion}" + $".{dataSetMessage.MetaDataVersion.MinorVersion}", Timestamp = dataSetMessage.Timestamp, Payload = new Dictionary <string, DataValueModel>() }; foreach (var datapoint in dataSetMessage.Payload) { dataset.Payload[datapoint.Key] = new DataValueModel() { Value = datapoint.Value?.Value, Status = (datapoint.Value?.StatusCode.Code == StatusCodes.Good) ? null : StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code), SourceTimestamp = (datapoint.Value?.SourceTimestamp == DateTime.MinValue) ? null : (DateTime?)datapoint.Value?.SourceTimestamp, ServerTimestamp = (datapoint.Value?.ServerTimestamp == DateTime.MinValue) ? null : (DateTime?)datapoint.Value?.ServerTimestamp }; } await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset))); } } } catch (Exception ex) { _logger.Error(ex, "Subscriber json network message handling failed - skip"); } } }
/// <inheritdoc/> public async Task HandleAsync(string deviceId, string moduleId, byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint) { var json = Encoding.UTF8.GetString(payload); using (var stream = new MemoryStream(payload)) { var context = new ServiceMessageContext(); try { using (var decoder = new BinaryDecoder(stream, context)) { var networkMessage = decoder.ReadEncodeable(null, typeof(NetworkMessage)) as NetworkMessage; foreach (var message in networkMessage.Messages) { var dataset = new DataSetMessageModel { PublisherId = networkMessage.PublisherId, MessageId = networkMessage.MessageId, DataSetClassId = networkMessage.DataSetClassId, DataSetWriterId = message.DataSetWriterId, SequenceNumber = message.SequenceNumber, Status = StatusCode.LookupSymbolicId(message.Status.Code), MetaDataVersion = $"{message.MetaDataVersion.MajorVersion}.{message.MetaDataVersion.MinorVersion}", Timestamp = message.Timestamp, Payload = new Dictionary <string, DataValueModel>() }; foreach (var datapoint in message.Payload) { dataset.Payload[datapoint.Key] = new DataValueModel() { Value = datapoint.Value?.Value, Status = StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code), TypeId = (datapoint.Value?.WrappedValue.TypeInfo != null) ? TypeInfo.GetSystemType( datapoint.Value.WrappedValue.TypeInfo.BuiltInType, datapoint.Value.WrappedValue.TypeInfo.ValueRank) : null, Timestamp = datapoint.Value?.SourceTimestamp }; } await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset))); } } } catch (Exception ex) { _logger.Error(ex, "Subscriber binary network message handling failed - skip"); } } }
/// <summary> /// Message received handler /// </summary> /// <param name="sender"></param> /// <param name="args"></param> private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args) { if (_diagnosticStart == DateTime.MinValue) { _diagnosticStart = DateTime.UtcNow; if (_batchTriggerInterval > TimeSpan.Zero) { _batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan); } } if (_sinkBlock.InputCount >= _maxEgressMessageQueue) { _sinkBlockInputDroppedCount += (ulong)args.Notifications.Count(); } else { _batchDataSetMessageBlock.Post(args); } }
/// <summary> /// handle subscription change messages /// </summary> /// <param name="sender"></param> /// <param name="sequenceNumber"></param> /// <param name="notification"></param> private void CallMessageReceiverDelegates(object sender, uint sequenceNumber, SubscriptionNotificationModel notification) { try { var message = new DataSetMessageModel { // TODO: Filter changes on the monitored items contained in the template Notifications = notification.Notifications.ToList(), ServiceMessageContext = notification.ServiceMessageContext, SubscriptionId = notification.SubscriptionId, SequenceNumber = sequenceNumber, ApplicationUri = notification.ApplicationUri, EndpointUrl = notification.EndpointUrl, TimeStamp = notification.Timestamp, PublisherId = _outer._publisherId, Writer = _dataSetWriter, WriterGroup = _outer._writerGroup }; lock (_lock) { if (_outer.DataChangesCount >= kNumberOfInvokedMessagesResetThreshold || _outer.ValueChangesCount >= kNumberOfInvokedMessagesResetThreshold) { // reset both _outer._logger.Information("Notifications counter has been reset to prevent overflow. " + "So far, {DataChangesCount} data changes and {ValueChangesCount}" + " value changes were invoked by message source.", _outer.DataChangesCount, _outer.ValueChangesCount); _outer.DataChangesCount = 0; _outer.ValueChangesCount = 0; _outer.FireOnCounterResetEvent(); } _outer.ValueChangesCount += (ulong)message.Notifications.Count(); _outer.DataChangesCount++; _outer.OnMessage?.Invoke(sender, message); } } catch (Exception ex) { _outer._logger.Debug(ex, "Failed to produce message"); } }
/// <summary> /// Perform event to message encoding /// </summary> /// <param name="message"></param> /// <returns></returns> public IEnumerable <NetworkMessageModel> EncodeAsJson(DataSetMessageModel message) { foreach (var notification in message.Notifications) { using (var writer = new StringWriter()) { var value = new MonitoredItemMessage { MessageContentMask = (message.Writer?.MessageSettings? .DataSetMessageContentMask).ToMonitoredItemMessageMask( message.Writer?.DataSetFieldContentMask), ApplicationUri = message.ApplicationUri, EndpointUrl = message.EndpointUrl, ExtensionFields = message.Writer?.DataSet?.ExtensionFields, NodeId = notification.NodeId.ToExpandedNodeId(message.ServiceMessageContext.NamespaceUris), Timestamp = message.TimeStamp ?? DateTime.UtcNow, Value = notification.Value, DisplayName = notification.DisplayName }; using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext) { UseAdvancedEncoding = true, UseUriEncoding = true, UseReversibleEncoding = false }){ value.Encode(encoder); } var encoded = new NetworkMessageModel { Body = Encoding.UTF8.GetBytes(writer.ToString()), ContentEncoding = "utf-8", Timestamp = DateTime.UtcNow, ContentType = ContentMimeType.UaLegacyPublisher, MessageId = message.SequenceNumber.ToString(), MessageSchema = MessageSchemaTypes.MonitoredItemMessageJson }; yield return(encoded); } } }
/// <inheritdoc/> public Task <IEnumerable <NetworkMessageModel> > EncodeAsync(DataSetMessageModel message) { return(Task.FromResult(Encode(message))); }
/// <summary> /// Reads to identifier to show for notification in payload of IoT Hub method /// Prio 1: DataSetFieldId (need to be read from message) /// Prio 2: DisplayName - nothing to do, because notification.Id already contains DisplayName /// Prio 3: ExpandedNodeId /// </summary> /// <param name="notification">Notification, were ID need to be looked up for</param> /// <param name="message">subscription notification message, containing notifications</param> /// <param name="context">service context</param> /// <returns>identifier of payload element</returns> private string GetPayloadIdentifier(MonitoredItemNotificationModel notification, DataSetMessageModel message, ServiceMessageContext context) { if (notification is null) { throw new ArgumentNullException(nameof(notification)); } if (message is null) { throw new ArgumentNullException(nameof(message)); } if (context is null) { throw new ArgumentNullException(nameof(context)); } if (_knownPayloadIdentifiers.TryGetValue(notification.NodeId.ToString(), out var knownPayloadIdentifier)) { if (!string.IsNullOrEmpty(knownPayloadIdentifier)) { return(knownPayloadIdentifier); } } else //do the long running lookup as less as possible { foreach (var dataSetWriter in message.WriterGroup.DataSetWriters) { foreach (var publishedVariableData in dataSetWriter.DataSet.DataSetSource.PublishedVariables.PublishedData) { if ((publishedVariableData.PublishedVariableNodeId == notification.NodeId || publishedVariableData.PublishedVariableNodeId.ToExpandedNodeId(context).AsString(context) == notification.NodeId.ToExpandedNodeId(context.NamespaceUris).AsString(context)) && publishedVariableData.Id != notification.NodeId) { _knownPayloadIdentifiers[notification.NodeId.ToString()] = publishedVariableData.Id; return(publishedVariableData.Id); } else { _knownPayloadIdentifiers[notification.NodeId.ToString()] = string.Empty; } } } } return(!string.IsNullOrEmpty(notification.Id) ? notification.Id : notification.NodeId.ToExpandedNodeId(context.NamespaceUris) .AsString(message.ServiceMessageContext)); }
/// <inheritdoc/> public Task HandleMessageAsync(DataSetMessageModel message) { return(ProcessCdmSampleAsync(message)); }
/// <summary> /// Reads to identifier to show for notification in payload of IoT Hub method /// Prio 1: DataSetFieldId (need to be read from message) /// Prio 2: DisplayName - nothing to do, because notification.Id already contains DisplayName /// Prio 3: ExpandedNodeId /// </summary> /// <param name="notification">Notification, were ID need to be looked up for</param> /// <param name="message">subscription notification message, containing notifications</param> /// <param name="context">service context</param> /// <returns>identifier of payload element</returns> private string GetPayloadIdentifier(MonitoredItemNotificationModel notification, DataSetMessageModel message, ServiceMessageContext context) { if (notification is null) { throw new ArgumentNullException(nameof(notification)); } if (message is null) { throw new ArgumentNullException(nameof(message)); } if (context is null) { throw new ArgumentNullException(nameof(context)); } var notificationNodeId = notification.NodeId.ToString(); var notificationExpandedNodeId = notification.NodeId.ToExpandedNodeId(context.NamespaceUris).AsString(context); if (_knownPayloadIdentifiers.TryGetValue(notificationNodeId, out var knownPayloadIdentifier) && !string.IsNullOrEmpty(knownPayloadIdentifier)) { return(knownPayloadIdentifier); } else { //do the long running lookup as less as possible var dataSetWriter = message.Writer; foreach (var publishedVariableData in dataSetWriter.DataSet.DataSetSource.PublishedVariables.PublishedData) { if (publishedVariableData.PublishedVariableNodeId == notification.NodeId || publishedVariableData.PublishedVariableNodeId.ToExpandedNodeId(context).AsString(context) == notificationExpandedNodeId) { if (publishedVariableData.Id != notification.NodeId) { _knownPayloadIdentifiers[notificationNodeId] = publishedVariableData.Id; return(publishedVariableData.Id); } else { var notificationIdentifier = !string.IsNullOrEmpty(notification.Id) ? notification.Id : notificationExpandedNodeId; _knownPayloadIdentifiers[notificationNodeId] = notificationIdentifier; return(notificationIdentifier); } } } } // Fall back to id of the notification or expanded node id. var knownIdentifier = !string.IsNullOrEmpty(notification.Id) ? notification.Id : notificationExpandedNodeId; _knownPayloadIdentifiers[notification.NodeId.ToString()] = knownIdentifier; return(knownIdentifier); }
private bool CreateEntityData(DataSetMessageModel dataSet, out string partitionLocation, out string partitionDelimitor) { var key = GetNormalizedEntityName(dataSet.PublisherId, dataSet.DataSetWriterId, dataSet.MetaDataVersion); // check if the enetity was aleready added var entity = Manifest.Entities.Item(key); if (entity != null) { return(GetEntityData(dataSet, out partitionLocation, out partitionDelimitor)); } // add a new entity for the Message var newSampleEntity = _cdmCorpus.MakeObject <CdmEntityDefinition>( CdmObjectType.EntityDef, key, false); var info = typeof(DataSetMessageModel).GetProperties(); foreach (var property in info) { if (property.Name != nameof(DataSetMessageModel.Payload)) { // add the attributes required var attribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, property.Name, false); attribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); // if we handle a value, lookup it's type property if (property.Name == "Value" && typeof(DataSetMessageModel). GetProperty("TypeId")?.GetValue(dataSet) is Type typeId) { attribute.DataFormat = DataTypeToCdmDataFormat(typeId); } else { attribute.DataFormat = DataTypeToCdmDataFormat(property.PropertyType); } newSampleEntity.Attributes.Add(attribute); } else { // Parse the message payload foreach (var node in dataSet.Payload.OrderBy(i => i.Key)) { // add the attributes for value, status and timestamp var valueAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_value", false); valueAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); valueAttribute.DataFormat = DataTypeToCdmDataFormat(node.Value.TypeId); newSampleEntity.Attributes.Add(valueAttribute); var typeIdAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_typeId", false); typeIdAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); typeIdAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string)); newSampleEntity.Attributes.Add(typeIdAttribute); var statusAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_status", false); statusAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); statusAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string)); newSampleEntity.Attributes.Add(statusAttribute); var timestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_timestamp", false); timestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); timestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime)); newSampleEntity.Attributes.Add(timestampAttribute); } } } newSampleEntity.DisplayName = kPublisherDataSetEntityName; newSampleEntity.Version = "0.0.1"; newSampleEntity.Description = "Opc Ua Pub/Sub Data Set"; // Create a new document where the new entity's definition will be stored var newSampleEntityDoc = _cdmCorpus.MakeObject <CdmDocumentDefinition>( CdmObjectType.DocumentDef, $"{newSampleEntity.EntityName}.cdm.json", false); newSampleEntityDoc.Imports.Add($"{newSampleEntity.EntityName}.cdm.json"); newSampleEntityDoc.Imports.Add(kFoundationJsonPath); newSampleEntityDoc.Definitions.Add(newSampleEntity); _cdmCorpus.Storage.FetchRootFolder("adls").Documents.Add( newSampleEntityDoc, newSampleEntityDoc.Name); var newSampleEntityDef = Manifest.Entities.Add(newSampleEntity); // Define a partition and add it to the local declaration var newSampleEntityPartition = _cdmCorpus.MakeObject <CdmDataPartitionDefinition>( CdmObjectType.DataPartitionDef, newSampleEntity.EntityName); newSampleEntityDef.DataPartitions.Add(newSampleEntityPartition); newSampleEntityPartition.Location = $"adls:/{newSampleEntity.EntityName}/partition-data.csv"; newSampleEntityPartition.Explanation = "Opc Ua monitored item sample messages storage"; var csvTrait = newSampleEntityPartition.ExhibitsTraits.Add( "is.partition.format.CSV"); csvTrait.Arguments.Add("columnHeaders", "true"); csvTrait.Arguments.Add("delimiter", kCsvPartitionsDelimiter); partitionLocation = _cdmCorpus.Storage.CorpusPathToAdapterPath(newSampleEntityPartition.Location); partitionDelimitor = csvTrait?.Arguments?.FetchValue("delimiter") ?? kCsvPartitionsDelimiter; return(true); }
/// <inheritdoc/> public Task HandleMessageAsync(DataSetMessageModel message) { return(_client.ProcessAsync(message)); }
/// <inheritdoc/> private CdmDataPartitionDefinition GetOrCreateEntityDataPartition(string key, DataSetMessageModel dataSet, out bool persist, bool forceNew = false) { persist = false; if (string.IsNullOrEmpty(key) || dataSet == null) { return(null); } // check if the enetity was aleready added var entityDefinition = Manifest.Entities.Item(key); if (entityDefinition == null) { // add a new entity for the DataSet var newDataSetEntity = _cdmCorpus.MakeObject <CdmEntityDefinition>( CdmObjectType.EntityDef, key, false); var properties = typeof(DataSetMessageModel).GetProperties(); foreach (var property in properties) { if (property.Name != nameof(DataSetMessageModel.Payload)) { // add the attributes required var attribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, property.Name, false); attribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); attribute.DataFormat = DataTypeToCdmDataFormat(property.PropertyType); newDataSetEntity.Attributes.Add(attribute); } else { // Parse the message payload foreach (var node in dataSet.Payload.OrderBy(i => i.Key)) { // add the attributes for value, status and timestamp var valueAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_value", false); valueAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); valueAttribute.DataFormat = DataTypeToCdmDataFormat(node.Value.Value.GetType()); newDataSetEntity.Attributes.Add(valueAttribute); var statusAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_status", false); statusAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); statusAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string)); newDataSetEntity.Attributes.Add(statusAttribute); var sourceTimestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_sourceTimestamp", false); sourceTimestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); sourceTimestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime)); newDataSetEntity.Attributes.Add(sourceTimestampAttribute); var serverTimestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>( CdmObjectType.TypeAttributeDef, $"{node.Key}_serverTimestamp", false); serverTimestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>( CdmObjectType.PurposeRef, "hasA", true); serverTimestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime)); newDataSetEntity.Attributes.Add(serverTimestampAttribute); } } } newDataSetEntity.DisplayName = kPublisherDataSetEntityName; newDataSetEntity.Version = "0.0.1"; newDataSetEntity.Description = "OPC UA PubSub DataSet Entity"; // Create a new document where the new entity's definition will be stored var newEntityDoc = _cdmCorpus.MakeObject <CdmDocumentDefinition>( CdmObjectType.DocumentDef, $"{newDataSetEntity.EntityName}.cdm.json", false); newEntityDoc.Imports.Add($"{newDataSetEntity.EntityName}.cdm.json"); newEntityDoc.Imports.Add(kFoundationJsonPath); newEntityDoc.Definitions.Add(newDataSetEntity); _cdmCorpus.Storage.FetchRootFolder("adls").Documents.Add( newEntityDoc, newEntityDoc.Name); entityDefinition = Manifest.Entities.Add(newDataSetEntity); persist |= true; } var partition = entityDefinition.DataPartitions.Count != 0 ? entityDefinition.DataPartitions.Last() : null; if (forceNew || partition == null) { // Define a partition and add it to the local declaration var newPartition = _cdmCorpus.MakeObject <CdmDataPartitionDefinition>( CdmObjectType.DataPartitionDef, entityDefinition.EntityName); var timestamp = DateTime.UtcNow.ToString( "yyMMddHHmmss", DateTimeFormatInfo.InvariantInfo); newPartition.Location = $"adls:/{entityDefinition.EntityName}/partition-data-{timestamp}.csv"; newPartition.Explanation = "OPC UA PubSub DataSet Partition"; var partitionTrait = newPartition.ExhibitsTraits.Add( "is.partition.format.CSV"); partitionTrait.Arguments.Add("columnHeaders", "true"); partitionTrait.Arguments.Add("delimiter", kCsvPartitionsDelimiter); partition = entityDefinition.DataPartitions.Add(newPartition); persist |= true; } return(partition); }
/// <summary> /// Message received handler /// </summary> /// <param name="sender"></param> /// <param name="args"></param> private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args) { _batchDataSetMessageBlock.Post(args); }
public static IList <DataSetMessageModel> GenerateSampleMessages( uint numOfMessages, MessageEncoding encoding = MessageEncoding.Json ) { var messages = new List <DataSetMessageModel>(); for (uint i = 0; i < numOfMessages; i++) { var suffix = $"-{i}"; var notifications = new List <MonitoredItemNotificationModel>(); for (uint k = 0; k < i + 1; k++) { var notificationSuffix = suffix + $"-{k}"; var monitoredItemNotification = new MonitoredItemNotification { ClientHandle = k, Value = new DataValue(new Variant(k), new StatusCode(0), DateTime.UtcNow), Message = new NotificationMessage() }; var monitoredItem = new MonitoredItem { DisplayName = "DisplayName" + notificationSuffix, StartNodeId = new NodeId("NodeId" + notificationSuffix), AttributeId = k }; var notification = monitoredItemNotification.ToMonitoredItemNotification(monitoredItem); notifications.Add(notification); } var message = new DataSetMessageModel { SequenceNumber = i, PublisherId = "PublisherId" + suffix, Writer = new DataSetWriterModel { DataSet = new PublishedDataSetModel { DataSetSource = new PublishedDataSetSourceModel { PublishedVariables = new PublishedDataItemsModel { PublishedData = new List <PublishedDataSetVariableModel>() } } } }, WriterGroup = new WriterGroupModel { MessageSettings = new WriterGroupMessageSettingsModel { NetworkMessageContentMask = (NetworkMessageContentMask)0xffff }, MessageType = encoding }, TimeStamp = DateTime.UtcNow, ServiceMessageContext = new ServiceMessageContext { }, Notifications = notifications, SubscriptionId = "SubscriptionId" + suffix, EndpointUrl = "EndpointUrl" + suffix, ApplicationUri = "ApplicationUri" + suffix }; messages.Add(message); } return(messages); }