/// <summary>
/// handle subscription change messages
/// </summary>
/// <param name="sender"></param>
/// <param name="sequenceNumber"></param>
/// <param name="notification"></param>
private void CallMessageReceiverDelegates(object sender, uint sequenceNumber,
SubscriptionNotificationModel notification)
{
try {
var message = new DataSetMessageModel {
// TODO: Filter changes on the monitored items contained in the template
Notifications = notification.Notifications.ToList(),
ServiceMessageContext = notification.ServiceMessageContext,
SubscriptionId = notification.SubscriptionId,
SequenceNumber = sequenceNumber,
ApplicationUri = notification.ApplicationUri,
EndpointUrl = notification.EndpointUrl,
TimeStamp = DateTime.UtcNow,
PublisherId = _outer._publisherId,
Writer = _dataSetWriter,
WriterGroup = _outer._writerGroup
};
lock (_lock) {
if (_outer.NumberOfInvokedMessages >= kNumberOfInvokedMessagesResetThreshold)
{
_outer._logger.Debug("Message counter has been reset to prevent overflow. " +
"So far, {NumberOfInvokedMessages} messages has been invoked by message source.",
_outer.NumberOfInvokedMessages);
_outer.NumberOfInvokedMessages = 0;
}
_outer.NumberOfInvokedMessages += message.Notifications.Count();
_outer.OnMessage?.Invoke(sender, message);
}
}
catch (Exception ex) {
_outer._logger.Debug(ex, "Failed to produce message");
}
}
/// <summary>
/// Perform event to message encoding
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
public IEnumerable <NetworkMessageModel> Encode(DataSetMessageModel message)
{
foreach (var notification in message.Notifications)
{
var value = new MonitoredItemMessage {
MessageContentMask = (message.Writer?.MessageSettings?
.DataSetMessageContentMask).ToMonitoredItemMessageMask(
message.Writer?.DataSetFieldContentMask),
ApplicationUri = message.ApplicationUri,
EndpointUrl = message.EndpointUrl,
ExtensionFields = message.Writer?.DataSet?.ExtensionFields,
NodeId = notification.NodeId,
Value = notification.Value,
DisplayName = notification.DisplayName
};
using (var writer = new StringWriter()) {
using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext)
{
// TODO: Configure encoding further
UseUriEncoding = true
}) {
value.Encode(encoder);
}
var encoded = new NetworkMessageModel {
Body = Encoding.UTF8.GetBytes(writer.ToString()),
ContentEncoding = "utf-8",
Timestamp = DateTime.UtcNow,
ContentType = ContentMimeType.Json,
MessageId = message.SequenceNumber.ToString(),
MessageSchema = MessageSchemaTypes.MonitoredItemMessageJson
};
yield return(encoded);
}
}
}
private bool GetEntityData(DataSetMessageModel dataSet,
out string partitionLocation, out string partitionDelimitor)
{
partitionLocation = null;
partitionDelimitor = kCsvPartitionsDelimiter;
var key = GetNormalizedEntityName(dataSet.PublisherId,
dataSet.DataSetWriterId, dataSet.MetaDataVersion);
var entityDeclaration = Manifest.Entities.Item(key);
if (entityDeclaration == null)
{
return(false);
}
var partition = entityDeclaration?.DataPartitions[0];
if (partition == null)
{
return(false);
}
var csvTrait = partition.ExhibitsTraits.Item("is.partition.format.CSV");
partitionLocation = _cdmCorpus.Storage.CorpusPathToAdapterPath(partition.Location);
partitionDelimitor = csvTrait?.Arguments?.FetchValue("delimiter") ?? kCsvPartitionsDelimiter;
return(true);
}
/// <inheritdoc/>
public async Task HandleMessageAsync(DataSetMessageModel message)
{
foreach (var datapoint in message.Payload)
{
var arguments = new object[] {
new MonitoredItemMessageApiModel()
{
Value = datapoint.Value.GetType().IsPrimitive == true
? datapoint.Value.Value : datapoint.Value.Value?.ToString(),
Status = datapoint.Value.Status,
Timestamp = message.Timestamp,
DataSetWriterId = message.DataSetWriterId,
PublisherId = message.PublisherId,
NodeId = datapoint.Key,
DisplayName = datapoint.Key,
SourceTimestamp = datapoint.Value.SourceTimestamp,
ServerTimestamp = datapoint.Value.ServerTimestamp
}
};
if (!string.IsNullOrEmpty(message.DataSetWriterId))
{
// Send to endpoint listeners
await _callback.MulticastAsync(message.DataSetWriterId,
EventTargets.PublisherSampleTarget, arguments);
}
}
}
/// <inheritdoc/>
public async Task HandleMessageAsync(DataSetMessageModel message)
{
foreach (var datapoint in message.Payload)
{
var arguments = new object[] {
new MonitoredItemMessageApiModel {
Timestamp = message.Timestamp,
DataSetWriterId = message.DataSetWriterId,
PublisherId = message.PublisherId,
NodeId = datapoint.Key,
DisplayName = datapoint.Key,
Value = datapoint.Value?.Value?.Copy(),
Status = datapoint.Value?.Status,
SourceTimestamp = datapoint.Value?.SourceTimestamp,
SourcePicoseconds = datapoint.Value?.SourcePicoseconds,
ServerTimestamp = datapoint.Value?.ServerTimestamp,
ServerPicoseconds = datapoint.Value?.ServerPicoseconds,
DataType = datapoint.Value?.DataType,
EndpointId = null // TODO Remove
}
};
if (!string.IsNullOrEmpty(message.DataSetWriterId))
{
// Send to endpoint listeners
await _callback.MulticastAsync(message.DataSetWriterId,
EventTargets.PublisherSampleTarget, arguments);
}
}
}
/// <summary>
/// Perform json encoding
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
public IEnumerable <NetworkMessageModel> EncodeAsJson(DataSetMessageModel message)
{
foreach (var networkMessage in GetNetworkMessages(message.YieldReturn()))
{
using (var writer = new StringWriter()) {
using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext)
{
UseAdvancedEncoding = true,
UseUriEncoding = true,
UseReversibleEncoding = false
}) {
networkMessage.Encode(encoder);
}
var json = writer.ToString();
var encoded = new NetworkMessageModel {
Body = Encoding.UTF8.GetBytes(json),
ContentEncoding = "utf-8",
Timestamp = DateTime.UtcNow,
ContentType = ContentMimeType.Json,
MessageId = message.SequenceNumber.ToString(),
MessageSchema = MessageSchemaTypes.NetworkMessageJson
};
yield return(encoded);
}
}
}
/// <summary>
/// Perform event to message encoding
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
public IEnumerable <NetworkMessageModel> EncodeAsUadp(DataSetMessageModel message)
{
foreach (var notification in message.Notifications)
{
using (var encoder = new BinaryEncoder(message.ServiceMessageContext)) {
var value = new MonitoredItemMessage {
MessageContentMask = (message.Writer?.MessageSettings?
.DataSetMessageContentMask).ToMonitoredItemMessageMask(
message.Writer?.DataSetFieldContentMask),
ApplicationUri = message.ApplicationUri,
SubscriptionId = message.SubscriptionId,
EndpointUrl = message.EndpointUrl,
ExtensionFields = message.Writer?.DataSet?.ExtensionFields,
NodeId = notification.NodeId.ToExpandedNodeId(message.ServiceMessageContext.NamespaceUris),
Timestamp = message.TimeStamp ?? DateTime.UtcNow,
Value = notification.Value,
DisplayName = notification.DisplayName
};
value.Encode(encoder);
var encoded = new NetworkMessageModel {
Body = encoder.CloseAndReturnBuffer(),
ContentEncoding = "utf-8",
Timestamp = DateTime.UtcNow,
ContentType = ContentMimeType.UaBinary,
MessageId = message.SequenceNumber.ToString(),
MessageSchema = MessageSchemaTypes.MonitoredItemMessageBinary
};
yield return(encoded);
}
}
}
/// <summary>
/// Message received handler
/// </summary>
private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args)
{
_logger.Debug("Message trigger for {Name} received message with sequenceNumber {SequenceNumber}",
Name, args.SequenceNumber);
if (_diagnosticStart == DateTime.MinValue)
{
if (_batchTriggerInterval > TimeSpan.Zero)
{
_batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan);
}
// reset diagnostic counter, to be aligned with publishing
if (_diagnosticInterval > TimeSpan.Zero)
{
_diagnosticsOutputTimer.Change(_diagnosticInterval, _diagnosticInterval);
}
_diagnosticStart = DateTime.UtcNow;
}
if (_sinkBlock.InputCount >= _maxOutgressMessages)
{
_sinkBlockInputDroppedCount++;
}
else
{
_batchDataSetMessageBlock.Post(args);
}
}
/// <summary>
/// Message received handler
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args)
{
if (_diagnosticStart == DateTime.MinValue)
{
_diagnosticStart = DateTime.UtcNow;
}
_batchDataSetMessageBlock.Post(args);
}
/// <inheritdoc/>
public async Task HandleAsync(string deviceId, string moduleId,
byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint)
{
try {
var context = new ServiceMessageContext();
var decoder = new BinaryDecoder(new MemoryStream(payload), context);
var messages = decoder.ReadBoolean(null) // is Batch?
? decoder.ReadEncodeableArray(null, typeof(NetworkMessage))
as NetworkMessage[]
: (decoder.ReadEncodeable(null, typeof(NetworkMessage))
as NetworkMessage).YieldReturn();
foreach (var message in messages)
{
foreach (var dataSetMessage in message.Messages)
{
var dataset = new DataSetMessageModel {
PublisherId = message.PublisherId,
MessageId = message.MessageId,
DataSetClassId = message.DataSetClassId,
DataSetWriterId = dataSetMessage.DataSetWriterId,
SequenceNumber = dataSetMessage.SequenceNumber,
Status = StatusCode.LookupSymbolicId(dataSetMessage.Status.Code),
MetaDataVersion = $"{dataSetMessage.MetaDataVersion.MajorVersion}" +
$".{dataSetMessage.MetaDataVersion.MinorVersion}",
Timestamp = dataSetMessage.Timestamp,
Payload = new Dictionary <string, DataValueModel>()
};
foreach (var datapoint in dataSetMessage.Payload)
{
var codec = _encoder.Create(context);
var type = BuiltInType.Null;
dataset.Payload[datapoint.Key] = new DataValueModel {
Value = datapoint.Value == null
? null : codec.Encode(datapoint.Value.WrappedValue, out type),
DataType = type == BuiltInType.Null
? null : type.ToString(),
Status = (datapoint.Value?.StatusCode.Code == StatusCodes.Good)
? null : StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code),
SourceTimestamp = (datapoint.Value?.SourceTimestamp == DateTime.MinValue)
? null : datapoint.Value?.SourceTimestamp,
SourcePicoseconds = (datapoint.Value?.SourcePicoseconds == 0)
? null : datapoint.Value?.SourcePicoseconds,
ServerTimestamp = (datapoint.Value?.ServerTimestamp == DateTime.MinValue)
? null : datapoint.Value?.ServerTimestamp,
ServerPicoseconds = (datapoint.Value?.ServerPicoseconds == 0)
? null : datapoint.Value?.ServerPicoseconds
};
}
await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset)));
}
}
}
catch (Exception ex) {
_logger.Error(ex, "Subscriber binary network message handling failed - skip");
}
}
/// <inheritdoc/>
private string GetNormalizedEntityName(DataSetMessageModel dataSet)
{
if (string.IsNullOrEmpty(dataSet.PublisherId) ||
string.IsNullOrEmpty(dataSet.DataSetWriterId))
{
return(null);
}
return(GetNormalizedKey($"{dataSet.PublisherId}_{dataSet.DataSetWriterId}" +
$"_{dataSet.DataSetClassId}_{dataSet.MetaDataVersion}"));
}
/// <inheritdoc/>
public Task HandleMessageAsync(DataSetMessageModel message)
{
var properties = new Dictionary <string, string>()
{
[CommonProperties.EventSchemaType] =
MessageSchemaTypes.NetworkMessageModelJson
};
return(_client.SendAsync(Encoding.UTF8.GetBytes(
JsonConvertEx.SerializeObject(message)), properties));
}
/// <inheritdoc/>
public Task HandleMessageAsync(DataSetMessageModel message)
{
var properties = new Dictionary <string, string>()
{
[CommonProperties.EventSchemaType] =
Core.MessageSchemaTypes.NetworkMessageModelJson
};
return(_client.SendAsync(_serializer.SerializeToBytes(message).ToArray(),
properties, message.DataSetWriterId));
}
/// <summary>
/// Message received handler
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args)
{
if (_diagnosticStart == DateTime.MinValue)
{
_diagnosticStart = DateTime.UtcNow;
if (_batchTriggerInterval > TimeSpan.Zero)
{
_batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan);
}
}
_batchDataSetMessageBlock.Post(args);
}
/// <inheritdoc/>
public Task <IEnumerable <NetworkMessageModel> > EncodeAsync(DataSetMessageModel message)
{
switch (message.WriterGroup?.MessageType ?? MessageEncoding.Json)
{
case MessageEncoding.Json:
return(Task.FromResult(EncodeAsJson(message)));
case MessageEncoding.Uadp:
return(Task.FromResult(EncodeAsUadp(message)));
default:
return(Task.FromException <IEnumerable <NetworkMessageModel> >(
new NotSupportedException("Type not supported")));
}
}
/// <summary>
/// Perform uadp encoding
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
public IEnumerable <NetworkMessageModel> EncodeAsUadp(DataSetMessageModel message)
{
foreach (var networkMessage in GetNetworkMessages(message.YieldReturn()))
{
using (var encoder = new BinaryEncoder(message.ServiceMessageContext)) {
networkMessage.Encode(encoder);
var encoded = new NetworkMessageModel {
Body = encoder.CloseAndReturnBuffer(),
Timestamp = DateTime.UtcNow,
ContentType = ContentMimeType.Uadp,
MessageId = message.SequenceNumber.ToString(),
MessageSchema = MessageSchemaTypes.NetworkMessageUadp
};
yield return(encoded);
}
}
}
/// <inheritdoc/>
public async Task HandleAsync(string deviceId, string moduleId,
byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint)
{
var json = Encoding.UTF8.GetString(payload);
using (var stream = new MemoryStream(payload)) {
var context = new ServiceMessageContext();
try {
using (var decoder = new JsonDecoderEx(stream, context)) {
var networkMessage = decoder.ReadEncodeable(null, typeof(NetworkMessage)) as NetworkMessage;
foreach (var dataSetMessage in networkMessage.Messages)
{
var dataset = new DataSetMessageModel {
PublisherId = networkMessage.PublisherId,
MessageId = networkMessage.MessageId,
DataSetClassId = networkMessage.DataSetClassId,
DataSetWriterId = dataSetMessage.DataSetWriterId,
SequenceNumber = dataSetMessage.SequenceNumber,
Status = StatusCode.LookupSymbolicId(dataSetMessage.Status.Code),
MetaDataVersion = $"{dataSetMessage.MetaDataVersion.MajorVersion}" +
$".{dataSetMessage.MetaDataVersion.MinorVersion}",
Timestamp = dataSetMessage.Timestamp,
Payload = new Dictionary <string, DataValueModel>()
};
foreach (var datapoint in dataSetMessage.Payload)
{
dataset.Payload[datapoint.Key] = new DataValueModel()
{
Value = datapoint.Value?.Value,
Status = (datapoint.Value?.StatusCode.Code == StatusCodes.Good)
? null : StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code),
SourceTimestamp = (datapoint.Value?.SourceTimestamp == DateTime.MinValue)
? null : (DateTime?)datapoint.Value?.SourceTimestamp,
ServerTimestamp = (datapoint.Value?.ServerTimestamp == DateTime.MinValue)
? null : (DateTime?)datapoint.Value?.ServerTimestamp
};
}
await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset)));
}
}
}
catch (Exception ex) {
_logger.Error(ex, "Subscriber json network message handling failed - skip");
}
}
}
/// <inheritdoc/>
public async Task HandleAsync(string deviceId, string moduleId,
byte[] payload, IDictionary <string, string> properties, Func <Task> checkpoint)
{
var json = Encoding.UTF8.GetString(payload);
using (var stream = new MemoryStream(payload)) {
var context = new ServiceMessageContext();
try {
using (var decoder = new BinaryDecoder(stream, context)) {
var networkMessage = decoder.ReadEncodeable(null, typeof(NetworkMessage)) as NetworkMessage;
foreach (var message in networkMessage.Messages)
{
var dataset = new DataSetMessageModel {
PublisherId = networkMessage.PublisherId,
MessageId = networkMessage.MessageId,
DataSetClassId = networkMessage.DataSetClassId,
DataSetWriterId = message.DataSetWriterId,
SequenceNumber = message.SequenceNumber,
Status = StatusCode.LookupSymbolicId(message.Status.Code),
MetaDataVersion = $"{message.MetaDataVersion.MajorVersion}.{message.MetaDataVersion.MinorVersion}",
Timestamp = message.Timestamp,
Payload = new Dictionary <string, DataValueModel>()
};
foreach (var datapoint in message.Payload)
{
dataset.Payload[datapoint.Key] = new DataValueModel()
{
Value = datapoint.Value?.Value,
Status = StatusCode.LookupSymbolicId(datapoint.Value.StatusCode.Code),
TypeId = (datapoint.Value?.WrappedValue.TypeInfo != null) ?
TypeInfo.GetSystemType(
datapoint.Value.WrappedValue.TypeInfo.BuiltInType,
datapoint.Value.WrappedValue.TypeInfo.ValueRank) : null,
Timestamp = datapoint.Value?.SourceTimestamp
};
}
await Task.WhenAll(_handlers.Select(h => h.HandleMessageAsync(dataset)));
}
}
}
catch (Exception ex) {
_logger.Error(ex, "Subscriber binary network message handling failed - skip");
}
}
}
/// <summary>
/// Message received handler
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args)
{
if (_diagnosticStart == DateTime.MinValue)
{
_diagnosticStart = DateTime.UtcNow;
if (_batchTriggerInterval > TimeSpan.Zero)
{
_batchTriggerIntervalTimer.Change(_batchTriggerInterval, Timeout.InfiniteTimeSpan);
}
}
if (_sinkBlock.InputCount >= _maxEgressMessageQueue)
{
_sinkBlockInputDroppedCount += (ulong)args.Notifications.Count();
}
else
{
_batchDataSetMessageBlock.Post(args);
}
}
/// <summary>
/// handle subscription change messages
/// </summary>
/// <param name="sender"></param>
/// <param name="sequenceNumber"></param>
/// <param name="notification"></param>
private void CallMessageReceiverDelegates(object sender, uint sequenceNumber,
SubscriptionNotificationModel notification)
{
try {
var message = new DataSetMessageModel {
// TODO: Filter changes on the monitored items contained in the template
Notifications = notification.Notifications.ToList(),
ServiceMessageContext = notification.ServiceMessageContext,
SubscriptionId = notification.SubscriptionId,
SequenceNumber = sequenceNumber,
ApplicationUri = notification.ApplicationUri,
EndpointUrl = notification.EndpointUrl,
TimeStamp = notification.Timestamp,
PublisherId = _outer._publisherId,
Writer = _dataSetWriter,
WriterGroup = _outer._writerGroup
};
lock (_lock) {
if (_outer.DataChangesCount >= kNumberOfInvokedMessagesResetThreshold ||
_outer.ValueChangesCount >= kNumberOfInvokedMessagesResetThreshold)
{
// reset both
_outer._logger.Information("Notifications counter has been reset to prevent overflow. " +
"So far, {DataChangesCount} data changes and {ValueChangesCount}" +
" value changes were invoked by message source.",
_outer.DataChangesCount, _outer.ValueChangesCount);
_outer.DataChangesCount = 0;
_outer.ValueChangesCount = 0;
_outer.FireOnCounterResetEvent();
}
_outer.ValueChangesCount += (ulong)message.Notifications.Count();
_outer.DataChangesCount++;
_outer.OnMessage?.Invoke(sender, message);
}
}
catch (Exception ex) {
_outer._logger.Debug(ex, "Failed to produce message");
}
}
/// <summary>
/// Perform event to message encoding
/// </summary>
/// <param name="message"></param>
/// <returns></returns>
public IEnumerable <NetworkMessageModel> EncodeAsJson(DataSetMessageModel message)
{
foreach (var notification in message.Notifications)
{
using (var writer = new StringWriter()) {
var value = new MonitoredItemMessage {
MessageContentMask = (message.Writer?.MessageSettings?
.DataSetMessageContentMask).ToMonitoredItemMessageMask(
message.Writer?.DataSetFieldContentMask),
ApplicationUri = message.ApplicationUri,
EndpointUrl = message.EndpointUrl,
ExtensionFields = message.Writer?.DataSet?.ExtensionFields,
NodeId = notification.NodeId.ToExpandedNodeId(message.ServiceMessageContext.NamespaceUris),
Timestamp = message.TimeStamp ?? DateTime.UtcNow,
Value = notification.Value,
DisplayName = notification.DisplayName
};
using (var encoder = new JsonEncoderEx(writer, message.ServiceMessageContext)
{
UseAdvancedEncoding = true,
UseUriEncoding = true,
UseReversibleEncoding = false
}){
value.Encode(encoder);
}
var encoded = new NetworkMessageModel {
Body = Encoding.UTF8.GetBytes(writer.ToString()),
ContentEncoding = "utf-8",
Timestamp = DateTime.UtcNow,
ContentType = ContentMimeType.UaLegacyPublisher,
MessageId = message.SequenceNumber.ToString(),
MessageSchema = MessageSchemaTypes.MonitoredItemMessageJson
};
yield return(encoded);
}
}
}
/// <inheritdoc/>
public Task <IEnumerable <NetworkMessageModel> > EncodeAsync(DataSetMessageModel message)
{
return(Task.FromResult(Encode(message)));
}
/// <summary>
/// Reads to identifier to show for notification in payload of IoT Hub method
/// Prio 1: DataSetFieldId (need to be read from message)
/// Prio 2: DisplayName - nothing to do, because notification.Id already contains DisplayName
/// Prio 3: ExpandedNodeId
/// </summary>
/// <param name="notification">Notification, were ID need to be looked up for</param>
/// <param name="message">subscription notification message, containing notifications</param>
/// <param name="context">service context</param>
/// <returns>identifier of payload element</returns>
private string GetPayloadIdentifier(MonitoredItemNotificationModel notification, DataSetMessageModel message, ServiceMessageContext context)
{
if (notification is null)
{
throw new ArgumentNullException(nameof(notification));
}
if (message is null)
{
throw new ArgumentNullException(nameof(message));
}
if (context is null)
{
throw new ArgumentNullException(nameof(context));
}
if (_knownPayloadIdentifiers.TryGetValue(notification.NodeId.ToString(), out var knownPayloadIdentifier))
{
if (!string.IsNullOrEmpty(knownPayloadIdentifier))
{
return(knownPayloadIdentifier);
}
}
else //do the long running lookup as less as possible
{
foreach (var dataSetWriter in message.WriterGroup.DataSetWriters)
{
foreach (var publishedVariableData in dataSetWriter.DataSet.DataSetSource.PublishedVariables.PublishedData)
{
if ((publishedVariableData.PublishedVariableNodeId == notification.NodeId ||
publishedVariableData.PublishedVariableNodeId.ToExpandedNodeId(context).AsString(context) == notification.NodeId.ToExpandedNodeId(context.NamespaceUris).AsString(context)) &&
publishedVariableData.Id != notification.NodeId)
{
_knownPayloadIdentifiers[notification.NodeId.ToString()] = publishedVariableData.Id;
return(publishedVariableData.Id);
}
else
{
_knownPayloadIdentifiers[notification.NodeId.ToString()] = string.Empty;
}
}
}
}
return(!string.IsNullOrEmpty(notification.Id)
? notification.Id
: notification.NodeId.ToExpandedNodeId(context.NamespaceUris)
.AsString(message.ServiceMessageContext));
}
/// <inheritdoc/>
public Task HandleMessageAsync(DataSetMessageModel message)
{
return(ProcessCdmSampleAsync(message));
}
/// <summary>
/// Reads to identifier to show for notification in payload of IoT Hub method
/// Prio 1: DataSetFieldId (need to be read from message)
/// Prio 2: DisplayName - nothing to do, because notification.Id already contains DisplayName
/// Prio 3: ExpandedNodeId
/// </summary>
/// <param name="notification">Notification, were ID need to be looked up for</param>
/// <param name="message">subscription notification message, containing notifications</param>
/// <param name="context">service context</param>
/// <returns>identifier of payload element</returns>
private string GetPayloadIdentifier(MonitoredItemNotificationModel notification, DataSetMessageModel message, ServiceMessageContext context)
{
if (notification is null)
{
throw new ArgumentNullException(nameof(notification));
}
if (message is null)
{
throw new ArgumentNullException(nameof(message));
}
if (context is null)
{
throw new ArgumentNullException(nameof(context));
}
var notificationNodeId = notification.NodeId.ToString();
var notificationExpandedNodeId = notification.NodeId.ToExpandedNodeId(context.NamespaceUris).AsString(context);
if (_knownPayloadIdentifiers.TryGetValue(notificationNodeId, out var knownPayloadIdentifier) && !string.IsNullOrEmpty(knownPayloadIdentifier))
{
return(knownPayloadIdentifier);
}
else
{
//do the long running lookup as less as possible
var dataSetWriter = message.Writer;
foreach (var publishedVariableData in dataSetWriter.DataSet.DataSetSource.PublishedVariables.PublishedData)
{
if (publishedVariableData.PublishedVariableNodeId == notification.NodeId ||
publishedVariableData.PublishedVariableNodeId.ToExpandedNodeId(context).AsString(context) == notificationExpandedNodeId)
{
if (publishedVariableData.Id != notification.NodeId)
{
_knownPayloadIdentifiers[notificationNodeId] = publishedVariableData.Id;
return(publishedVariableData.Id);
}
else
{
var notificationIdentifier = !string.IsNullOrEmpty(notification.Id)
? notification.Id
: notificationExpandedNodeId;
_knownPayloadIdentifiers[notificationNodeId] = notificationIdentifier;
return(notificationIdentifier);
}
}
}
}
// Fall back to id of the notification or expanded node id.
var knownIdentifier = !string.IsNullOrEmpty(notification.Id)
? notification.Id
: notificationExpandedNodeId;
_knownPayloadIdentifiers[notification.NodeId.ToString()] = knownIdentifier;
return(knownIdentifier);
}
private bool CreateEntityData(DataSetMessageModel dataSet,
out string partitionLocation, out string partitionDelimitor)
{
var key = GetNormalizedEntityName(dataSet.PublisherId,
dataSet.DataSetWriterId, dataSet.MetaDataVersion);
// check if the enetity was aleready added
var entity = Manifest.Entities.Item(key);
if (entity != null)
{
return(GetEntityData(dataSet, out partitionLocation, out partitionDelimitor));
}
// add a new entity for the Message
var newSampleEntity = _cdmCorpus.MakeObject <CdmEntityDefinition>(
CdmObjectType.EntityDef, key, false);
var info = typeof(DataSetMessageModel).GetProperties();
foreach (var property in info)
{
if (property.Name != nameof(DataSetMessageModel.Payload))
{
// add the attributes required
var attribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, property.Name, false);
attribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
// if we handle a value, lookup it's type property
if (property.Name == "Value" &&
typeof(DataSetMessageModel).
GetProperty("TypeId")?.GetValue(dataSet) is Type typeId)
{
attribute.DataFormat = DataTypeToCdmDataFormat(typeId);
}
else
{
attribute.DataFormat = DataTypeToCdmDataFormat(property.PropertyType);
}
newSampleEntity.Attributes.Add(attribute);
}
else
{
// Parse the message payload
foreach (var node in dataSet.Payload.OrderBy(i => i.Key))
{
// add the attributes for value, status and timestamp
var valueAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_value", false);
valueAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
valueAttribute.DataFormat = DataTypeToCdmDataFormat(node.Value.TypeId);
newSampleEntity.Attributes.Add(valueAttribute);
var typeIdAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_typeId", false);
typeIdAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
typeIdAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string));
newSampleEntity.Attributes.Add(typeIdAttribute);
var statusAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_status", false);
statusAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
statusAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string));
newSampleEntity.Attributes.Add(statusAttribute);
var timestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_timestamp", false);
timestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
timestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime));
newSampleEntity.Attributes.Add(timestampAttribute);
}
}
}
newSampleEntity.DisplayName = kPublisherDataSetEntityName;
newSampleEntity.Version = "0.0.1";
newSampleEntity.Description = "Opc Ua Pub/Sub Data Set";
// Create a new document where the new entity's definition will be stored
var newSampleEntityDoc = _cdmCorpus.MakeObject <CdmDocumentDefinition>(
CdmObjectType.DocumentDef, $"{newSampleEntity.EntityName}.cdm.json", false);
newSampleEntityDoc.Imports.Add($"{newSampleEntity.EntityName}.cdm.json");
newSampleEntityDoc.Imports.Add(kFoundationJsonPath);
newSampleEntityDoc.Definitions.Add(newSampleEntity);
_cdmCorpus.Storage.FetchRootFolder("adls").Documents.Add(
newSampleEntityDoc, newSampleEntityDoc.Name);
var newSampleEntityDef = Manifest.Entities.Add(newSampleEntity);
// Define a partition and add it to the local declaration
var newSampleEntityPartition = _cdmCorpus.MakeObject <CdmDataPartitionDefinition>(
CdmObjectType.DataPartitionDef, newSampleEntity.EntityName);
newSampleEntityDef.DataPartitions.Add(newSampleEntityPartition);
newSampleEntityPartition.Location =
$"adls:/{newSampleEntity.EntityName}/partition-data.csv";
newSampleEntityPartition.Explanation = "Opc Ua monitored item sample messages storage";
var csvTrait = newSampleEntityPartition.ExhibitsTraits.Add(
"is.partition.format.CSV");
csvTrait.Arguments.Add("columnHeaders", "true");
csvTrait.Arguments.Add("delimiter", kCsvPartitionsDelimiter);
partitionLocation = _cdmCorpus.Storage.CorpusPathToAdapterPath(newSampleEntityPartition.Location);
partitionDelimitor = csvTrait?.Arguments?.FetchValue("delimiter") ?? kCsvPartitionsDelimiter;
return(true);
}
/// <inheritdoc/>
public Task HandleMessageAsync(DataSetMessageModel message)
{
return(_client.ProcessAsync(message));
}
/// <inheritdoc/>
private CdmDataPartitionDefinition GetOrCreateEntityDataPartition(string key,
DataSetMessageModel dataSet, out bool persist, bool forceNew = false)
{
persist = false;
if (string.IsNullOrEmpty(key) || dataSet == null)
{
return(null);
}
// check if the enetity was aleready added
var entityDefinition = Manifest.Entities.Item(key);
if (entityDefinition == null)
{
// add a new entity for the DataSet
var newDataSetEntity = _cdmCorpus.MakeObject <CdmEntityDefinition>(
CdmObjectType.EntityDef, key, false);
var properties = typeof(DataSetMessageModel).GetProperties();
foreach (var property in properties)
{
if (property.Name != nameof(DataSetMessageModel.Payload))
{
// add the attributes required
var attribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, property.Name, false);
attribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
attribute.DataFormat = DataTypeToCdmDataFormat(property.PropertyType);
newDataSetEntity.Attributes.Add(attribute);
}
else
{
// Parse the message payload
foreach (var node in dataSet.Payload.OrderBy(i => i.Key))
{
// add the attributes for value, status and timestamp
var valueAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_value", false);
valueAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
valueAttribute.DataFormat = DataTypeToCdmDataFormat(node.Value.Value.GetType());
newDataSetEntity.Attributes.Add(valueAttribute);
var statusAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_status", false);
statusAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
statusAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(string));
newDataSetEntity.Attributes.Add(statusAttribute);
var sourceTimestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_sourceTimestamp", false);
sourceTimestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
sourceTimestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime));
newDataSetEntity.Attributes.Add(sourceTimestampAttribute);
var serverTimestampAttribute = _cdmCorpus.MakeObject <CdmTypeAttributeDefinition>(
CdmObjectType.TypeAttributeDef, $"{node.Key}_serverTimestamp", false);
serverTimestampAttribute.Purpose = _cdmCorpus.MakeRef <CdmPurposeReference>(
CdmObjectType.PurposeRef, "hasA", true);
serverTimestampAttribute.DataFormat = DataTypeToCdmDataFormat(typeof(DateTime));
newDataSetEntity.Attributes.Add(serverTimestampAttribute);
}
}
}
newDataSetEntity.DisplayName = kPublisherDataSetEntityName;
newDataSetEntity.Version = "0.0.1";
newDataSetEntity.Description = "OPC UA PubSub DataSet Entity";
// Create a new document where the new entity's definition will be stored
var newEntityDoc = _cdmCorpus.MakeObject <CdmDocumentDefinition>(
CdmObjectType.DocumentDef, $"{newDataSetEntity.EntityName}.cdm.json", false);
newEntityDoc.Imports.Add($"{newDataSetEntity.EntityName}.cdm.json");
newEntityDoc.Imports.Add(kFoundationJsonPath);
newEntityDoc.Definitions.Add(newDataSetEntity);
_cdmCorpus.Storage.FetchRootFolder("adls").Documents.Add(
newEntityDoc, newEntityDoc.Name);
entityDefinition = Manifest.Entities.Add(newDataSetEntity);
persist |= true;
}
var partition = entityDefinition.DataPartitions.Count != 0
? entityDefinition.DataPartitions.Last() : null;
if (forceNew || partition == null)
{
// Define a partition and add it to the local declaration
var newPartition = _cdmCorpus.MakeObject <CdmDataPartitionDefinition>(
CdmObjectType.DataPartitionDef, entityDefinition.EntityName);
var timestamp = DateTime.UtcNow.ToString(
"yyMMddHHmmss", DateTimeFormatInfo.InvariantInfo);
newPartition.Location =
$"adls:/{entityDefinition.EntityName}/partition-data-{timestamp}.csv";
newPartition.Explanation = "OPC UA PubSub DataSet Partition";
var partitionTrait = newPartition.ExhibitsTraits.Add(
"is.partition.format.CSV");
partitionTrait.Arguments.Add("columnHeaders", "true");
partitionTrait.Arguments.Add("delimiter", kCsvPartitionsDelimiter);
partition = entityDefinition.DataPartitions.Add(newPartition);
persist |= true;
}
return(partition);
}
/// <summary>
/// Message received handler
/// </summary>
/// <param name="sender"></param>
/// <param name="args"></param>
private void MessageTriggerMessageReceived(object sender, DataSetMessageModel args)
{
_batchDataSetMessageBlock.Post(args);
}
public static IList <DataSetMessageModel> GenerateSampleMessages(
uint numOfMessages,
MessageEncoding encoding = MessageEncoding.Json
)
{
var messages = new List <DataSetMessageModel>();
for (uint i = 0; i < numOfMessages; i++)
{
var suffix = $"-{i}";
var notifications = new List <MonitoredItemNotificationModel>();
for (uint k = 0; k < i + 1; k++)
{
var notificationSuffix = suffix + $"-{k}";
var monitoredItemNotification = new MonitoredItemNotification {
ClientHandle = k,
Value = new DataValue(new Variant(k), new StatusCode(0), DateTime.UtcNow),
Message = new NotificationMessage()
};
var monitoredItem = new MonitoredItem {
DisplayName = "DisplayName" + notificationSuffix,
StartNodeId = new NodeId("NodeId" + notificationSuffix),
AttributeId = k
};
var notification = monitoredItemNotification.ToMonitoredItemNotification(monitoredItem);
notifications.Add(notification);
}
var message = new DataSetMessageModel {
SequenceNumber = i,
PublisherId = "PublisherId" + suffix,
Writer = new DataSetWriterModel {
DataSet = new PublishedDataSetModel {
DataSetSource = new PublishedDataSetSourceModel {
PublishedVariables = new PublishedDataItemsModel {
PublishedData = new List <PublishedDataSetVariableModel>()
}
}
}
},
WriterGroup = new WriterGroupModel {
MessageSettings = new WriterGroupMessageSettingsModel {
NetworkMessageContentMask = (NetworkMessageContentMask)0xffff
},
MessageType = encoding
},
TimeStamp = DateTime.UtcNow,
ServiceMessageContext = new ServiceMessageContext {
},
Notifications = notifications,
SubscriptionId = "SubscriptionId" + suffix,
EndpointUrl = "EndpointUrl" + suffix,
ApplicationUri = "ApplicationUri" + suffix
};
messages.Add(message);
}
return(messages);
}
