protected PipelineExecutionMessage GetMessage()
{
var msg = new PipelineExecutionMessage()
{
Id = "messageid",
ErrorMessages = new List <Logging.Error>(),
CurrentInstruction = new PipelineExecutionInstruction()
{
Name = "First Step",
QueueId = "MODULE1",
Type = "Process"
},
Device = new DeviceManagement.Core.Models.Device()
{
DeviceRepository = EntityHeader.Create("id", "text")
},
Instructions = new List <PipelineExecutionInstruction>(),
Status = StatusTypes.PendingExecution
};
msg.Instructions.Add(msg.CurrentInstruction);
return(msg);
}
protected void Execute(PipelineExecutionMessage pem)
{
Task.Run(async() =>
{
await ExecuteAsync(pem);
});
}
private async Task UpdateDevice(PipelineExecutionMessage message)
{
message.Device.LastContact = message.CreationTimeStamp;
message.Device.Status = EntityHeader <DeviceStates> .Create(DeviceStates.Ready);
await PEMBus.DeviceStorage.UpdateDeviceAsync(message.Device);
var notification = new Notification()
{
Payload = JsonConvert.SerializeObject(message.Device),
Channel = EntityHeader <Channels> .Create(Channels.Device),
ChannelId = message.Device.Id,
PayloadType = typeof(Device).ToString(),
DateStamp = DateTime.UtcNow.ToJSONString(),
MessageId = Guid.NewGuid().ToId(),
Text = "Device Updated",
Title = "Device Updated"
};
await PEMBus.NotificationPublisher.PublishAsync(Targets.WebSocket, notification);
notification = new Notification()
{
Payload = JsonConvert.SerializeObject(message.Device),
Channel = EntityHeader <Channels> .Create(Channels.DeviceRepository),
ChannelId = message.Device.DeviceRepository.Id,
PayloadType = typeof(Device).ToString(),
DateStamp = DateTime.UtcNow.ToJSONString(),
MessageId = Guid.NewGuid().ToId(),
Text = "Device Updated",
Title = "Device Updated"
};
await PEMBus.NotificationPublisher.PublishAsync(Targets.WebSocket, notification);
}
public async Task ShouldApplySensorValues()
{
var pem = new PipelineExecutionMessage()
{
Device = _device,
Envelope = new MessageEnvelope()
{
Topic = $"nuviot/srvr/dvcsrvc/{DEVICE_ID}/iovalues",
},
TextPayload = "1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8,1.9,2.8,3.7,4.6,5.5,6.4,7.3,8.8"
};
await _listener.ProcessAsync(pem);
Assert.AreEqual("4.6", _device.SensorCollection.Where(sns => sns.PortIndex == 3 && sns.Technology.Value == SensorTechnology.ADC).First().Value);
Assert.AreEqual("6.4", _device.SensorCollection.Where(sns => sns.PortIndex == 5 && sns.Technology.Value == SensorTechnology.ADC).First().Value);
Assert.AreEqual("2.2", _device.SensorCollection.Where(sns => sns.PortIndex == 1 && sns.Technology.Value == SensorTechnology.IO).First().Value);
Assert.AreEqual("3.3", _device.SensorCollection.Where(sns => sns.PortIndex == 2 && sns.Technology.Value == SensorTechnology.IO).First().Value);
_deviceStorage.Verify(ds => ds.UpdateDeviceAsync(It.Is <Device>(dvc => dvc.DeviceId == DEVICE_ID)), Times.Once);
_notificationPublisher.Verify(np => np.PublishAsync(
It.IsAny <Targets>(),
It.Is <Notification>(not => not.Channel.Value == Channels.Device && not.ChannelId == _device.Id),
It.IsAny <NotificationVerbosity>()), Times.Once);
}
public async Task <InvokeResult> AddBinaryMessageAsync(byte[] buffer, DateTime startTimeStamp, String deviceId = "", String topic = "")
{
try
{
var message = new PipelineExecutionMessage()
{
PayloadType = MessagePayloadTypes.Binary,
BinaryPayload = buffer,
CreationTimeStamp = startTimeStamp.ToJSONString()
};
Metrics.MessagesProcessed++;
if (buffer != null)
{
message.PayloadLength = buffer.Length;
}
Metrics.BytesProcessed = message.PayloadLength + (String.IsNullOrEmpty(topic) ? 0 : topic.Length);
message.Envelope.DeviceId = deviceId;
message.Envelope.Topic = topic;
var listenerInstruction = new PipelineExecutionInstruction()
{
Name = _listenerConfiguration.Name,
Type = GetType().Name,
QueueId = "N/A",
StartDateStamp = startTimeStamp.ToJSONString(),
ProcessByHostId = PEMBus.Instance.PrimaryHost.Id,
ExecutionTimeMS = (DateTime.UtcNow - startTimeStamp).TotalMilliseconds,
};
message.Instructions.Add(listenerInstruction);
var planner = PEMBus.Instance.Solution.Value.Planner.Value;
var plannerInstruction = new PipelineExecutionInstruction()
{
Name = "Planner",
Type = "Planner",
QueueId = "N/A",
};
message.CurrentInstruction = plannerInstruction;
message.Instructions.Add(plannerInstruction);
var plannerQueue = PEMBus.Queues.Where(queue => queue.ForModuleType == PipelineModuleType.Planner).FirstOrDefault();
await plannerQueue.EnqueueAsync(message);
return(InvokeResult.Success);
}
catch (Exception ex)
{
PEMBus.InstanceLogger.AddException("ListenerModule_AddBinaryMessageAsync", ex);
return(InvokeResult.FromException("ListenerModule_AddBinaryMessageAsync", ex));
}
}
protected async Task ProcessMessageAsync(PipelineExecutionMessage pem)
{
await _pipelineModule.PorcessMessageForTestAsync(pem);
// await _listenerQueue.EnqueueAsync(pem);
// Give it time to process on a different thread
// await Task.Delay(500);
// await _pipelineModule.StopAsync();
}
public override Task <ProcessResult> ProcessAsync(PipelineExecutionMessage message)
{
if (ProcessHandler == null)
{
return(Task.FromResult(ResultToReturn));
}
else
{
return(ProcessHandler(message));
}
}
public InvokeResult Parse(PipelineExecutionMessage pem, DeviceMessageDefinition definition)
{
foreach (var kvp in KVPs)
{
pem.Envelope.Values.Add(kvp.Key, new MessageValue()
{
Value = kvp.Value
});
}
return(InvokeResult.Success);
}
private async Task UpdateDevice(PipelineExecutionMessage message)
{
message.Device.LastContact = message.CreationTimeStamp;
message.Device.Status = EntityHeader <DeviceStates> .Create(DeviceStates.Ready);
await PEMBus.DeviceStorage.UpdateDeviceAsync(message.Device);
var json = JsonConvert.SerializeObject(Models.DeviceForNotification.FromDevice(message.Device), _camelCaseSettings);
var notification = new Notification()
{
Payload = json,
Channel = EntityHeader <Channels> .Create(Channels.Device),
ChannelId = message.Device.Id,
PayloadType = "Device",
DateStamp = DateTime.UtcNow.ToJSONString(),
MessageId = Guid.NewGuid().ToId(),
Text = "Device Updated",
Title = "Device Updated"
};
await PEMBus.NotificationPublisher.PublishAsync(Targets.WebSocket, notification);
notification = new Notification()
{
Payload = json,
Channel = EntityHeader <Channels> .Create(Channels.DeviceRepository),
ChannelId = message.Device.DeviceRepository.Id,
PayloadType = "Device",
DateStamp = DateTime.UtcNow.ToJSONString(),
MessageId = Guid.NewGuid().ToId(),
Text = "Device Updated",
Title = "Device Updated"
};
await PEMBus.NotificationPublisher.PublishAsync(Targets.WebSocket, notification);
foreach (var group in message.Device.DeviceGroups)
{
notification = new Notification()
{
Payload = json,
Channel = EntityHeader <Channels> .Create(Channels.DeviceGroup),
ChannelId = group.Id,
PayloadType = "Device",
DateStamp = DateTime.UtcNow.ToJSONString(),
MessageId = Guid.NewGuid().ToId(),
Text = "Device Updated",
Title = "Device Updated"
};
await PEMBus.NotificationPublisher.PublishAsync(Targets.WebSocket, notification);
}
}
public void PEM_Serialization_BinaryData()
{
/* Ensure if we stuff binary data into BinaryPayload field it will be serialzied and deserialized propertly */
var container = new PipelineExecutionMessage();
container.BinaryPayload = new byte[] { 0x34, 0x03, 0xAC, 0xDD };
var json = JsonConvert.SerializeObject(container);
var data = JsonConvert.DeserializeObject <PipelineExecutionMessage>(json);
Assert.IsNotNull(data.BinaryPayload);
Assert.AreEqual(4, data.BinaryPayload.Length);
Assert.AreEqual(0x34, data.BinaryPayload[0]);
Assert.AreEqual(0x03, data.BinaryPayload[1]);
Assert.AreEqual(0xAC, data.BinaryPayload[2]);
Assert.AreEqual(0xDD, data.BinaryPayload[3]);
}
private Task FinalizeMessage(PipelineExecutionMessage message)
{
return(Task.FromResult(default(object)));
}
public Task PorcessMessageForTestAsync(PipelineExecutionMessage pem)
{
return(base.ExecuteAsync(pem));
}
protected async Task ExecuteAsync(PipelineExecutionMessage message)
{
Metrics.ActiveCount++;
var sw = new Stopwatch();
try
{
sw.Start();
message.CurrentInstruction.StartDateStamp = DateTime.UtcNow.ToJSONString();
var result = await ProcessAsync(message);
sw.Stop();
message.CurrentInstruction.ExecutionTimeMS = Math.Round(sw.Elapsed.TotalMilliseconds, 3);;
message.ExecutionTimeMS += message.CurrentInstruction.ExecutionTimeMS;
message.CurrentInstruction.ProcessByHostId = ModuleHost.Id;
Metrics.BytesProcessed += message.PayloadLength;
message.ErrorMessages.AddRange(result.ErrorMessages);
message.InfoMessages.AddRange(result.InfoMessages);
message.WarningMessages.AddRange(result.WarningMessages);
if (result.Success)
{
var instructionIndex = message.Instructions.IndexOf(message.CurrentInstruction);
instructionIndex++;
if (instructionIndex == message.Instructions.Count) /* We are done processing the pipe line */
{
message.Status = message.WarningMessages.Count > 0 ? StatusTypes.CompletedWithWarnings : StatusTypes.Completed;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
//For now since we are working on the same instance we don't have to write this to external storage while enqueing, will need to when we run on different nodes
await UpdateDevice(message);
await PEMBus.PEMStorage.AddMessageAsync(message);
}
else
{
if (instructionIndex > message.Instructions.Count) /* Somehow we overrun the index for the pipeline steps */
{
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogError(Resources.ErrorCodes.PipelineEnqueing.InvalidMessageIndex, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"));
message.ErrorMessages.Add(Resources.ErrorCodes.PipelineEnqueing.InvalidMessageIndex.ToError());
message.Status = StatusTypes.Failed;
message.ErrorReason = ErrorReason.UnexepctedError;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
message.CurrentInstruction = null;
Metrics.ErrorCount++;
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
}
else
{
message.CurrentInstruction = message.Instructions[instructionIndex];
message.CurrentInstruction.Enqueued = DateTime.UtcNow.ToJSONString();
var nextQueue = PEMBus.Queues.Where(que => que.PipelineModuleId == message.CurrentInstruction.QueueId).FirstOrDefault();
if (nextQueue == null) /* We couldn't find the queue for the next step */
{
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogError(Resources.ErrorCodes.PipelineEnqueing.InvalidMessageIndex, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"));
message.ErrorMessages.Add(Resources.ErrorCodes.PipelineEnqueing.MissingPipelineQueue.ToError());
message.Status = StatusTypes.Failed;
message.ErrorReason = ErrorReason.UnexepctedError;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
}
else
{
await nextQueue.EnqueueAsync(message);
}
}
}
}
else /* Processing Failed*/
{
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
if (message.ErrorReason == ErrorReason.None)
{
message.ErrorReason = ErrorReason.SeeErrorLog;
}
message.ExecutionTimeMS += Math.Round(sw.Elapsed.TotalMilliseconds, 3);
message.Status = StatusTypes.Failed;
Metrics.ErrorCount++;
Metrics.DeadLetterCount++;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
await PEMBus.PEMStorage.AddMessageAsync(message);
}
}
catch (ValidationException ex)
{
if (sw.IsRunning)
{
sw.Stop();
}
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogException($"pipeline.{this.GetType().Name.ToLower()}", ex, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"), (string.IsNullOrEmpty(message.MessageId) ? "????".ToKVP("messageId") : message.MessageId.ToKVP("messageId")));
message.ErrorMessages.Add(new Error()
{
Message = ex.Message,
Details = ex.StackTrace,
DeviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN",
});
message.CurrentInstruction = null;
message.ErrorReason = ErrorReason.UnexepctedError;
message.Status = StatusTypes.Failed;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
}
catch (Exception ex)
{
if (sw.IsRunning)
{
sw.Stop();
}
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
message.CurrentInstruction = null;
message.ErrorMessages.Add(new Error()
{
Message = ex.Message,
Details = ex.StackTrace,
DeviceId = deviceId,
});
message.ErrorReason = ErrorReason.UnexepctedError;
message.Status = StatusTypes.Failed;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
LogException($"pipeline.{this.GetType().Name.ToLower()}", ex, message.Id.ToKVP("pemid"), deviceId.ToKVP("deviceId"), (string.IsNullOrEmpty(message.MessageId) ? "????".ToKVP("messageId") : message.MessageId.ToKVP("messageId")));
}
finally
{
Metrics.ProcessingMS += sw.Elapsed.TotalMilliseconds;
Metrics.MessagesProcessed++;
Metrics.ActiveCount--;
}
}
protected async Task QueueForNextExecutionAsync(PipelineExecutionMessage message)
{
await _outputQueue.EnqueueAsync(message);
}
public async Task <InvokeResult> AddStringMessageAsync(string buffer, DateTime startTimeStamp, string path = "", string deviceId = "", string topic = "", Dictionary <string, string> headers = null)
{
try
{
var message = new PipelineExecutionMessage()
{
PayloadType = MessagePayloadTypes.Text,
TextPayload = buffer,
CreationTimeStamp = startTimeStamp.ToJSONString()
};
var headerLength = 0;
if (headers != null)
{
if (headers.ContainsKey("method"))
{
message.Envelope.Method = headers["method"];
}
if (headers.ContainsKey("topic"))
{
message.Envelope.Topic = headers["topic"];
foreach (var header in headers)
{
headerLength += header.Key.Length + (String.IsNullOrEmpty(header.Value) ? 0 : header.Value.Length);
}
}
if (headers != null)
{
foreach (var hdr in headers)
{
message.Envelope.Headers.Add(hdr.Key, hdr.Value);
}
}
}
message.PayloadLength = String.IsNullOrEmpty(buffer) ? 0 : buffer.Length;
var bytesProcessed = message.PayloadLength + (String.IsNullOrEmpty(path) ? 0 : path.Length) + headerLength;
Metrics.BytesProcessed += bytesProcessed;
Metrics.MessagesProcessed++;
var json = JsonConvert.SerializeObject(Metrics);
/*
* Console.WriteLine("LISTENER => " + Id);
* Console.ForegroundColor = ConsoleColor.Yellow;
* Console.WriteLine(json);
* Console.WriteLine("----------------------------");
*/
message.Envelope.DeviceId = deviceId;
message.Envelope.Path = path;
message.Envelope.Topic = topic;
var listenerInstruction = new PipelineExecutionInstruction()
{
Name = _listenerConfiguration.Name,
Type = GetType().Name,
QueueId = "N/A",
StartDateStamp = startTimeStamp.ToJSONString(),
ProcessByHostId = ModuleHost.Id,
Enqueued = startTimeStamp.ToJSONString(),
ExecutionTimeMS = (DateTime.UtcNow - startTimeStamp).TotalMilliseconds,
};
message.Instructions.Add(listenerInstruction);
var planner = PEMBus.Instance.Solution.Value.Planner.Value;
var plannerInstruction = new PipelineExecutionInstruction()
{
Name = planner.Name,
Type = "Planner",
QueueId = _plannerQueue.InstanceId,
Enqueued = DateTime.UtcNow.ToJSONString()
};
message.CurrentInstruction = plannerInstruction;
message.Instructions.Add(plannerInstruction);
await _plannerQueue.EnqueueAsync(message);
return(InvokeResult.Success);
}
catch (Exception ex)
{
PEMBus.InstanceLogger.AddException("ListenerModule_AddStringMessageAsync", ex);
return(InvokeResult.FromException("ListenerModule_AddStringMessageAsync", ex));
}
}
public Task <InvokeResult> EnqueueAsync(PipelineExecutionMessage message)
{
_messageQueue.Enqueue(message);
return(Task.FromResult(InvokeResult.Success));
}
public async Task <InvokeResult> AddStringMessageAsync(string buffer, DateTime startTimeStamp, string path = "", string deviceId = "", string topic = "", Dictionary <string, string> headers = null)
{
try
{
if (!String.IsNullOrEmpty(topic))
{
Console.WriteLine($"Received Message with topic [{topic}]");
}
if (!String.IsNullOrEmpty(topic) && topic.StartsWith("nuviot/srvr/dvcsrvc"))
{
return(await HandleSystemMessageAsync(topic, buffer));
}
if (!String.IsNullOrEmpty(path) && path.StartsWith("/nuviot/srvr/dvcsrvc"))
{
return(await HandleSystemMessageAsync(path.TrimStart('/'), buffer));
}
var message = new PipelineExecutionMessage()
{
PayloadType = MessagePayloadTypes.Text,
TextPayload = buffer,
CreationTimeStamp = startTimeStamp.ToJSONString()
};
var headerLength = 0;
if (headers != null)
{
if (headers.ContainsKey("method"))
{
message.Envelope.Method = headers["method"];
}
if (headers.ContainsKey("topic"))
{
message.Envelope.Topic = headers["topic"];
foreach (var header in headers)
{
headerLength += header.Key.Length + (String.IsNullOrEmpty(header.Value) ? 0 : header.Value.Length);
}
}
if (headers != null)
{
foreach (var hdr in headers)
{
message.Envelope.Headers.Add(hdr.Key, hdr.Value);
}
}
}
message.PayloadLength = String.IsNullOrEmpty(buffer) ? 0 : buffer.Length;
var bytesProcessed = message.PayloadLength + (String.IsNullOrEmpty(path) ? 0 : path.Length) + headerLength;
Metrics.BytesProcessed += bytesProcessed;
Metrics.MessagesProcessed++;
var json = JsonConvert.SerializeObject(Metrics);
/*
* Console.WriteLine("LISTENER => " + Id);
* Console.ForegroundColor = ConsoleColor.Yellow;
* Console.WriteLine(json);
* Console.WriteLine("----------------------------");
*/
message.Envelope.DeviceId = deviceId;
message.Envelope.Path = path;
message.Envelope.Topic = topic;
var listenerInstruction = new PipelineExecutionInstruction()
{
Name = _listenerConfiguration.Name,
Type = GetType().Name,
QueueId = _listenerConfiguration.Id,
StartDateStamp = startTimeStamp.ToJSONString(),
ProcessByHostId = PEMBus.Instance.PrimaryHost.Id,
Enqueued = startTimeStamp.ToJSONString(),
ExecutionTimeMS = (DateTime.UtcNow - startTimeStamp).TotalMilliseconds,
};
message.Instructions.Add(listenerInstruction);
var plannerQueue = PEMBus.Queues.Where(queue => queue.ForModuleType == PipelineModuleType.Planner).FirstOrDefault();
if (plannerQueue == null)
{
PEMBus.InstanceLogger.AddError("ListenerModule_AddStringMessageAsync", "Could not find planner queue.");
return(InvokeResult.FromError("Could not find planner queue."));
}
var planner = PEMBus.Instance.Solution.Value.Planner.Value;
var plannerInstruction = new PipelineExecutionInstruction()
{
Name = planner.Name,
Type = "Planner",
QueueId = plannerQueue.PipelineModuleId,
Enqueued = DateTime.UtcNow.ToJSONString()
};
message.CurrentInstruction = plannerInstruction;
message.Instructions.Add(plannerInstruction);
await plannerQueue.EnqueueAsync(message);
return(InvokeResult.Success);
}
catch (Exception ex)
{
Metrics.DeadLetterCount++;
Metrics.ErrorCount++;
PEMBus.InstanceLogger.AddException("ListenerModule_AddStringMessageAsync", ex);
Console.WriteLine(ex.StackTrace);
return(InvokeResult.FromException("ListenerModule_AddStringMessageAsync", ex));
}
}
public abstract Task <InvokeResult> SendResponseAsync(PipelineExecutionMessage message, OutgoingMessage msg);
public async Task <InvokeResult> AddMediaMessageAsync(Stream stream, string contentType, long contentLength, DateTime startTimeStamp, string path, String deviceId = "", String topic = "", Dictionary <string, string> headers = null)
{
try
{
var message = new PipelineExecutionMessage()
{
PayloadType = MessagePayloadTypes.Media,
CreationTimeStamp = startTimeStamp.ToJSONString()
};
Metrics.MessagesProcessed++;
message.PayloadLength = contentLength;
Metrics.BytesProcessed += message.PayloadLength + (String.IsNullOrEmpty(topic) ? 0 : topic.Length);
message.Envelope.DeviceId = deviceId;
message.Envelope.Topic = topic;
message.Envelope.Path = path;
var headerLength = 0;
if (headers != null)
{
if (headers.ContainsKey("method"))
{
message.Envelope.Method = headers["method"];
}
if (headers.ContainsKey("topic"))
{
message.Envelope.Topic = headers["topic"];
foreach (var header in headers)
{
headerLength += header.Key.Length + (String.IsNullOrEmpty(header.Value) ? 0 : header.Value.Length);
}
}
foreach (var hdr in headers)
{
message.Envelope.Headers.Add(hdr.Key, hdr.Value);
}
}
var listenerInstruction = new PipelineExecutionInstruction()
{
Name = _listenerConfiguration.Name,
Type = GetType().Name,
QueueId = "N/A",
StartDateStamp = startTimeStamp.ToJSONString(),
ProcessByHostId = PEMBus.Instance.PrimaryHost.Id,
ExecutionTimeMS = (DateTime.UtcNow - startTimeStamp).TotalMilliseconds,
};
message.Instructions.Add(listenerInstruction);
var plannerQueue = PEMBus.Queues.Where(queue => queue.ForModuleType == PipelineModuleType.Planner).FirstOrDefault();
var planner = PEMBus.Instance.Solution.Value.Planner.Value;
var plannerInstruction = new PipelineExecutionInstruction()
{
Name = "Planner",
Type = "Planner",
QueueId = plannerQueue.PipelineModuleId,
Enqueued = DateTime.UtcNow.ToJSONString()
};
message.CurrentInstruction = plannerInstruction;
message.Instructions.Add(plannerInstruction);
double?lat = null, lon = null;
if (headers != null &&
headers.ContainsKey("x-latitude") && headers.ContainsKey("x-longitude"))
{
if (double.TryParse(headers["x-latitude"], out double tmpLatitude) &&
double.TryParse(headers["x-longitude"], out double tmpLongitude))
{
lat = tmpLatitude;
lon = tmpLongitude;
}
}
var insertResult = await PEMBus.DeviceMediaStorage.StoreMediaItemAsync(stream, message.Id, contentType, contentLength, lat, lon);
if (!insertResult.Successful)
{
return(insertResult.ToInvokeResult());
}
message.MediaItemId = insertResult.Result;
await plannerQueue.EnqueueAsync(message);
return(InvokeResult.Success);
}
catch (Exception ex)
{
PEMBus.InstanceLogger.AddException("ListenerModule_AddBinaryMessageAsync", ex);
return(InvokeResult.FromException("ListenerModule_AddBinaryMessageAsync", ex));
}
}
protected async Task ExecuteAsync(PipelineExecutionMessage message)
{
Metrics.ActiveCount++;
var sw = new Stopwatch();
try
{
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Start processing {this.Configuration.Name}");
}
var currentInstruction = message.Instructions.Where(pm => pm.QueueId == message.CurrentInstruction.QueueId).FirstOrDefault();
sw.Start();
currentInstruction.StartDateStamp = DateTime.UtcNow.ToJSONString();
var result = await ProcessAsync(message);
sw.Stop();
currentInstruction.ExecutionTimeMS = Math.Round(sw.Elapsed.TotalMilliseconds, 3);;
message.ExecutionTimeMS += currentInstruction.ExecutionTimeMS;
currentInstruction.ProcessByHostId = PEMBus.Instance.Id;
Metrics.BytesProcessed += message.PayloadLength;
message.ErrorMessages.AddRange(result.ErrorMessages);
message.InfoMessages.AddRange(result.InfoMessages);
message.WarningMessages.AddRange(result.WarningMessages);
if (result.Success)
{
var instructionIndex = message.Instructions.IndexOf(currentInstruction);
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Success processing {instructionIndex} - {currentInstruction.Name} - {this.Configuration.Name}");
}
instructionIndex++;
if (instructionIndex == message.Instructions.Count) /* We are done processing the pipe line */
{
message.Status = message.WarningMessages.Count > 0 ? StatusTypes.CompletedWithWarnings : StatusTypes.Completed;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
//For now since we are working on the same instance we don't have to write this to external storage while enqueing, will need to when we run on different nodes
await UpdateDevice(message);
await PEMBus.PEMStorage.AddMessageAsync(message);
await PEMBus.DeviceWatchdog.DeviceUpdatedAsync(message.Device);
await PEMBus.MessageWatchdog.MessageProcessedAsync(message.Device, message.MessageId);
//await PEMBus.Watchdog.DeviceUpdatedAsync(message.Device, message.Device.WatchdogSecondsOverride)
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Completed processing {this.Configuration.Name}");
}
}
else
{
if (instructionIndex > message.Instructions.Count) /* Somehow we overrun the index for the pipeline steps */
{
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogError(Resources.ErrorCodes.PipelineEnqueing.InvalidMessageIndex, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"));
message.ErrorMessages.Add(Resources.ErrorCodes.PipelineEnqueing.InvalidMessageIndex.ToError());
message.Status = StatusTypes.Failed;
message.ErrorReason = ErrorReason.UnexepctedError;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
message.CurrentInstruction = null;
Metrics.ErrorCount++;
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Invalid Message Index {this.Configuration.Name}");
}
}
else
{
//Find and set the next instruction.
message.CurrentInstruction = message.Instructions[instructionIndex];
var nextQueue = PEMBus.Queues.Where(que => que.PipelineModuleId == message.CurrentInstruction.QueueId).FirstOrDefault();
if (nextQueue == null) /* We couldn't find the queue for the next step */
{
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogError(Resources.ErrorCodes.PipelineEnqueing.MissingPipelineQueue, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"), message.CurrentInstruction.Name.ToKVP("queueName"), instructionIndex.ToString().ToKVP("instructionIndex"));
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR FINDING QUEUE");
Console.WriteLine("\t\t" + instructionIndex.ToString());
Console.WriteLine("\t\t" + message.CurrentInstruction.Name);
Console.WriteLine("\t\t" + message.CurrentInstruction.QueueId);
Console.WriteLine("----------------------");
Console.ResetColor();
message.ErrorMessages.Add(Resources.ErrorCodes.PipelineEnqueing.MissingPipelineQueue.ToError());
message.Status = StatusTypes.Failed;
message.ErrorReason = ErrorReason.UnexepctedError;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Could not find next queue {instructionIndex}. {this.Configuration.Name}");
}
}
else
{
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Enqueued for next: {message.CurrentInstruction.Name} - {nextQueue.ForModuleType + " " + nextQueue.Key}");
}
message.CurrentInstruction.Enqueued = DateTime.UtcNow.ToJSONString();
await nextQueue.EnqueueAsync(message);
}
}
}
}
else /* Processing Failed*/
{
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Failed processing {this.Configuration.Name} - {message.ErrorReason}");
}
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
if (message.ErrorReason == ErrorReason.None)
{
message.ErrorReason = ErrorReason.SeeErrorLog;
}
message.ExecutionTimeMS += Math.Round(sw.Elapsed.TotalMilliseconds, 3);
message.Status = StatusTypes.Failed;
Metrics.ErrorCount++;
Metrics.DeadLetterCount++;
message.CurrentInstruction = null;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
await PEMBus.PEMStorage.AddMessageAsync(message);
}
}
catch (ValidationException ex)
{
if (sw.IsRunning)
{
sw.Stop();
}
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Failed processing {this.Configuration.Name} - Validation Exception {ex.Message}");
}
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
LogException($"pipeline_{this.GetType().Name.ToLower()}_ExecuteAsync", ex, message.Id.ToKVP("pemId"), deviceId.ToKVP("deviceId"), (string.IsNullOrEmpty(message.MessageId) ? "????".ToKVP("messageId") : message.MessageId.ToKVP("messageId")));
message.ErrorMessages.Add(new Error()
{
Message = ex.Message,
Details = ex.StackTrace,
DeviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN",
});
message.CurrentInstruction = null;
message.ErrorReason = ErrorReason.UnexepctedError;
message.Status = StatusTypes.Failed;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
}
catch (Exception ex)
{
if (sw.IsRunning)
{
sw.Stop();
}
if (message.Device != null && message.Device.DebugMode)
{
SendDeviceNotification(Targets.WebSocket, message.Device.Id, $"Failed processing {this.Configuration.Name} - Exception {ex.Message}");
}
var deviceId = message.Device != null ? message.Device.DeviceId : "UNKNOWN";
message.CurrentInstruction = null;
message.ErrorMessages.Add(new Error()
{
Message = ex.Message,
Details = ex.StackTrace,
DeviceId = deviceId,
});
message.ErrorReason = ErrorReason.UnexepctedError;
message.Status = StatusTypes.Failed;
message.CompletionTimeStamp = DateTime.UtcNow.ToJSONString();
Metrics.DeadLetterCount++;
await PEMBus.PEMStorage.AddMessageAsync(message);
LogException($"pipeline_{this.GetType().Name.ToLower()}_ExecuteAsync", ex, message.Id.ToKVP("pemid"), deviceId.ToKVP("deviceId"), (string.IsNullOrEmpty(message.MessageId) ? "????".ToKVP("messageId") : message.MessageId.ToKVP("messageId")));
Console.WriteLine(ex.Message);
Console.WriteLine(ex.StackTrace);
}
finally
{
Metrics.ProcessingMS += sw.Elapsed.TotalMilliseconds;
Metrics.MessagesProcessed++;
Metrics.ActiveCount--;
}
}
public async override Task <ProcessResult> ProcessAsync(PipelineExecutionMessage message)
{
await AddStringMessageAsync(message.TextPayload, DateTime.Now, deviceId : message.Device.DeviceId, topic : message.Envelope.Topic);
return(ProcessResult.FromSuccess);
}
public abstract Task <ProcessResult> ProcessAsync(PipelineExecutionMessage message);
public override Task <InvokeResult> SendResponseAsync(PipelineExecutionMessage message, OutgoingMessage msg)
{
throw new NotImplementedException();
}