public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
var batch = new TableBatchOperation();
foreach(var msg in messages)
{
var snap = JsonConvert.DeserializeObject<BusSnapshotInfo>(Encoding.UTF8.GetString(msg.GetBytes()));
var entity = new DynamicTableEntity(snap.RouteShortName, snap.VehicleId.ToString());
entity.Properties.Add("RouteShortName", EntityProperty.GeneratePropertyForString(snap.RouteShortName));
entity.Properties.Add("VehicleId", EntityProperty.GeneratePropertyForInt(snap.VehicleId));
entity.Properties.Add("TripId", EntityProperty.GeneratePropertyForInt(snap.TripId));
entity.Properties.Add("Latitude", EntityProperty.GeneratePropertyForDouble(snap.Latitude));
entity.Properties.Add("Longitude", EntityProperty.GeneratePropertyForDouble(snap.Longitude));
entity.Properties.Add("DirectionOfTravel", EntityProperty.GeneratePropertyForString(snap.DirectionOfTravel.ToString()));
entity.Properties.Add("NextStopId", EntityProperty.GeneratePropertyForInt(snap.NextStopId));
entity.Properties.Add("Timeliness", EntityProperty.GeneratePropertyForString(snap.Timeliness.ToString()));
entity.Properties.Add("TimelinessOffset", EntityProperty.GeneratePropertyForInt(snap.TimelinessOffset));
entity.Properties.Add("Timestamp", EntityProperty.GeneratePropertyForDateTimeOffset(snap.Timestamp));
batch.Add(TableOperation.InsertOrReplace(entity));
}
var tableClient = _account.CreateCloudTableClient();
var table = tableClient.GetTableReference("snapshots");
await table.CreateIfNotExistsAsync();
await table.ExecuteBatchAsync(batch);
await context.CheckpointAsync();
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> events)
{
// Workaround for event hub sending null on timeout
events = events ?? Enumerable.Empty<EventData>();
if(!await _elasticSearchWriter.WriteAsync(events.ToList(), _token).ConfigureAwait(false))
{
return;
}
try
{
EventData checkpointEventData = events.LastOrDefault();
await context.CheckpointAsync(checkpointEventData);
WarmStorageEventSource.Log.CheckpointCompleted(ProcessorName, _eventHubName, context.Lease.PartitionId, checkpointEventData.Offset);
}
catch (Exception ex)
{
if (!(ex is StorageException || ex is LeaseLostException))
{
throw;
}
WarmStorageEventSource.Log.UnableToCheckpoint(ex, ProcessorName, _eventHubName, context.Lease.PartitionId);
}
}
Task IEventProcessor.OpenAsync(PartitionContext context)
{
Console.WriteLine("SimpleEventProcessor initialized. Partition: '{0}', Offset: '{1}'", context.Lease.PartitionId, context.Lease.Offset);
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public Task OpenAsync(PartitionContext context)
{
this.partitionContext = context;
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
try
{
foreach (EventData message in messages)
{
string contents = Encoding.UTF8.GetString(message.GetBytes());
Console.WriteLine(string.Format("SimpleEventProcessor: {0}", contents));
OnMessageReceived(this, new MessageReceivedEventArgs() { ReceivedOn = DateTimeOffset.UtcNow, Message = message });
}
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(2))
{
lock (this)
{
this.checkpointStopWatch.Reset();
return context.CheckpointAsync();
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
return Task.FromResult<object>(null);
}
public IEventProcessor CreateEventProcessor(PartitionContext context)
{
var processor = new DeviceAdministrationProcessor(_deviceLogic, _configurationProvider);
processor.ProcessorClosed += this.ProcessorOnProcessorClosed;
this.eventProcessors.TryAdd(context.Lease.PartitionId, processor);
return processor;
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
Trace.TraceInformation("\n");
Trace.TraceInformation("........ProcessEventsAsync........");
foreach (EventData eventData in messages)
{
try
{
string jsonString = Encoding.UTF8.GetString(eventData.GetBytes());
Trace.TraceInformation(string.Format("Message received at '{0}'. Partition: '{1}'",
eventData.EnqueuedTimeUtc.ToLocalTime(), this.partitionContext.Lease.PartitionId));
Trace.TraceInformation(string.Format("-->Raw Data: '{0}'", jsonString));
SensorEvent newSensorEvent = this.DeserializeEventData(jsonString);
Trace.TraceInformation(string.Format("-->Serialized Data: '{0}', '{1}', '{2}', '{3}', '{4}'",
newSensorEvent.timestart, newSensorEvent.dsplalert, newSensorEvent.alerttype, newSensorEvent.message, newSensorEvent.targetalarmdevice));
// Issuing alarm to device.
string commandParameterNew = "{\"Name\":\"AlarmThreshold\",\"Parameters\":{\"SensorId\":\"" + newSensorEvent.dsplalert + "\"}}";
Trace.TraceInformation("Issuing alarm to device: '{0}', from sensor: '{1}'", newSensorEvent.targetalarmdevice, newSensorEvent.dsplalert);
Trace.TraceInformation("New Command Parameter: '{0}'", commandParameterNew);
await WorkerRole.iotHubServiceClient.SendAsync(newSensorEvent.targetalarmdevice, new Microsoft.Azure.Devices.Message(Encoding.UTF8.GetBytes(commandParameterNew)));
}
catch (Exception ex)
{
Trace.TraceInformation("Error in ProssEventsAsync -- {0}\n", ex.Message);
}
}
await context.CheckpointAsync();
}
async Task IEventProcessor.ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
foreach (EventData eventData in messages)
{
byte[] data = eventData.GetBytes();
if (eventData.Properties.ContainsKey("messageType") && (string)eventData.Properties["messageType"] == "interactive")
{
var messageId = (string)eventData.SystemProperties["message-id"];
var queueMessage = new BrokeredMessage(new MemoryStream(data));
queueMessage.MessageId = messageId;
queueMessage.Properties["messageType"] = "interactive";
await queueClient.SendAsync(queueMessage);
WriteHighlightedMessage(string.Format("Received interactive message: {0}", messageId));
continue;
}
if (toAppend.Length + data.Length > MAX_BLOCK_SIZE || stopwatch.Elapsed > MAX_CHECKPOINT_TIME)
{
await AppendAndCheckpoint(context);
}
await toAppend.WriteAsync(data, 0, data.Length);
Console.WriteLine(string.Format("Message received. Partition: '{0}', Data: '{1}'",
context.Lease.PartitionId, Encoding.UTF8.GetString(data)));
}
}
public Task OpenAsync(PartitionContext context)
{
Console.WriteLine(string.Format("Processor open. Partition: '{0}', Offset: '{1}'", context.Lease.PartitionId, context.Lease.Offset));
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> events)
{
try
{
Notifications notifications = new Notifications();
foreach (EventData eventData in events)
{
var dataString = Encoding.UTF8.GetString(eventData.GetBytes());
var newData = JsonConvert.DeserializeObject<MetricEvent>(dataString);
EventMessage message = GetMessage(newData.Type, dataString);
if (message == null)
message = newData.GetMessage();
notifications.Notify(message);
Console.WriteLine(string.Format("Message received.Partition:'{0}',{1},Device:'{2}'", this.partitionContext.Lease.PartitionId, newData.Type, newData.DeviceId));
}
//Call checkpoint every 5 minutes, so that worker can resume processing from the 5 minutes back if it restarts.
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(5))
{
await context.CheckpointAsync();
this.checkpointStopWatch.Restart();
}
}
catch (Exception exp)
{
Console.WriteLine("Error in processing: " + exp.Message);
}
}
public Task OpenAsync(PartitionContext context)
{
/*client = new HttpClient();
client.DefaultRequestHeaders.Add("X-ZUMO-APPLICATION", APP_KEY_MOBILE_SERVICES);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
return Task.FromResult<object>(null);*/
var storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
var tableClient = storageAccount.CreateCloudTableClient();
sensorLogTable = tableClient.GetTableReference("SensorLog");
//sensorLogTable = tableClient.GetTableReference("SensorLog" + context.Lease.PartitionId);
// 閾値の取得
if (thresholdTempWarning == null)
{
var sensorConfigTable = tableClient.GetTableReference("SensorConfig");
var query = new TableQuery<SensorConfig>();
var configData = sensorConfigTable.ExecuteQuery(query);
foreach (SensorConfig config in configData)
{
if (config.PartitionKey == "TemperatureWarning")
{
thresholdTempWarning = config.Threshold;
System.Console.WriteLine("ThresholdTempWarning: " + thresholdTempWarning);
}
else if (config.PartitionKey == "TemperatureDanger")
{
thresholdTempDanger = config.Threshold;
System.Console.WriteLine("ThresholdTempDanger: " + thresholdTempDanger);
}
}
}
return sensorLogTable.CreateIfNotExistsAsync();
}
public async Task CloseAsync(PartitionContext context, CloseReason reason)
{
if (reason == CloseReason.Shutdown)
{
await context.CheckpointAsync();
}
}
async Task IEventProcessor.ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
var iDbService = new DbService();
_client = iDbService.GetFirebaseClient();
foreach (EventData eventData in messages)
{
string data = Encoding.UTF8.GetString(eventData.GetBytes());
FirebaseResponse response = await _client.PushAsync("event", new EHdata
{
offset = eventData.Offset,
body = data,
partitionId = context.Lease.PartitionId
});
Console.WriteLine(String.Format("Message received. Partition: '{0}', Data: '{1}', Offset: '{2}'",
context.Lease.PartitionId, data, eventData.Offset));
}
//Call checkpoint every 5 minutes, so that worker can resume processing from the 5 minutes back if it restarts.
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(5))
{
Console.WriteLine(this.checkpointStopWatch.Elapsed);
await context.CheckpointAsync();
this.checkpointStopWatch.Restart();
}
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> events)
{
// Workaround for event hub sending null on timeout
events = events ?? Enumerable.Empty<EventData>();
foreach (var eventData in events)
{
var updateTemperatureEvent = JsonConvert.DeserializeObject<UpdateTemperatureEvent>(Encoding.UTF8.GetString(eventData.GetBytes()));
eventData.Properties["BuildingId"] = _buildingLookupService.GetBuildingId(updateTemperatureEvent.DeviceId);
}
if(!await _elasticSearchWriter.WriteAsync(events.ToList(), _token).ConfigureAwait(false))
{
return;
}
try
{
EventData checkpointEventData = events.LastOrDefault();
await context.CheckpointAsync(checkpointEventData);
WarmStorageEventSource.Log.CheckpointCompleted(ProcessorName, _eventHubName, context.Lease.PartitionId, checkpointEventData.Offset);
}
catch (Exception ex)
{
if (!(ex is StorageException || ex is LeaseLostException))
{
throw;
}
WarmStorageEventSource.Log.UnableToCheckpoint(ex, ProcessorName, _eventHubName, context.Lease.PartitionId);
}
}
async Task IEventProcessor.ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
var partitionedMessages = messages.GroupBy(data => data.PartitionKey).ToDictionary(datas => datas.Key, datas => datas.ToList());
//For each partition spawn a Task which will sequentially iterate over its own block
//Wait for all Tasks to complete, before proceeding.
await Task.WhenAll(partitionedMessages.Select(partition => Task.Run(async () =>
{
var block = partition.Value;
foreach (var eventData in block)
{
try
{
var data = Encoding.UTF8.GetString(eventData.GetBytes());
System.Console.WriteLine(DateTime.Now + ":Message received. Partition: '{0}', Data: '{1}', Partition Key: '{2}'", context.Lease.PartitionId, data, eventData.PartitionKey);
}
catch (Exception e)
{
//do something with your logs..
}
}
})));
//Call checkpoint every 5 minutes, so that worker can resume processing from the 5 minutes back if it restarts.
if (checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(5))
{
await context.CheckpointAsync();
checkpointStopWatch.Restart();
}
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
foreach (EventData eventData in messages)
{
if (eventData.Properties.ContainsKey("time"))
{
if (eventData.Properties.ContainsKey("temp"))
Console.WriteLine(string.Format("time = {0}, temp = {1}", eventData.Properties["time"], eventData.Properties["temp"]));
if (eventData.Properties.ContainsKey("hmdt"))
Console.WriteLine(string.Format("time = {0}, hmdt = {1}", eventData.Properties["time"], eventData.Properties["hmdt"]));
if (eventData.Properties.ContainsKey("accx") &&
eventData.Properties.ContainsKey("accy") &&
eventData.Properties.ContainsKey("accz"))
Console.WriteLine(string.Format("time = {0}, accx = {1}, accy = {2}, accz = {3}", eventData.Properties["time"], eventData.Properties["accx"], eventData.Properties["accy"], eventData.Properties["accz"]));
if (eventData.Properties.ContainsKey("bpm"))
Console.WriteLine(string.Format("time = {0}, bpm = {1}", eventData.Properties["time"], eventData.Properties["bpm"]));
}
}
//Call checkpoint every 5 minutes, so that worker can resume processing from the 5 minutes back if it restarts.
//if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(5))
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromSeconds(30))
{
await context.CheckpointAsync();
this.checkpointStopWatch.Restart();
}
}
public Task OpenAsync(PartitionContext context)
{
Console.WriteLine("EventProcessor started");
this.stopWatch = new Stopwatch();
this.stopWatch.Start();
return Task.FromResult<object>(null);
}
public async Task OpenAsync(PartitionContext context)
{
_log.InfoEvent("Open",
new Facet("eventHubPath", context.EventHubPath),
new Facet("partitionId", context.Lease.PartitionId),
new Facet("offset", context.Lease.Offset));
}
async Task IEventProcessor.CloseAsync(PartitionContext context, CloseReason reason)
{
if (reason == CloseReason.Shutdown)
{
await context.CheckpointAsync();
}
}
public Task OpenAsync(PartitionContext context)
{
if (!WebJobsHelper.RunAsWebJobs)
Console.WriteLine(string.Format("EventProcessor initialization. Partition: '{0}', Offset: '{1}'",
context.Lease.PartitionId, context.Lease.Offset));
partitionContext = context;
var retries = 3;
while (retries > 0)
{
try
{
retries--;
var username = ConfigurationManager.AppSettings["SendGridUser"];
var pswd = ConfigurationManager.AppSettings["SendGridPass"];
var apiKey = ConfigurationManager.AppSettings["SendGridAPIKey"];
transportWeb = new Web(apiKey);
retries = 0;
}
catch (Exception e)
{
Console.Error.WriteLine("Error opening destination Event Hub: " + e.Message);
if (retries == 0)
throw;
}
}
checkpointStopWatch = new Stopwatch();
checkpointStopWatch.Start();
return Task.CompletedTask;
}
public async Task CloseAsync(PartitionContext context, CloseReason reason)
{
_log.InfoEvent("Close",
new Facet("reason", reason),
new Facet("partitionId", context.Lease.PartitionId),
new Facet("offset", context.Lease.Offset));
}
public async Task OpenAsync(PartitionContext context)
{
if (!WebJobsHelper.RunAsWebJobs)
Console.WriteLine(string.Format("EventProcessor initialization. Partition: '{0}', Offset: '{1}'",
context.Lease.PartitionId, context.Lease.Offset));
partitionContext = context;
var retries = 3;
while (retries > 0)
{
try
{
retries--;
hubClient = EventHubClient.CreateFromConnectionString(
ConfigurationManager.ConnectionStrings["SigfoxDemoAlertSender"].ConnectionString,
"alert");
cacheConnection = await ConnectionMultiplexer.ConnectAsync(ConfigurationManager.ConnectionStrings["SigfoxDemoCache"].ConnectionString);
cacheDatabase = cacheConnection.GetDatabase();
sqlConnection = new SqlConnection(ConfigurationManager.ConnectionStrings["SigfoxDemoDatabase"].ConnectionString);
//sqlConnection.Open();
//sqlCommand = new SqlCommand("InsertAlert", sqlConnection) { CommandType = CommandType.StoredProcedure };
//sqlCommand.Parameters.Add(new SqlParameter("@Device", SqlDbType.VarChar));
retries = 0;
}
catch (Exception e)
{
Console.Error.WriteLine("Error opening destination Event Hub: " + e.Message);
if (retries == 0)
throw;
}
}
checkpointStopWatch = new Stopwatch();
checkpointStopWatch.Start();
}
async Task IEventProcessor.ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
foreach (EventData eventData in messages)
{
_Logger.LogInfo(string.Format("Event received from partition: {0} - {1}", context.Lease.PartitionId,eventData.PartitionKey));
try
{
var httpMessage = HttpMessage.Parse(eventData.GetBodyStream());
await _MessageContentProcessor.ProcessHttpMessage(httpMessage);
}
catch (Exception ex)
{
_Logger.LogError(ex.Message);
}
}
//Call checkpoint every 5 minutes, so that worker can resume processing from the 5 minutes back if it restarts.
if (this.checkpointStopWatch.Elapsed > TimeSpan.FromMinutes(5))
{
_Logger.LogInfo("Checkpointing");
await context.CheckpointAsync();
this.checkpointStopWatch.Restart();
}
}
//------------------------------------------------------------------------------------------------------------------------
Task IEventProcessor.OpenAsync(PartitionContext context)
{
DebugEx.TraceLog("SimpleEventProcessor initialized. Partition: " + context.Lease.PartitionId + ", Offset: " + context.Lease.Offset);
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public Task OpenAsync(PartitionContext context)
{
if (!WebJobsHelper.RunAsWebJobs)
Console.WriteLine(string.Format("EventProcessor initialization. Partition: '{0}', Offset: '{1}'",
context.Lease.PartitionId, context.Lease.Offset));
partitionContext = context;
var retries = 3;
while (retries > 0)
{
try
{
retries--;
hubClient = EventHubClient.CreateFromConnectionString(
ConfigurationManager.ConnectionStrings["SigfoxDemoDispatchSender"].ConnectionString,
"dispatch");
retries = 0;
}
catch (Exception e)
{
Console.Error.WriteLine("Error opening destination Event Hub: " + e.Message);
if (retries == 0)
throw;
}
}
checkpointStopWatch = new Stopwatch();
checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public async Task OpenAsync(PartitionContext context)
{
if (!WebJobsHelper.RunAsWebJobs)
Console.WriteLine(string.Format("EventProcessor initialization. Partition: '{0}', Offset: '{1}'",
context.Lease.PartitionId, context.Lease.Offset));
partitionContext = context;
var retries = 3;
while (retries > 0)
{
var s = string.Empty;
try
{
retries--;
s = "storage";
storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["SigfoxDemoStorage"].ConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
blobContainer = blobClient.GetContainerReference("device");
blobContainer.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Off });
blobContainer.CreateIfNotExists();
s = "cache";
cacheConnection = await ConnectionMultiplexer.ConnectAsync(ConfigurationManager.ConnectionStrings["SigfoxDemoCache"].ConnectionString);
cacheDatabase = cacheConnection.GetDatabase();
s = "database";
sqlConnection = new SqlConnection(ConfigurationManager.ConnectionStrings["SigfoxDemoDatabase"].ConnectionString);
sqlConnection.Open();
sqlCommand = new SqlCommand("InsertMessage", sqlConnection) { CommandType = CommandType.StoredProcedure };
sqlCommand.Parameters.Add(new SqlParameter("@Device", SqlDbType.VarChar));
sqlCommand.Parameters.Add(new SqlParameter("@Data", SqlDbType.VarChar));
sqlCommand.Parameters.Add(new SqlParameter("@Mode", SqlDbType.TinyInt));
sqlCommand.Parameters.Add(new SqlParameter("@Periode", SqlDbType.TinyInt));
sqlCommand.Parameters.Add(new SqlParameter("@FrameType", SqlDbType.TinyInt));
sqlCommand.Parameters.Add(new SqlParameter("@Battery", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Temperature", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Humidity", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@ILS", SqlDbType.Bit));
sqlCommand.Parameters.Add(new SqlParameter("@Light", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Version", SqlDbType.VarChar));
sqlCommand.Parameters.Add(new SqlParameter("@AlertCount", SqlDbType.Int));
sqlCommand.Parameters.Add(new SqlParameter("@TimeStamp", SqlDbType.DateTime));
sqlCommand.Parameters.Add(new SqlParameter("@Duplicate", SqlDbType.Bit));
sqlCommand.Parameters.Add(new SqlParameter("@Signal", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Station", SqlDbType.VarChar));
sqlCommand.Parameters.Add(new SqlParameter("@AvgSignal", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Latitude", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Longitude", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@Rssi", SqlDbType.Float));
sqlCommand.Parameters.Add(new SqlParameter("@SeqNumber", SqlDbType.Int));
retries = 0;
}
catch (Exception e)
{
Console.Error.WriteLine("Error opening destination Event Hub: " + e.Message + "(" + s + ")");
if (retries == 0)
throw;
}
}
checkpointStopWatch = new Stopwatch();
checkpointStopWatch.Start();
}
async Task IEventProcessor.CloseAsync(PartitionContext context, CloseReason reason)
{
Console.WriteLine("Processor Shutting Down. Partition '{0}', Reason: '{1}'.", context.Lease.PartitionId, reason);
if (reason == CloseReason.Shutdown)
{
await context.CheckpointAsync();
}
}
public async Task CloseAsync(PartitionContext context, CloseReason reason)
{
Console.WriteLine(string.Format("Processor Shuting Down. Partition '{0}', Reason: '{1}'.", context.Lease.PartitionId, reason.ToString()));
if (reason == CloseReason.Shutdown)
{
await context.CheckpointAsync();
}
}
public Task OpenAsync(PartitionContext context)
{
WarmStorageEventSource.Log.LeaseObtained(ProcessorName, _eventHubName, context.Lease.PartitionId);
_elasticSearchWriter = _elasticSearchWriterFactory(context.Lease.PartitionId);
return Task.FromResult(false);
}
public Task OpenAsync(PartitionContext context)
{
Trace.TraceInformation(string.Format("Initializing EventProcessor: Partition: '{0}', Offset: '{1}'", context.Lease.PartitionId, context.Lease.Offset));
this.partitionContext = context;
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
/// <summary>
/// Process all events from OPC UA servers and update the last value of each node in the topology.
/// </summary>
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> ingestedMessages)
{
_sessionUpdateStopwatch.Start();
// process each message
foreach (var eventData in ingestedMessages)
{
string message = null;
try
{
// checkpoint, so that the processor does not need to start from the beginning if it restarts
if (_checkpointStopwatch.Elapsed > TimeSpan.FromMinutes(_checkpointPeriodInMinutes))
{
Task.Run(async() => await CheckpointAsync(context, _checkpointStopwatch));
}
message = Encoding.UTF8.GetString(eventData.Body.Array, eventData.Body.Offset, eventData.Body.Count);
if (message != null)
{
_processorHostMessages++;
// support batched messages as well as simple message ingests
if (message.StartsWith("["))
{
List <PublisherMessage> publisherMessages = JsonConvert.DeserializeObject <List <PublisherMessage> >(message);
foreach (var publisherMessage in publisherMessages)
{
if (publisherMessage != null)
{
_publisherMessages++;
try
{
ProcessPublisherMessage(publisherMessage.OpcUri, publisherMessage.NodeId, publisherMessage.Value.SourceTimestamp, publisherMessage.Value.Value, eventhub_warning);
_lastSourceTimestamp = publisherMessage.Value.SourceTimestamp;
_lastOpcUri = publisherMessage.OpcUri;
_lastNodeId = publisherMessage.NodeId;
}
catch (Exception e)
{
Trace.TraceError($"Exception '{e.Message}' while processing message {publisherMessage}");
}
}
}
}
else
{
PublisherMessage publisherMessage = JsonConvert.DeserializeObject <PublisherMessage>(message);
_publisherMessagesInvalidFormat++;
if (publisherMessage != null)
{
ProcessPublisherMessage(publisherMessage.OpcUri, publisherMessage.NodeId, publisherMessage.Value.SourceTimestamp, publisherMessage.Value.Value, eventhub_warning);
_lastSourceTimestamp = publisherMessage.Value.SourceTimestamp;
_lastOpcUri = publisherMessage.OpcUri;
_lastNodeId = publisherMessage.NodeId;
}
}
// if there are sessions looking at stations we update sessions each second
if (_sessionUpdateStopwatch.ElapsedMilliseconds > TimeSpan.FromSeconds(1).TotalMilliseconds)
{
if (SessionsViewingStationsCount() != 0)
{
try
{
Trace.TraceInformation($"processorHostMessages {_processorHostMessages}, publisherMessages {_publisherMessages}/{_publisherMessagesInvalidFormat}, sourceTimestamp: '{_lastSourceTimestamp}'");
Trace.TraceInformation($"opcUri '{_lastOpcUri}', nodeid '{_lastNodeId}'");
TriggerSessionChildrenDataUpdate();
}
catch (Exception e)
{
Trace.TraceError($"Exception '{e.Message}' while updating browser sessions");
}
finally
{
_sessionUpdateStopwatch.Restart();
}
}
}
}
}
catch (Exception e)
{
Trace.TraceError($"Exception '{e.Message}' processing message '{message}'");
}
}
}
public Task CloseAsync(PartitionContext context, CloseReason reason)
{
Log.DebugFormat("Close lease: Reason: {0}, {1}", reason, new PartitionContextInfo(context));
return(Task.CompletedTask);
}
public Task OpenAsync(PartitionContext context)
{
_partitionContext = context;
return(Task.FromResult <object>(null));
}
/// <inheritdoc/>
public async Task ProcessEventsAsync(PartitionContext context,
IEnumerable <EventData> messages)
{
if (messages == null || !messages.Any())
{
return;
}
var messagesCount = 0;
var messageSequence = 0L;
foreach (var eventData in messages)
{
messagesCount++;
if (_outer._config.SkipEventsOlderThan != null &&
eventData.SystemProperties.TryGetValue("x-opt-enqueued-time", out var enqueued) &&
(DateTime)enqueued + _outer._config.SkipEventsOlderThan < DateTime.UtcNow)
{
kOldEvent.WithLabels(_processorId, context.EventHubPath, context.ConsumerGroupName, context.PartitionId).Inc();
continue;
}
var properties = new EventProperties(eventData.SystemProperties,
eventData.Properties);
if (eventData.Body.Array == null)
{
_logger.Verbose("WARNING: Received empty message with properties {@properties}",
properties);
continue;
}
await _handler.HandleAsync(eventData.Body.Array, properties,
() => CheckpointAsync(context, eventData));
if (context.CancellationToken.IsCancellationRequested)
{
// Checkpoint to the last processed event.
await CheckpointAsync(context, eventData);
context.CancellationToken.ThrowIfCancellationRequested();
}
// sequence number of the message in eventhub which is used to calculate lag
messageSequence = eventData.SystemProperties.SequenceNumber;
}
var lastEnqueuedSequence = context.RuntimeInformation.LastSequenceNumber;
var sequenceDifference = lastEnqueuedSequence - messageSequence;
TotalMessagesCount += messagesCount;
kEventProcessorMessages.WithLabels(_processorId, context.EventHubPath, context.ConsumerGroupName,
context.PartitionId).Set(TotalMessagesCount);
kEventProcessorLag.WithLabels(_processorId, context.EventHubPath, context.ConsumerGroupName,
context.PartitionId).Set(sequenceDifference);
// Checkpoint if needed
if (_sw.ElapsedMilliseconds >= _interval)
{
try {
_logger.Debug("Checkpointing EventProcessor {id} for partition {partitionId}...",
_processorId, context.PartitionId);
await context.CheckpointAsync();
_sw.Restart();
}
catch (Exception ex) {
_logger.Warning(ex, "Failed checkpointing EventProcessor {id} for partition {partitionId}...",
_processorId, context.PartitionId);
kEventProcessorDetails.WithLabels(_processorId, context.EventHubPath, context.ConsumerGroupName,
context.PartitionId, "checkpoint_failed").Inc();
if (_sw.ElapsedMilliseconds >= 2 * _interval)
{
// Give up checkpointing after trying a couple more times
_sw.Restart();
}
}
}
await Try.Async(_handler.OnBatchCompleteAsync);
}
public Task OpenAsync(PartitionContext context)
{
return(Task.CompletedTask);
}
IEventProcessor IEventProcessorFactory.CreateEventProcessor(PartitionContext context)
{
return(new EventProcessor(context, logger, Interlocked.Increment(ref numCreated)));
}
public EventProcessor(PartitionContext context, TextWriter logger, int numCreated)
{
log = logger;
partitionID = context.Lease.PartitionId;
}
protected abstract bool OnSubmit(EventData message, PartitionContext context);
public Task ProcessErrorAsync(PartitionContext context, Exception error)
{
// ToDo: improve error handling
Log.ErrorFormat("Partition {0} error", error, new PartitionContextInfo(context));
return(Task.CompletedTask);
}
public Task CloseAsync(PartitionContext context, CloseReason reason)
{
System.Diagnostics.Trace.TraceInformation("Closing RouteItemProcessor: {0}", reason);
return(Task.FromResult(false));
}
public Task ProcessErrorAsync(PartitionContext context, Exception error)
{
Console.WriteLine($"Error on Partition: {context.PartitionId}, Error: {error.Message}");
return(Task.CompletedTask);
}
/// <summary>
/// Called when the ownership of partition moves to a different node for load-balancing purpose, or when the host is shutting down. Called in response to <see cref="M:Microsoft.ServiceBus.Messaging.EventHubConsumerGroup.UnregisterProcessorAsync(Microsoft.ServiceBus.Messaging.Lease,Microsoft.ServiceBus.Messaging.CloseReason)"/>.
/// </summary>
/// <param name="context">Partition ownership information for the partition on which this processor instance works. You can call <see cref="M:Microsoft.ServiceBus.Messaging.PartitionContext.CheckpointAsync"/> to checkpoint progress in the processing of messages from Event Hub streams.</param>
/// <param name="reason">The reason for calling <see cref="M:Microsoft.ServiceBus.Messaging.IEventProcessor.CloseAsync(Microsoft.ServiceBus.Messaging.PartitionContext,Microsoft.ServiceBus.Messaging.CloseReason)"/>.</param>
/// <returns>
/// A task indicating that the Close operation is complete.
/// </returns>
public Task CloseAsync(PartitionContext context, CloseReason reason)
{
return(Task.FromResult <object>(null));
}
/// <summary>
/// Initializes the Event Hub processor instance. This method is called before any event data is passed to this processor instance.
/// </summary>
/// <param name="context">Ownership information for the partition on which this processor instance works. Any attempt to call <see cref="M:Microsoft.ServiceBus.Messaging.PartitionContext.CheckpointAsync"/> will fail during the Open operation.</param>
/// <returns>
/// The task that indicates that the Open operation is complete.
/// </returns>
public Task OpenAsync(PartitionContext context)
{
Logger.LogInfo("Open Async");
return(Task.FromResult <object>(null));
}
IEventProcessor IEventProcessorFactory.CreateEventProcessor(PartitionContext context)
{
return(new MediaServicesEventProcessor(jobName, jobWaitingEvent, liveEventName));
}
public Task CloseAsync(PartitionContext context, CloseReason reason)
{
return(Task.CompletedTask);
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
{
foreach (var eventData in messages)
{
Log.Logger.Information("Processing event: {sequenceNumber}", eventData.SequenceNumber);
var data = Encoding.UTF8.GetString(eventData.GetBytes());
try
{
if (data.Contains("<"))
{
var eventXml = XElement.Parse(data);
var lastChangeNode =
eventXml.Descendants(XName.Get("LastChange"))
.SingleOrDefault();
if (lastChangeNode != null)
{
var rawEvent = XElement.Parse(lastChangeNode.Value);
var currentTrackMetaDataNode = rawEvent
.Descendants(XName.Get("CurrentTrackMetaData",
"urn:schemas-upnp-org:metadata-1-0/AVT/"))
.SingleOrDefault(n => !String.IsNullOrEmpty(n.Attribute("val").Value));
if (currentTrackMetaDataNode != null)
{
var metadata = currentTrackMetaDataNode.Attribute(XName.Get("val")).Value;
var metaXml = XElement.Parse(metadata);
var classTypeNode =
metaXml.Descendants(XName.Get("class", "urn:schemas-upnp-org:metadata-1-0/upnp/"))
.SingleOrDefault();
if (classTypeNode != null)
{
var classType = classTypeNode.Value;
switch (classType)
{
case "object.item.audioItem.musicTrack":
var res =
metaXml.Descendants(XName.Get("res",
"urn:schemas-upnp-org:metadata-1-0/DIDL-Lite/"))
.SingleOrDefault();
if (res != null)
{
var urlString = res.Value;
var albumNode =
metaXml.Descendants(XName.Get("album",
"urn:schemas-upnp-org:metadata-1-0/upnp/"))
.SingleOrDefault();
string album = null;
if (albumNode != null)
{
album = albumNode.Value;
}
var track =
metaXml.Descendants(XName.Get("title",
"http://purl.org/dc/elements/1.1/"))
.Single()
.Value;
var artist =
metaXml.Descendants(XName.Get("creator",
"http://purl.org/dc/elements/1.1/"))
.Single()
.Value;
urlString = urlString.Replace("pndrradio-", string.Empty);
var fileLocation = new Uri(urlString);
var albumArtUriNode = metaXml.Descendants(XName.Get("albumArtURI",
"urn:schemas-upnp-org:metadata-1-0/upnp/")).SingleOrDefault();
Uri albumArtUri = null;
if (albumArtUriNode != null)
{
albumArtUri = new Uri(albumArtUriNode.Value);
}
recordCreator.Tell(new NewRecordMessage(artist, album, track,
fileLocation, albumArtUri));
}
break;
}
}
}
}
}
}
catch (Exception e)
{
Log.Logger.Error(e, "Error: {e}");
}
await context.CheckpointAsync(eventData);
}
}
public Task ProcessErrorAsync(PartitionContext context, Exception error)
{
_telemetryClient.TrackException(error);
return(Task.CompletedTask);
}
public async Task OpenAsync(PartitionContext context)
{
Log.Information("SimpleEventProcessor initialize. Partition: '{partitionId}', Offset: '{offset}'",
context.Lease.PartitionId, context.Lease.Offset);
}
/// <inheritdoc/>
public IEventProcessor CreateEventProcessor(PartitionContext context)
{
return(new DefaultProcessor(this, context, _logger));
}
public TeleEventProcessor(PartitionContext context, string cs, string sharedCs)
{
_pointRepo = new PointRepo(cs);
_pointRepoShared = new PointRepo(sharedCs);
}
public Task ProcessErrorAsync(PartitionContext partitionContext, Exception error)
{
_context.Telemetry.Error(_context, $"Error on Partition: {partitionContext.PartitionId}, Error: {error.Message}", error);
return Task.CompletedTask;
}
public Task OpenAsync(PartitionContext partitionContext)
{
_context.Telemetry.Verbose(_context, $"SimpleEventProcessor initialized. Partition: '{partitionContext.PartitionId}'");
return Task.CompletedTask;
}
Task IEventProcessor.CloseAsync(PartitionContext context, CloseReason reason)
{
Console.WriteLine("Processor Shutting Down. Partition '{0}', Reason: '{1}'.", context.Lease.PartitionId, reason);
return(Task.FromResult <object>(null));
}
public Task CloseAsync(PartitionContext partitionContext, CloseReason reason)
{
_context.Telemetry.Verbose(_context, $"Processor Shutting Down. Partition '{partitionContext.PartitionId}', Reason: '{reason}'.");
return Task.CompletedTask;
}
public Task OpenAsync(PartitionContext context)
{
Log.DebugFormat("Open lease: {0}", new PartitionContextInfo(context));
return(Task.CompletedTask);
}
public Task OpenAsync(PartitionContext context)
{
System.Diagnostics.Trace.TraceInformation("Opening RouteItemProcessor");
return(Task.FromResult(false));
}
public Task CloseAsync(PartitionContext context, CloseReason reason)
{
Console.WriteLine($"Processor Shutting Down. Partition '{context.PartitionId}', Reason: '{reason}'.");
return(Task.CompletedTask);
}
public Task OpenAsync(PartitionContext context)
{
Console.WriteLine($"SimpleEventProcessor initialized. Partition: '{context.PartitionId}'");
return(Task.CompletedTask);
}
public Task ProcessErrorAsync(PartitionContext context, Exception error)
{
Trace.TraceError($"Message processor error '{error.Message}' on partition with id '{context.PartitionId}'");
Trace.TraceError($"Exception stack '{error.StackTrace}'");
return(Task.CompletedTask);
}
private async Task Checkpoint(EventData message, PartitionContext context)
{
Log.DebugFormat("Will checkpoint at Offset: {0}, {1}", message.SystemProperties.Offset, new PartitionContextInfo(context));
await context.CheckpointAsync(message);
}
