protected override async void OnNavigatedTo(NavigationEventArgs e)
{
_state = StreamingState.Idle;
if (_faceTracker == null)
{
_faceTracker = await FaceTracker.CreateAsync();
}
if (_faceApiHelper == null)
{
try
{
_faceApiHelper = new FaceApiHelper();
_eventHubHelper = new EventHubHelper();
//用不到
//await _faceApiHelper.CheckGroupExistAsync();
}
catch (Microsoft.ProjectOxford.Face.FaceAPIException faceEx)
{
ShowErrorHelper.ShowDialog(faceEx.ErrorMessage, faceEx.ErrorCode);
}
catch (Microsoft.Azure.EventHubs.EventHubsException eventhubEx)
{
ShowErrorHelper.ShowDialog(eventhubEx.Message);
}
catch (Exception ex)
{
ShowErrorHelper.ShowDialog(ex.Message);
}
}
}
protected override async void OnNavigatedTo(NavigationEventArgs e)
{
base.OnNavigatedTo(e);
await sensor.Init();
EHub = new EventHubHelper();
EHub.serviceNamespace = "weathercenter-ns";
EHub.hubName = "weatherhub";
EHub.deviceName = "shwarspi";
EHub.sharedAccessPolicyName = "all";
EHub.sharedAccessKey = "cFsp8GEvk/iRnjehSt/JBHjIyAV0lGVBWGJqAc9/IMw=";
var timer = new DispatcherTimer
{
Interval = TimeSpan.FromSeconds(3)
};
timer.Tick += async(sender, o) =>
{
var temp = sensor.Temperature;
Debug.WriteLine("Temperature = " + temp);
TheTextBlock.Text = $"Temperature : {temp}°C";
await EHub.SendMessage(
$"{{\"temperature\":\"{temp.ToString().Replace(',', '.')}\", \"source\":\"RPi2\", \"timewhen\":\"{DateTime.Now.ToString("o")}\"}}");
};
timer.Start();
}
public async Task ProcessDeviceTwin(EventData[] source, ILogger log, string tenant, string deviceId)
{
foreach (EventData message in source)
{
try
{
if (tenant != null)
{
var connectionString = TenantConnectionHelper.GetEventHubConnectionString(Convert.ToString(tenant));
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
string eventData = Encoding.UTF8.GetString(message.Body.Array);
message.Properties.TryGetValue("opType", out object operationType);
message.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubEnqueuedTime, out object dateTimeReceived);
var timeStamp = new TelemetryTimestamp(Convert.ToDateTime(dateTimeReceived));
DeviceService deviceService = new DeviceService();
await deviceService.SaveDeviceTwinOperationAsync(eventData, log, Convert.ToString(tenant), deviceId.ToString(), operationType.ToString(), timeStamp, eventHubHelper);
}
}
catch (Exception ex)
{
log.LogError($"Error occurrred in for loop: {ex.Message} StackTrace: {ex.StackTrace} Inner Exception: {(string.IsNullOrEmpty(ex.StackTrace) ? string.Empty : ex.StackTrace)}");
}
}
}
private void GenerateAction(EventHubElement eventHub)
{
// Event hub client
var eventHubHelper = new EventHubHelper(eventHub.EventHubName, eventHub.ConnectionString);
// Send order to event hub
var order = GenerateOrder(eventHub.Region);
Console.WriteLine($"Sending order message to {eventHub.Region}");
eventHubHelper.SendMessageToEventHub(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(order))).GetAwaiter().GetResult();
// Close connection
eventHubHelper.CloseEventHub().GetAwaiter().GetResult();
}
private async Task AddDeviceGroupToADX(DeviceGroup deviceGroup, bool isDeleted = false)
{
bool isKustoEnabled = this.config.DeviceTelemetryService.Messages.TelemetryStorageType.Equals(Common.Services.Models.TelemetryStorageTypeConstants.Ade, StringComparison.OrdinalIgnoreCase);
if (isKustoEnabled)
{
string tenantId = this.httpContextAccessor.HttpContext.Request.GetTenant();
var connectionString = this.tenantConnectionHelper.GetEventHubConnectionString(tenantId);
if (!string.IsNullOrWhiteSpace(connectionString))
{
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
var result = this.GetDeviceGroupsForADX(deviceGroup, isDeleted);
await eventHubHelper.SendMessageToEventHub($"{tenantId}-devicegroup", new EventData[] { result });
}
}
}
private void WaitingForHub()
{
ReceiveAny(message =>
{
message.Match()
.With <SetHub>(h =>
{
_hub = h.Hub;
Become(InitializePubSub);
})
.Default(o =>
{
Stash.Stash();
});
});
}
public async Task Start()
{
List <TenantModel> tenantList = await this.GetAllActiveTenantAsync();
var tasks = tenantList.Select(async tenant =>
{
string tenantId = tenant.TenantId;
try
{
this.logger.LogInformation($"Started device twin migration of tenant: {tenantId} ");
List <Twin> deviceTwinList = await this.GetDevices(tenantId);
var connectionString = this.tenantConnectionHelper.GetEventHubConnectionString(Convert.ToString(tenantId));
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
var eventDatas = new ConcurrentBag <EventData>();
Parallel.ForEach(deviceTwinList, deviceTwin =>
{
JObject deviceTwinJson = new JObject();
deviceTwinJson.Add(DeviceId, deviceTwin.DeviceId);
deviceTwinJson.Add(TimeStamp, DateTime.UtcNow);
deviceTwinJson.Add(TimeReceived, DateTime.UnixEpoch);
deviceTwinJson.Add(EventOpType, "updateTwin");
deviceTwinJson.Add(IsDeleted, false);
deviceTwinJson.Add(Data, deviceTwin.ToJson());
deviceTwinJson.Add(DeviceCreatedDate, default(DateTime));
var byteMessage = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(deviceTwinJson));
var eventDeviceTwinData = new Azure.Messaging.EventHubs.EventData(byteMessage);
eventDeviceTwinData.Properties.Add(DeviceId, deviceTwin.DeviceId);
eventDatas.Add(eventDeviceTwinData);
});
await eventHubHelper.SendMessageToEventHub($"{tenantId}-devicetwin", eventDatas.ToList());
this.logger.LogInformation($"Completed device twin migration of tenant: {tenantId} ");
}
catch (Exception ex)
{
this.logger.LogError(ex, $"Failed device twin migration of tenant: {tenantId} ");
}
});
await Task.WhenAll(tasks);
}
private async Task ProcessBatchedTelemetryMessagesAsync(
EventData[] batchedSource,
EventHubHelper eventHubHelper,
ILogger log,
object tenant,
object deviceId,
TelemetryTimestamp telemetryTimestamp,
int batchThreshold,
int batchWriteDelay)
{
// get the final data point for timestamp calculation
// if batchedSource is empty set to null, as it's value will never be used.
var finalDataPoint = batchedSource.Length > 0 ? batchedSource.Last() : null;
var finalTelemetry = finalDataPoint != null?this.ConvertEventDataToJObject(finalDataPoint, log) : null;
var finalTimestamp = this.GetBatchedDataPointTimestamp(finalTelemetry, telemetryTimestamp);
await Task.WhenAll(
batchedSource.Select((dataPoint, idx) =>
{
return(Task.Run(async() =>
{
// apply some delay as the batch size grows
await Task.Delay(idx / batchThreshold *batchWriteDelay);
var telemetry = this.ConvertEventDataToJObject(dataPoint, log);
var dataPointTimestamp = this.GetBatchedDataPointTimestamp(
telemetry,
telemetryTimestamp);
var computedOffsetTimestamp = this.ComputeBatchedEnqueuedTime(
telemetryTimestamp,
dataPointTimestamp,
finalTimestamp);
await this.ProcessTelemetryMessageAndWriteToEventHub(
telemetry,
eventHubHelper,
log,
tenant,
deviceId,
computedOffsetTimestamp);
}));
}));
}
private async Task ProcessTelemetryMessageAndWriteToEventHub(
JObject telemetry,
EventHubHelper eventHubHelper,
ILogger log,
object tenant,
object deviceId,
TelemetryTimestamp telemetryTimestamp)
{
try
{
if (telemetry == null || !telemetry.HasValues)
{
throw new Exception("Telemetry message had no values to write to cosmos");
}
/*
* telemetry[DeviceTelemetryKeyConstants.DeviceId] = deviceId.ToString();
*
* telemetry.Add(DeviceTelemetryKeyConstants.DateTimeReceived, telemetryTimestamp.DateTime.ToString());
* telemetry.Add(DeviceTelemetryKeyConstants.TimeReceived, telemetryTimestamp.EpochTimestamp);
* telemetry.Add(DeviceTelemetryKeyConstants.Schema, DeviceTelemetryKeyConstants.MessageSchema);
*/
JObject telemetryData = new JObject();
telemetryData.Add(DeviceTelemetryKeyConstants.Data, telemetry);
telemetryData.Add(DeviceTelemetryKeyConstants.DeviceId, deviceId.ToString());
telemetryData.Add(DeviceTelemetryKeyConstants.DateTimeReceived, telemetryTimestamp.DateTime);
telemetryData.Add(DeviceTelemetryKeyConstants.TimeReceived, telemetryTimestamp.EpochTimestamp);
telemetryData.Add(DeviceTelemetryKeyConstants.Schema, DeviceTelemetryKeyConstants.MessageSchema);
// Save the telemetry message to EventHub for further processing
var byteMessage = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(telemetryData));
var eventData = new Azure.Messaging.EventHubs.EventData(byteMessage);
eventData.Properties.Add("deviceid", deviceId.ToString());
await eventHubHelper.SendMessageToEventHub($"{tenant}-telemetry", new Azure.Messaging.EventHubs.EventData[] { eventData });
}
catch (Exception ex)
{
this.success = false;
log.LogError($"Error occurrred : {ex.Message}");
}
}
public async Task Start()
{
List <TenantModel> tenantList = await this.GetAllActiveTenantAsync();
var tasks = tenantList.Select(async tenant =>
{
string tenantId = tenant.TenantId;
try
{
this.logger.LogInformation($"Started device group migration of tenant: {tenantId} ");
IEnumerable <DeviceGroup> deviceGroupList = await this.GetDeviceGroups(tenantId);
var connectionString = this.tenantConnectionHelper.GetEventHubConnectionString(Convert.ToString(tenantId));
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
List <EventData> events = new List <EventData>();
foreach (var deviceGroup in deviceGroupList)
{
var result = this.GetDeviceGroupsForADX(deviceGroup);
if (result != null)
{
events.Add(result);
}
}
await eventHubHelper.SendMessageToEventHub($"{tenantId}-devicegroup", events);
this.logger.LogInformation($"Completed device twin migration of tenant: {tenantId} ");
}
catch (Exception ex)
{
this.logger.LogError(ex, $"Failed device twin migration of tenant: {tenantId} ");
}
});
await Task.WhenAll(tasks);
}
static void Main(string[] args)
{
EventHubHelper.GenerateEventHubMessages().Wait();
}
private async Task <PartitionReceiver> ConnectToEventHubAsync(string eventHubConnectionString, string hubName,
IReliableDictionary <string, string> streamOffsetDictionary)
{
var eventHubHelper = new EventHubHelper();
var eventHubClient = eventHubHelper.CreatEventHubClientIfExist(eventHubConnectionString, hubName);
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
PartitionReceiver partitionReceiver = null;
string[] partitionIds = eventHubRuntimeInfo.PartitionIds;
_eventHubPartitionId = await GetMatchingEventHubPartitionId(partitionIds);
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await streamOffsetDictionary.TryGetValueAsync(tx, Names.HubStreamOffSetKey);
EventPosition eventPosition;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
eventPosition = EventPosition.FromSequenceNumber(long.Parse(offsetResult.Value));
}
else
{
// first time this service is running so there is no offset value yet. start with the current time.
// Load from database sequence number
eventPosition = await LoadEventPositionFromDatabaseAsync() ??
EventPosition.FromEnqueuedTime(DateTime.UtcNow);
//EventPosition.FromEnqueuedTime(DateTime.UtcNow.Subtract(TimeSpan.FromHours(5)));//EventPosition.FromEnqueuedTime(DateTime.UtcNow);
if (eventPosition.SequenceNumber != null)
{
_latestSequenceNumber = eventPosition.SequenceNumber.Value;
await streamOffsetDictionary.SetAsync(tx, Names.HubStreamOffSetKey, eventPosition.SequenceNumber.ToString());
await tx.CommitAsync();
}
}
ServiceEventSource.Current.ServiceMessage(this.Context, "Creating EventHub listener on partition {0} with SequenceNumber {1}",
_eventHubPartitionId, eventPosition.SequenceNumber);
partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, _eventHubPartitionId, eventPosition);
}
}
catch (Exception e)
{
//ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ConnectToEventHubAsync met exception= ( {e} )");
string err = $"RouterService ConnectToEventHubAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId} .";
//ServiceEventSource.Current.CriticalError("RouterService", err);
throw;
}
return(partitionReceiver);
}
public async Task ProcessTelemetryAsync(EventData[] source, ILogger log, AppConfigHelper configHelper, int batchThreshold = 12, int batchWriteDelay = 85)
{
if (batchThreshold <= 0)
{
throw new ArgumentOutOfRangeException("batchThreshold must be greater than 0");
}
List <Task> messagingTasks = new List <Task>();
// iterate over the eventData array and add a task to process each element
foreach (EventData eventData in source)
{
// get the iothub-enqueuedtime, iothub-connection-device-id, and tenent from the eventData system properties
// these values are to be added to the telemetry message before being stored in cosmos (tenent is used to save telemetry to cosmos)
eventData.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubEnqueuedTime, out object dateTimeReceived);
eventData.SystemProperties.TryGetValue(DeviceTelemetryKeyConstants.IotHubConnectionDeviceId, out object deviceId);
eventData.Properties.TryGetValue(DeviceTelemetryKeyConstants.Tenant, out object tenant);
if (tenant == null || dateTimeReceived == null || deviceId == null)
{
this.success = false;
log.LogError($"EventData did not contain one or more of the following: {DeviceTelemetryKeyConstants.Tenant}, {DeviceTelemetryKeyConstants.IotHubEnqueuedTime}, {DeviceTelemetryKeyConstants.IotHubEnqueuedTime}.");
continue; // go on to the next event data - this one cannot be written
}
var telemetryTimestamp = new TelemetryTimestamp(Convert.ToDateTime(dateTimeReceived));
var connectionString = await configHelper.GetValueByKey($"tenant:{tenant}:eventHubConn");
EventHubHelper eventHubHelper = new EventHubHelper(connectionString);
bool isBatchedTelemetry = false;
if (eventData.Properties.TryGetValue(DeviceTelemetryKeyConstants.BatchedTelemetry, out object isBatchedValue))
{
bool.TryParse(isBatchedValue.ToString(), out isBatchedTelemetry);
}
try
{
if (isBatchedTelemetry)
{
messagingTasks.Add(
Task.Run(async() =>
{
var batchedEventData = this.ConvertBatchedDataToKeyValueData(eventData, log);
await this.ProcessBatchedTelemetryMessagesAsync(
batchedEventData,
eventHubHelper,
log,
tenant,
deviceId,
telemetryTimestamp,
batchThreshold,
batchWriteDelay);
}));
}
else
{
messagingTasks.Add(Task.Run(async() =>
{
var telemetry = this.ConvertEventDataToJObject(eventData, log);
await this.ProcessTelemetryMessageAndWriteToEventHub(
telemetry,
eventHubHelper,
log,
tenant,
deviceId,
telemetryTimestamp);
}));
}
}
catch (Exception)
{
this.success = false;
log.LogError($"An error occurred while writing some eventData to cosmos");
}
}
await Task.WhenAll(messagingTasks.ToArray());
if (!this.success)
{
throw new Exception("Failed to process one or more telemetry messages.");
}
}
public async Task SaveDeviceTwinOperationAsync(string eventData, ILogger log, string tenant, string deviceId, string operationType, TelemetryTimestamp timeStamp, EventHubHelper eventHubHelper)
{
try
{
string deviceTwin = eventData;
Twin twin = null;
JObject deviceTwinJson = new JObject();
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceId, deviceId.ToString());
deviceTwinJson.Add(DeviceTelemetryKeyConstants.TimeStamp, timeStamp.DateTime);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.TimeReceived, timeStamp.EpochTimestamp);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.EventOpType, operationType);
deviceTwinJson.Add(DeviceTelemetryKeyConstants.IsDeleted, false);
if (operationType.ToString().Equals("createDeviceIdentity"))
{
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, timeStamp.DateTime);
try
{
twin = await TenantConnectionHelper.GetRegistry(Convert.ToString(tenant)).GetTwinAsync(deviceId.ToString());
deviceTwin = twin.ToJson();
}
catch (Exception e)
{
log.LogError($"Unable to fetch DeviceId: {deviceId} device twin from iothub of tenant: {tenant}.", e);
}
}
else
{
JObject previousTwin = null;
KustoOperations kustoClient = await KustoOperations.GetClientAsync();
string kustoQuery = $"DeviceTwin | where DeviceId == \"{deviceId}\" | summarize arg_max(TimeStamp, *) by DeviceId | where IsDeleted == false";
var deviceTwinList = await kustoClient.QueryAsync <DeviceTwinModel>($"IoT-{tenant}", kustoQuery, null);
DeviceTwinModel preDeviceTwin = deviceTwinList.FirstOrDefault();
if (preDeviceTwin != null)
{
previousTwin = preDeviceTwin.Twin;
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, preDeviceTwin.DeviceCreatedDate);
}
else
{
deviceTwinJson.Add(DeviceTelemetryKeyConstants.DeviceCreatedDate, default(DateTime)); // Set Device Created Date to Default if twin is not present in storage.
try
{
twin = await TenantConnectionHelper.GetRegistry(Convert.ToString(tenant)).GetTwinAsync(deviceId.ToString());
if (twin != null)
{
previousTwin = JObject.Parse(twin.ToJson());
}
}
catch (Exception e)
{
log.LogError($"Unable to fetch DeviceId: {deviceId} device twin from iothub of tenant: {tenant}.", e);
}
}
switch (operationType)
{
case "deviceConnected":
if (preDeviceTwin != null)
{
previousTwin["connectionState"] = "Connected";
previousTwin["lastActivityTime"] = timeStamp.DateTime;
}
break;
case "deviceDisconnected":
if (preDeviceTwin != null)
{
previousTwin["connectionState"] = "Disconnected";
previousTwin["lastActivityTime"] = timeStamp.DateTime;
}
break;
case "updateTwin":
if (preDeviceTwin != null)
{
JObject twinFragment = JObject.Parse(eventData);
previousTwin = previousTwin.UpdateJson(twinFragment);
}
else
{
previousTwin = JObject.Parse(eventData);
}
break;
case "deleteDeviceIdentity":
deviceTwinJson[DeviceTelemetryKeyConstants.IsDeleted] = true;
break;
default:
break;
}
deviceTwin = previousTwin?.ToString();
}
deviceTwinJson.Add(DeviceTelemetryKeyConstants.Data, deviceTwin);
// Save the Device Twin Data to EventHub for further processing
var byteMessage = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(deviceTwinJson));
var eventDeviceTwinData = new Azure.Messaging.EventHubs.EventData(byteMessage);
eventDeviceTwinData.Properties.Add("deviceid", deviceId.ToString());
await eventHubHelper.SendMessageToEventHub($"{tenant}-devicetwin", new Azure.Messaging.EventHubs.EventData[] { eventDeviceTwinData });
}
catch (Exception exception)
{
throw new ApplicationException($"Save Device Twin operation failed: {exception}, operation type: {operationType}, tenant: {tenant}, deviceId {deviceId}");
}
}
public SetHub(EventHubHelper hub)
{
Hub = hub;
}
