public static void CreateListenerPalAndReceiveMessages()
{
var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
{
EntityPath = Configuration.IoTHub.EventHubCompatibleName
};
var eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
EventHubRuntimeInformation eventRuntimeInformation = eventHubClient.GetRuntimeInformationAsync().Result;
int eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
foreach (string partitionId in eventRuntimeInformation.PartitionIds)
{
try
{
PartitionReceiver receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.Now.AddMinutes(-s_lookbackTimeInMinutes.TotalMinutes));
s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from {s_lookbackTimeInMinutes}");
Task.Run(async() =>
{
while (true)
{
IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false);
ProcessEventData(eventDatas);
}
});
}
catch (EventHubsException ex)
{
s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}");
}
}
}
public static EventHubRuntimeInformation GetEventHubRuntimeInfo(AmqpMessage amqpMessage)
{
EventHubRuntimeInformation eventHubRuntimeInformation = null;
object obj = null;
if (MessageConverter.TryGetNetObjectFromAmqpObject(amqpMessage.ValueBody.Value, MappingType.MessageBody, out obj))
{
Dictionary <string, object> strs = obj as Dictionary <string, object>;
if (strs != null)
{
eventHubRuntimeInformation = new EventHubRuntimeInformation();
object obj1 = null;
if (strs.TryGetValue("name", out obj1))
{
eventHubRuntimeInformation.Path = (string)obj1;
}
if (strs.TryGetValue("partition_count", out obj1))
{
eventHubRuntimeInformation.PartitionCount = (int)obj1;
}
if (strs.TryGetValue("created_at", out obj1))
{
eventHubRuntimeInformation.CreatedAt = (DateTime)obj1;
}
if (strs.TryGetValue("partition_ids", out obj1))
{
eventHubRuntimeInformation.PartitionIds = (string[])obj1;
}
}
}
return(eventHubRuntimeInformation);
}
private static async Task CheckpointLatest(EventHubRuntimeInformation runtimeInformation)
{
var leasedPartitionIds = new HashSet <string>();
var stopwatch = new Stopwatch();
EventProcessorHost processorHost = GetEventProcessorHost();
var factory = new CheckpointerFactory(() => new Checkpointer
{
OnOpen = partitionContext => leasedPartitionIds.Add(partitionContext.PartitionId),
OnCheckpoint = eventData => stopwatch.Restart()
});
await processorHost.RegisterEventProcessorFactoryAsync(factory);
do
{
await Task.Delay(10);
}while (leasedPartitionIds.Count < runtimeInformation.PartitionCount);
stopwatch.Start();
do
{
await Task.Delay(10);
}while (stopwatch.Elapsed.TotalSeconds < 1.0);
stopwatch.Stop();
await processorHost.UnregisterEventProcessorAsync();
}
public static void CreateListenerPalAndReceiveMessages()
{
var eventHubClient = EventHubClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString, "messages/events");
EventHubRuntimeInformation eventRuntimeInformation = eventHubClient.GetRuntimeInformation();
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
foreach (string partitionId in eventRuntimeInformation.PartitionIds)
{
try
{
EventHubReceiver receiver = eventHubClient.GetConsumerGroup(consumerGroupName).CreateReceiver(partitionId, DateTime.Now.AddMinutes(-s_lookbackTimeInMinutes.TotalMinutes));
s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from past {s_lookbackTimeInMinutes}");
Task.Run(async() =>
{
while (true)
{
IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false);
ProcessEventData(eventDatas);
}
});
}
catch (QuotaExceededException ex)
{
s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}");
}
}
}
protected override IEnumerator <IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .AsyncStep> GetAsyncSteps()
{
if (!this.managementLink.FaultTolerantLink.TryGetOpenedObject(out this.requestResponseLink))
{
AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult getEventHubRuntimeInfoAsyncResult = this;
IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .BeginCall beginCall = (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, TimeSpan t, AsyncCallback c, object s) => thisPtr.managementLink.FaultTolerantLink.BeginGetInstance(t, c, s);
yield return(getEventHubRuntimeInfoAsyncResult.CallAsync(beginCall, (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, IAsyncResult r) => thisPtr.requestResponseLink = thisPtr.managementLink.FaultTolerantLink.EndGetInstance(r), IteratorAsyncResult <TIteratorAsyncResult> .ExceptionPolicy.Transfer));
}
AmqpMessage applicationProperty = AmqpMessage.Create();
applicationProperty.ApplicationProperties = new ApplicationProperties();
applicationProperty.ApplicationProperties.Map["name"] = this.eventHubPath;
applicationProperty.ApplicationProperties.Map["operation"] = "READ";
applicationProperty.ApplicationProperties.Map["type"] = "com.microsoft:eventhub";
AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult getEventHubRuntimeInfoAsyncResult1 = this;
IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .BeginCall beginCall1 = (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, TimeSpan t, AsyncCallback c, object s) => thisPtr.requestResponseLink.BeginRequest(applicationProperty, t, c, s);
yield return(getEventHubRuntimeInfoAsyncResult1.CallAsync(beginCall1, (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, IAsyncResult r) => thisPtr.getRuntimeInfoResponse = thisPtr.requestResponseLink.EndRequest(r), IteratorAsyncResult <TIteratorAsyncResult> .ExceptionPolicy.Transfer));
int item = (int)this.getRuntimeInfoResponse.ApplicationProperties.Map["status-code"];
string str = (string)this.getRuntimeInfoResponse.ApplicationProperties.Map["status-description"];
if (item != 200)
{
base.Complete(AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult.ConvertToException(item, str));
}
else
{
this.Result = MessageConverter.GetEventHubRuntimeInfo(this.getRuntimeInfoResponse);
}
}
public async Task <IList <EventData> > GetMessagesFromAllPartitions(DateTime startTime, int maxPerPartition = 10, int waitTimeSecs = 5)
{
var messages = new List <EventData>();
EventHubRuntimeInformation rtInfo = await this.eventHubClient.GetRuntimeInformationAsync();
foreach (string partition in rtInfo.PartitionIds)
{
PartitionReceiver partitionReceiver = this.eventHubClient.CreateReceiver(
PartitionReceiver.DefaultConsumerGroupName,
partition,
startTime);
// Retry a few times to make sure we get all expected messages.
for (int i = 0; i < 3; i++)
{
IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs));
if (events != null)
{
messages.AddRange(events);
}
if (i < 3)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
await partitionReceiver.CloseAsync();
}
return(messages);
}
private static void listenForAsyncCommandUpdates(string requestId)
{
var connectionStringBuilder = new EventHubsConnectionStringBuilder(EVENTHUB_CONNECTION_STRING);
var partitionReceivers = new List <PartitionReceiver>();
var eventHubClient = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
var eventhubReceiverThreads = new List <Thread>();
EventHubRuntimeInformation runtimeInformation = eventHubClient.GetRuntimeInformationAsync().Result;
//spawn a thread per partition to receive over that partition
Console.WriteLine("Setting up EventHub listeners for updates to the async method...");
foreach (string partitionId in runtimeInformation.PartitionIds)
{
var partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, EventPosition.FromEnqueuedTime(DateTime.Now));
Thread partitionReceiverThread = new Thread(() => receive(partitionReceiver, requestId));
eventhubReceiverThreads.Add(partitionReceiverThread);
partitionReceiverThread.Start();
}
Console.WriteLine("EventHub listener now listening for updates containing the request id " + requestId);
Console.WriteLine();
//Wait for user to enter a key to exit the program
Console.ReadLine();
foreach (Thread partitionReceiverThread in eventhubReceiverThreads)
{
partitionReceiverThread.Interrupt();
}
}
public async Task <IActionResult> Post([FromRoute] string name, [FromBody] EventsProcessorApplicationParams parameters)
{
// Determine the number of IoT Hub partitions.
// The events processing service will be created with the same number of partitions.
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(parameters.IotHubConnectionString, "messages/events");
EventHubRuntimeInformation eventHubInfo = await eventHubClient.GetRuntimeInformationAsync();
// Application parameters are passed to the Events Processing application instance.
NameValueCollection appInstanceParameters = new NameValueCollection();
appInstanceParameters["IotHubConnectionString"] = parameters.IotHubConnectionString;
appInstanceParameters["IotHubProcessOnlyFutureEvents"] = parameters.IotHubProcessOnlyFutureEvents;
appInstanceParameters["PublishDataServiceURLs"] = parameters.PublishDataServiceURLs;
ApplicationDescription application = new ApplicationDescription(
new Uri($"{Names.EventsProcessorApplicationPrefix}/{name}"),
Names.EventsProcessorApplicationTypeName,
parameters.Version,
appInstanceParameters);
// Create a named application instance
await this.fabricClient.ApplicationManager.CreateApplicationAsync(application, this.operationTimeout, this.appLifetime.ApplicationStopping);
// Next, create named instances of the services that run in the application.
ServiceUriBuilder serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.EventsProcessorRouterServiceName);
StatefulServiceDescription service = new StatefulServiceDescription()
{
ApplicationName = application.ApplicationName,
HasPersistedState = true,
MinReplicaSetSize = 3,
TargetReplicaSetSize = 3,
PartitionSchemeDescription = new UniformInt64RangePartitionSchemeDescription(eventHubInfo.PartitionCount, 0, eventHubInfo.PartitionCount - 1),
ServiceName = serviceNameUriBuilder.Build(),
ServiceTypeName = Names.EventsProcessorRouterServiceTypeName
};
await this.fabricClient.ServiceManager.CreateServiceAsync(service, this.operationTimeout, this.appLifetime.ApplicationStopping);
if (parameters.PublishDataServiceURLs != null && parameters.PublishDataServiceURLs.Length > 0)
{
// Next, create named instances of the services that run in the application.
serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.EventsProcessorExtenderServiceName);
StatelessServiceDescription extenderService = new StatelessServiceDescription()
{
ApplicationName = application.ApplicationName,
InstanceCount = 1,
PartitionSchemeDescription = new SingletonPartitionSchemeDescription(),
ServiceName = serviceNameUriBuilder.Build(),
ServiceTypeName = Names.EventsProcessorExtenderServiceTypeName
};
await this.fabricClient.ServiceManager.CreateServiceAsync(extenderService, this.operationTimeout, this.appLifetime.ApplicationStopping);
}
return(this.Ok());
}
private static async Task <EventHubRuntimeInformation> GetRuntimeInformation()
{
var eventHubClient = EventHubClient.CreateFromConnectionString(_eventHubConnectionString);
EventHubRuntimeInformation runtimeInformation = await eventHubClient.GetRuntimeInformationAsync();
await eventHubClient.CloseAsync();
return(runtimeInformation);
}
private async Task <List <EventHubReceiver> > GetReceivers()
{
EventHubConsumerGroup consumerGroup = eventHubClient.GetConsumerGroup(consumerGroupName);
EventHubRuntimeInformation runtimeInfo = await eventHubClient.GetRuntimeInformationAsync();
var receivers = new List <EventHubReceiver>();
foreach (string partition in runtimeInfo.PartitionIds)
{
receivers.Add(await consumerGroup.CreateReceiverAsync(partition, EndOfStream));
}
return(receivers);
}
/// <summary>
/// Get partition Ids from eventhub
/// </summary>
/// <returns></returns>
protected virtual async Task <string[]> GetPartitionIdsAsync()
{
#if NETSTANDARD
EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync();
return(runtimeInfo.PartitionIds);
#else
NamespaceManager namespaceManager = NamespaceManager.CreateFromConnectionString(hubSettings.ConnectionString);
EventHubDescription hubDescription = await namespaceManager.GetEventHubAsync(hubSettings.Path);
return(hubDescription.PartitionIds);
#endif
}
/// <summary>
/// Get runtime info of the fake event hub.
/// </summary>
/// <returns></returns>
public virtual Task <EventHubRuntimeInformation> GetRuntimeInformationAsync()
{
EventHubRuntimeInformation ehri = new EventHubRuntimeInformation();
ehri.PartitionCount = this.partitionCount;
ehri.PartitionIds = new string[this.partitionCount];
for (int i = 0; i < this.partitionCount; i++)
{
ehri.PartitionIds[i] = i.ToString();
}
ehri.Path = this.csb.EntityPath;
EventProcessorEventSource.Current.Message($"MOCK GetRuntimeInformationAsync for {ehri.Path}");
return(Task.FromResult <EventHubRuntimeInformation>(ehri));
}
public async Task <PartitionReceiver> GetEventHubReceiverAsync(string name, string consumerGroup = "$default")
{
EventHubClient client = await GetEventHubClientAsync(name);
EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync();
if (runtimeInfo.PartitionCount > 1)
{
throw new InvalidOperationException("No support for more then 1 partition");
}
var partitionId = runtimeInfo.PartitionIds[0];
return(client.CreateReceiver(consumerGroup, partitionId, EventPosition.FromStart()));
}
private async Task <PartitionReceiver> CreateEventHubReceiver(string deviceName)
{
var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
{
EntityPath = Configuration.IoTHub.EventHubCompatibleName
};
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
EventHubRuntimeInformation eventHubRuntime = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false);
int eventHubPartitionsCount = eventHubRuntime.PartitionCount;
string partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount);
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
return(eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5)));
}
/// <summary>
/// Retrieves information about an Event Hub, including the number of partitions present
/// and their identifiers.
/// </summary>
///
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>The set of information for the Event Hub that this client is associated with.</returns>
///
public override async Task <EventHubProperties> GetPropertiesAsync(CancellationToken cancellationToken)
{
try
{
EventHubRuntimeInformation runtimeInformation = await TrackOneClient.GetRuntimeInformationAsync().ConfigureAwait(false);
return(new EventHubProperties
(
TrackOneClient.EventHubName,
runtimeInformation.CreatedAt,
runtimeInformation.PartitionIds
));
}
catch (TrackOne.EventHubsException ex)
{
throw ex.MapToTrackTwoException();
}
}
private async Task CreateReceivers()
{
EventHubRuntimeInformation runtimeInfo = await _hubClient.GetRuntimeInformationAsync();
foreach (string partitionId in runtimeInfo.PartitionIds)
{
if (_partitionIds == null || _partitionIds.Contains(partitionId))
{
PartitionReceiver receiver = _hubClient.CreateReceiver(
_consumerGroupName ?? PartitionReceiver.DefaultConsumerGroupName,
partitionId,
(await _state.GetPartitionOffset(partitionId)),
false);
_partitonReceivers.Add(receiver);
}
}
}
/// <summary>
/// Starts receiving messages from Event Hub
/// </summary>
public async Task StartAsync(Func <IEnumerable <EventMessage>, string, Task> onMessagesReceived, string consumerGroup, CancellationToken token)
{
_onMessagesReceived = onMessagesReceived;
Func <string, string, Task <PartitionReceiver> > setupTask = _checkpointer == null ?
(Func <string, string, Task <PartitionReceiver> >)SetupFromEndOfStreamAsync :
(Func <string, string, Task <PartitionReceiver> >)SetupWithCheckpointAsync;
EventHubRuntimeInformation info = await _client.GetRuntimeInformationAsync();
foreach (string partition in info.PartitionIds)
{
PartitionReceiver receiver = await setupTask(consumerGroup, partition);
AzureEventHubReader reader = new AzureEventHubReader(receiver, RefreshPartitionReaderAsync, _onMessagesReceived);
_readers.Add(reader);
reader.Open(token);
}
}
public async Task <EventHubsTriggerMetrics> GetMetricsAsync()
{
EventHubsTriggerMetrics metrics = new EventHubsTriggerMetrics();
EventHubRuntimeInformation runtimeInfo = null;
try
{
runtimeInfo = await _client.Value.GetRuntimeInformationAsync();
}
catch (NotSupportedException e)
{
_logger.LogWarning($"EventHubs Trigger does not support NotificationHubs. Error: {e.Message}");
return(metrics);
}
catch (MessagingEntityNotFoundException)
{
_logger.LogWarning($"EventHub '{_eventHubName}' was not found.");
return(metrics);
}
catch (TimeoutException e)
{
_logger.LogWarning($"Encountered a timeout while checking EventHub '{_eventHubName}'. Error: {e.Message}");
return(metrics);
}
catch (Exception e)
{
_logger.LogWarning($"Encountered an exception while checking EventHub '{_eventHubName}'. Error: {e.Message}");
return(metrics);
}
// Get the PartitionRuntimeInformation for all partitions
_logger.LogInformation($"Querying partition information for {runtimeInfo.PartitionCount} partitions.");
var tasks = new Task <EventHubPartitionRuntimeInformation> [runtimeInfo.PartitionCount];
for (int i = 0; i < runtimeInfo.PartitionCount; i++)
{
tasks[i] = _client.Value.GetPartitionRuntimeInformationAsync(i.ToString());
}
await Task.WhenAll(tasks);
return(await CreateTriggerMetrics(tasks.Select(t => t.Result).ToList()));
}
private async void ControlarServos_Load(object sender, EventArgs e)
{
HabilitaControles(false);
connectionString = new EventHubsConnectionStringBuilder(new Uri(s_eventHubsCompatibleEndpoint), s_eventHubsCompatiblePath, s_iotHubSasKeyName, s_iotHubSasKey);
s_eventHubClient = EventHubClient.CreateFromConnectionString(connectionString.ToString());
// Create a PartitionReciever for each partition on the hub.
runtimeInfo = await s_eventHubClient.GetRuntimeInformationAsync();
d2cPartitions = runtimeInfo.PartitionIds;
cts = new CancellationTokenSource();
HabilitaControles(true);
foreach (string partition in d2cPartitions)
{
await ReceiveMessagesFromDeviceAsync(partition, cts.Token);
}
}
/// <summary>
/// Creates an EventHubReceiver connected though ServiceBus
/// with AMQP protocol
/// </summary>
/// <param name="eventHubClient">Event Hub Client</param>
/// <param name="partitionKey">IOT Hub Partition Key</param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task <EventHubReceiver> GetEventHubReceiverAsync(EventHubClient eventHubClient, long partitionKey, IReliableDictionary <string, string> offsetDictionary)
{
//Get the EventHubRuntimeInfo
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[partitionKey];
EventHubReceiver eventHubReceiver;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult =
await offsetDictionary.TryGetValueAsync(tx, OffsetKey, LockMode.Default);
if (offsetResult.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RoutingService partition Key {partitionKey} connecting to IoT Hub partition ID {eventHubPartitionId} with offset {offsetResult.Value}");
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, offsetResult.Value);
}
else
{
var dateTimeUtc = DateTime.UtcNow;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RoutingService partition Key {partitionKey} connecting to IoT Hub partition ID {eventHubPartitionId} with offset {dateTimeUtc}");
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, dateTimeUtc);
}
}
return(eventHubReceiver);
}
private async Task <IEnumerable <PartitionReceiver> > CreateReceiversAsync()
{
var receivers = new List <PartitionReceiver>();
EventHubRuntimeInformation runtimeInfo = await _hubClient.GetRuntimeInformationAsync();
foreach (string partitionId in runtimeInfo.PartitionIds)
{
if (_partitionIds == null || _partitionIds.Contains(partitionId))
{
PartitionReceiver receiver = _hubClient.CreateReceiver(
_consumerGroupName ?? PartitionReceiver.DefaultConsumerGroupName,
partitionId,
(await _state.GetPartitionPosition(partitionId)));
receivers.Add(receiver);
}
}
return(receivers);
}
static async Task MonitorEventHub()
{
EventHubRuntimeInformation runtimeInfo = await _client.GetRuntimeInformationAsync();
Console.WriteLine($"Partition count: {runtimeInfo.PartitionCount}");
long[] sequenceNumbers = new long[runtimeInfo.PartitionCount];
for (;;)
{
var tasks = new Task <PartitionRuntimeInformation> [runtimeInfo.PartitionCount];
for (int i = 0; i < runtimeInfo.PartitionCount; i++)
{
tasks[i] = _client.GetPartitionRuntimeInformationAsync(i.ToString());
}
// Wait for all of them
await Task.WhenAll(tasks);
for (int i = 0; i < runtimeInfo.PartitionCount; i++)
{
PartitionRuntimeInformation part = tasks[i].GetAwaiter().GetResult();
Console.WriteLine($"BeginSequenceNumber: {part.BeginSequenceNumber}, LastEnqueuedOffset: {part.LastEnqueuedOffset}, LastEnqueuedSequenceNumber: {part.LastEnqueuedSequenceNumber}, LastEnqueuedTime: {part.LastEnqueuedTimeUtc}");
if (sequenceNumbers[i] != 0)
{
if (part.LastEnqueuedSequenceNumber != sequenceNumbers[i])
{
Console.WriteLine($"Sequence when up by {part.LastEnqueuedSequenceNumber - sequenceNumbers[i]} for partition {i}!");
}
}
sequenceNumbers[i] = part.LastEnqueuedSequenceNumber;
}
System.Threading.Thread.Sleep(5000);
}
}
private static async Task <IReadOnlyList <PartitionReceiver> > GetReceivers(EventHubClient eventHubClient, string consumerGroupName)
{
EventHubRuntimeInformation runtimeInformation = await eventHubClient.GetRuntimeInformationAsync();
var receivers = new List <PartitionReceiver>();
foreach (string partitionId in runtimeInformation.PartitionIds)
{
EventHubPartitionRuntimeInformation partitionRuntimeInformation =
await eventHubClient.GetPartitionRuntimeInformationAsync(partitionId);
PartitionReceiver receiver = eventHubClient.CreateReceiver(
consumerGroupName,
partitionId,
partitionRuntimeInformation.LastEnqueuedOffset,
offsetInclusive: false);
receivers.Add(receiver);
}
return(receivers);
}
public async Task <IActionResult> Post([FromRoute] string name, [FromBody] IngestionApplicationParams parameters)
{
// Determine the number of IoT Hub partitions.
// The ingestion service will be created with the same number of partitions.
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(parameters.IotHubConnectionString, "messages/events");
EventHubRuntimeInformation eventHubInfo = await eventHubClient.GetRuntimeInformationAsync();
// Application parameters are passed to the Ingestion application instance.
NameValueCollection appInstanceParameters = new NameValueCollection();
appInstanceParameters["IotHubConnectionString"] = parameters.IotHubConnectionString;
appInstanceParameters["ReplicaSetSize"] = parameters.ReplicaSetSize.ToString();
ApplicationDescription application = new ApplicationDescription(
new Uri($"{Names.IngestionApplicationPrefix}/{name}"),
Names.IngestionApplicationTypeName,
parameters.Version,
appInstanceParameters);
// Create a named application instance
await this.fabricClient.ApplicationManager.CreateApplicationAsync(application, this.operationTimeout, this.appLifetime.ApplicationStopping);
// Next, create named instances of the services that run in the application.
ServiceUriBuilder serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.IngestionRouterServiceName);
StatefulServiceDescription service = new StatefulServiceDescription()
{
ApplicationName = application.ApplicationName,
HasPersistedState = true,
MinReplicaSetSize = parameters.ReplicaSetSize, // Previous 3 for 5 node cluster and 1 for 1 node cluster
TargetReplicaSetSize = parameters.ReplicaSetSize,
PartitionSchemeDescription = new UniformInt64RangePartitionSchemeDescription(eventHubInfo.PartitionCount, 0, eventHubInfo.PartitionCount - 1),
ServiceName = serviceNameUriBuilder.Build(),
ServiceTypeName = Names.IngestionRouterServiceTypeName
};
await this.fabricClient.ServiceManager.CreateServiceAsync(service, this.operationTimeout, this.appLifetime.ApplicationStopping);
return(this.Ok());
}
private async Task <IEnumerable <EventData> > GetEventData()
{
EventHubRuntimeInformation runtimeInformation = _eventHubClient.GetRuntimeInformation();
List <EventData> dataRecieved = new List <EventData>();
foreach (string partitionId in runtimeInformation.PartitionIds)
{
string partitionOffset = await _stateHandler.GetOffset(partitionId);
//EventHubReceiver eventReceiver = _eventHubClient.GetDefaultConsumerGroup().CreateReceiver(partitionId, partitionOffset);
EventHubReceiver eventReceiver = _eventHubClient.GetConsumerGroup("mojo").CreateReceiver(partitionId, partitionOffset);
EventData eventData = eventReceiver.Receive();
await _stateHandler.SetOffset(eventData.Offset, partitionId);
await eventReceiver.CloseAsync();
dataRecieved.Add(eventData);
}
return(dataRecieved);
}
/// <summary>
/// Gets the IDs of event hub partitions
/// </summary>
/// <returns></returns>
public async Task <IEnumerable <string> > GetPartitionIdsAsync()
{
EventHubRuntimeInformation info = await _hubClient.GetRuntimeInformationAsync();
return(info.PartitionIds);
}
/// <summary>
/// Creates an EventHubReceiver from the given connection sting and partition key.
/// The Reliable Dictionaries are used to create a receiver from wherever the service last left off,
/// or from the current date/time if it's the first time the service is coming up.
/// </summary>
/// <param name="connectionString"></param>
/// <param name="servicePartitionKey"></param>
/// <param name="epochDictionary"></param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync(
string connectionString,
long servicePartitionKey,
IReliableDictionary <string, long> epochDictionary,
IReliableDictionary <string, string> offsetDictionary, string processOnlyFutureEvents)
{
// EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP.
ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString);
connectionStringBuilder.TransportType = TransportType.Amqp;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - connecting to IoT Hub at {0}",
String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString())));
// A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory.
// This gives each partition its own dedicated TCP connection to IoT Hub.
MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString());
EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events");
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
EventHubReceiver eventHubReceiver = null;
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey];
int retryCount = 1;
while (retryCount > 0)
{
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync -Creating EventHub listener on partition {eventHubPartitionId} with offset {offsetResult.Value}");
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch);
}
else
{
// first time this service is running so there is no offset value yet.
// start with the current time.
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - Creating EventHub listener on partition {eventHubPartitionId} with offset time now{DateTime.UtcNow} - Starting service");
if (processOnlyFutureEvents.Equals("yes"))
{
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch);
}
else
{
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, newEpoch);
}
}
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
retryCount = 0;
}
}
catch (TimeoutException te)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - TimeoutException Retry Count#{retryCount} : Message=[{te.ToString()}]");
retryCount++;
await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills);
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricTransientException : Message=[{fte.ToString()}]");
retryCount++;
await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills);
}
catch (FabricNotPrimaryException fnpe)
{
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricNotPrimaryException Exception - Message=[{fnpe}]");
retryCount = 0;
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - General Exception - Message=[{ex}]");
retryCount = 0;
}
}
return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory));
}
/// <summary>
/// Creates an EventHubReceiver from the given connection sting and partition key.
/// The Reliable Dictionaries are used to create a receiver from wherever the service last left off,
/// or from the current date/time if it's the first time the service is coming up.
/// </summary>
/// <param name="connectionString"></param>
/// <param name="servicePartitionKey"></param>
/// <param name="epochDictionary"></param>
/// <param name="offsetDictionary"></param>
/// <returns></returns>
private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync(
string connectionString,
long servicePartitionKey,
IReliableDictionary <string, long> epochDictionary,
IReliableDictionary <string, string> offsetDictionary)
{
// EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP.
ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString);
connectionStringBuilder.TransportType = TransportType.Amqp;
ServiceEventSource.Current.ServiceMessage(
this.Context,
"RouterService connecting to IoT Hub at {0}",
String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString())));
// A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory.
// This gives each partition its own dedicated TCP connection to IoT Hub.
MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString());
EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events");
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
EventHubReceiver eventHubReceiver;
// Get an IoT Hub partition ID that corresponds to this partition's low key.
// This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1.
// For example, given an IoT Hub with 32 partitions, this service should be created with:
// partition count = 32
// partition range = 0..31
string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey];
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default);
ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update);
long newEpoch = epochResult.HasValue
? epochResult.Value + 1
: 0;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1}",
eventHubPartitionId,
offsetResult.Value);
eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch);
}
else
{
// first time this service is running so there is no offset value yet.
// start with the current time.
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Creating EventHub listener on partition {0} with offset {1} - Starting service",
eventHubPartitionId,
DateTime.UtcNow);
eventHubReceiver =
await
eventHubClient.GetDefaultConsumerGroup()
.CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch);
}
// epoch is recorded each time the service fails over or restarts.
await epochDictionary.SetAsync(tx, "epoch", newEpoch);
await tx.CommitAsync();
}
return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory));
}
/// <summary>
/// Get partition Ids from eventhub
/// </summary>
/// <returns></returns>
protected virtual async Task <string[]> GetPartitionIdsAsync()
{
EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync();
return(runtimeInfo.PartitionIds);
}
private async Task <PartitionReceiver> ConnectToEventHubAsync(string eventHubConnectionString, string hubName,
IReliableDictionary <string, string> streamOffsetDictionary)
{
var eventHubHelper = new EventHubHelper();
var eventHubClient = eventHubHelper.CreatEventHubClientIfExist(eventHubConnectionString, hubName);
EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync();
PartitionReceiver partitionReceiver = null;
string[] partitionIds = eventHubRuntimeInfo.PartitionIds;
_eventHubPartitionId = await GetMatchingEventHubPartitionId(partitionIds);
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> offsetResult = await streamOffsetDictionary.TryGetValueAsync(tx, Names.HubStreamOffSetKey);
EventPosition eventPosition;
if (offsetResult.HasValue)
{
// continue where the service left off before the last failover or restart.
eventPosition = EventPosition.FromSequenceNumber(long.Parse(offsetResult.Value));
}
else
{
// first time this service is running so there is no offset value yet. start with the current time.
// Load from database sequence number
eventPosition = await LoadEventPositionFromDatabaseAsync() ??
EventPosition.FromEnqueuedTime(DateTime.UtcNow);
//EventPosition.FromEnqueuedTime(DateTime.UtcNow.Subtract(TimeSpan.FromHours(5)));//EventPosition.FromEnqueuedTime(DateTime.UtcNow);
if (eventPosition.SequenceNumber != null)
{
_latestSequenceNumber = eventPosition.SequenceNumber.Value;
await streamOffsetDictionary.SetAsync(tx, Names.HubStreamOffSetKey, eventPosition.SequenceNumber.ToString());
await tx.CommitAsync();
}
}
ServiceEventSource.Current.ServiceMessage(this.Context, "Creating EventHub listener on partition {0} with SequenceNumber {1}",
_eventHubPartitionId, eventPosition.SequenceNumber);
partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, _eventHubPartitionId, eventPosition);
}
}
catch (Exception e)
{
//ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ConnectToEventHubAsync met exception= ( {e} )");
string err = $"RouterService ConnectToEventHubAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId} .";
//ServiceEventSource.Current.CriticalError("RouterService", err);
throw;
}
return(partitionReceiver);
}
