public static void CreateListenerPalAndReceiveMessages() { var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString) { EntityPath = Configuration.IoTHub.EventHubCompatibleName }; var eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString()); EventHubRuntimeInformation eventRuntimeInformation = eventHubClient.GetRuntimeInformationAsync().Result; int eventHubPartitionsCount = eventRuntimeInformation.PartitionCount; string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup; foreach (string partitionId in eventRuntimeInformation.PartitionIds) { try { PartitionReceiver receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.Now.AddMinutes(-s_lookbackTimeInMinutes.TotalMinutes)); s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from {s_lookbackTimeInMinutes}"); Task.Run(async() => { while (true) { IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false); ProcessEventData(eventDatas); } }); } catch (EventHubsException ex) { s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}"); } } }
public static EventHubRuntimeInformation GetEventHubRuntimeInfo(AmqpMessage amqpMessage) { EventHubRuntimeInformation eventHubRuntimeInformation = null; object obj = null; if (MessageConverter.TryGetNetObjectFromAmqpObject(amqpMessage.ValueBody.Value, MappingType.MessageBody, out obj)) { Dictionary <string, object> strs = obj as Dictionary <string, object>; if (strs != null) { eventHubRuntimeInformation = new EventHubRuntimeInformation(); object obj1 = null; if (strs.TryGetValue("name", out obj1)) { eventHubRuntimeInformation.Path = (string)obj1; } if (strs.TryGetValue("partition_count", out obj1)) { eventHubRuntimeInformation.PartitionCount = (int)obj1; } if (strs.TryGetValue("created_at", out obj1)) { eventHubRuntimeInformation.CreatedAt = (DateTime)obj1; } if (strs.TryGetValue("partition_ids", out obj1)) { eventHubRuntimeInformation.PartitionIds = (string[])obj1; } } } return(eventHubRuntimeInformation); }
private static async Task CheckpointLatest(EventHubRuntimeInformation runtimeInformation) { var leasedPartitionIds = new HashSet <string>(); var stopwatch = new Stopwatch(); EventProcessorHost processorHost = GetEventProcessorHost(); var factory = new CheckpointerFactory(() => new Checkpointer { OnOpen = partitionContext => leasedPartitionIds.Add(partitionContext.PartitionId), OnCheckpoint = eventData => stopwatch.Restart() }); await processorHost.RegisterEventProcessorFactoryAsync(factory); do { await Task.Delay(10); }while (leasedPartitionIds.Count < runtimeInformation.PartitionCount); stopwatch.Start(); do { await Task.Delay(10); }while (stopwatch.Elapsed.TotalSeconds < 1.0); stopwatch.Stop(); await processorHost.UnregisterEventProcessorAsync(); }
public static void CreateListenerPalAndReceiveMessages() { var eventHubClient = EventHubClient.CreateFromConnectionString(Configuration.IoTHub.ConnectionString, "messages/events"); EventHubRuntimeInformation eventRuntimeInformation = eventHubClient.GetRuntimeInformation(); string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup; foreach (string partitionId in eventRuntimeInformation.PartitionIds) { try { EventHubReceiver receiver = eventHubClient.GetConsumerGroup(consumerGroupName).CreateReceiver(partitionId, DateTime.Now.AddMinutes(-s_lookbackTimeInMinutes.TotalMinutes)); s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from past {s_lookbackTimeInMinutes}"); Task.Run(async() => { while (true) { IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false); ProcessEventData(eventDatas); } }); } catch (QuotaExceededException ex) { s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}"); } } }
protected override IEnumerator <IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .AsyncStep> GetAsyncSteps() { if (!this.managementLink.FaultTolerantLink.TryGetOpenedObject(out this.requestResponseLink)) { AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult getEventHubRuntimeInfoAsyncResult = this; IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .BeginCall beginCall = (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, TimeSpan t, AsyncCallback c, object s) => thisPtr.managementLink.FaultTolerantLink.BeginGetInstance(t, c, s); yield return(getEventHubRuntimeInfoAsyncResult.CallAsync(beginCall, (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, IAsyncResult r) => thisPtr.requestResponseLink = thisPtr.managementLink.FaultTolerantLink.EndGetInstance(r), IteratorAsyncResult <TIteratorAsyncResult> .ExceptionPolicy.Transfer)); } AmqpMessage applicationProperty = AmqpMessage.Create(); applicationProperty.ApplicationProperties = new ApplicationProperties(); applicationProperty.ApplicationProperties.Map["name"] = this.eventHubPath; applicationProperty.ApplicationProperties.Map["operation"] = "READ"; applicationProperty.ApplicationProperties.Map["type"] = "com.microsoft:eventhub"; AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult getEventHubRuntimeInfoAsyncResult1 = this; IteratorAsyncResult <AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult> .BeginCall beginCall1 = (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, TimeSpan t, AsyncCallback c, object s) => thisPtr.requestResponseLink.BeginRequest(applicationProperty, t, c, s); yield return(getEventHubRuntimeInfoAsyncResult1.CallAsync(beginCall1, (AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult thisPtr, IAsyncResult r) => thisPtr.getRuntimeInfoResponse = thisPtr.requestResponseLink.EndRequest(r), IteratorAsyncResult <TIteratorAsyncResult> .ExceptionPolicy.Transfer)); int item = (int)this.getRuntimeInfoResponse.ApplicationProperties.Map["status-code"]; string str = (string)this.getRuntimeInfoResponse.ApplicationProperties.Map["status-description"]; if (item != 200) { base.Complete(AmqpManagementLink.GetEventHubRuntimeInfoAsyncResult.ConvertToException(item, str)); } else { this.Result = MessageConverter.GetEventHubRuntimeInfo(this.getRuntimeInfoResponse); } }
public async Task <IList <EventData> > GetMessagesFromAllPartitions(DateTime startTime, int maxPerPartition = 10, int waitTimeSecs = 5) { var messages = new List <EventData>(); EventHubRuntimeInformation rtInfo = await this.eventHubClient.GetRuntimeInformationAsync(); foreach (string partition in rtInfo.PartitionIds) { PartitionReceiver partitionReceiver = this.eventHubClient.CreateReceiver( PartitionReceiver.DefaultConsumerGroupName, partition, startTime); // Retry a few times to make sure we get all expected messages. for (int i = 0; i < 3; i++) { IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs)); if (events != null) { messages.AddRange(events); } if (i < 3) { await Task.Delay(TimeSpan.FromSeconds(5)); } } await partitionReceiver.CloseAsync(); } return(messages); }
private static void listenForAsyncCommandUpdates(string requestId) { var connectionStringBuilder = new EventHubsConnectionStringBuilder(EVENTHUB_CONNECTION_STRING); var partitionReceivers = new List <PartitionReceiver>(); var eventHubClient = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); var eventhubReceiverThreads = new List <Thread>(); EventHubRuntimeInformation runtimeInformation = eventHubClient.GetRuntimeInformationAsync().Result; //spawn a thread per partition to receive over that partition Console.WriteLine("Setting up EventHub listeners for updates to the async method..."); foreach (string partitionId in runtimeInformation.PartitionIds) { var partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, EventPosition.FromEnqueuedTime(DateTime.Now)); Thread partitionReceiverThread = new Thread(() => receive(partitionReceiver, requestId)); eventhubReceiverThreads.Add(partitionReceiverThread); partitionReceiverThread.Start(); } Console.WriteLine("EventHub listener now listening for updates containing the request id " + requestId); Console.WriteLine(); //Wait for user to enter a key to exit the program Console.ReadLine(); foreach (Thread partitionReceiverThread in eventhubReceiverThreads) { partitionReceiverThread.Interrupt(); } }
public async Task <IActionResult> Post([FromRoute] string name, [FromBody] EventsProcessorApplicationParams parameters) { // Determine the number of IoT Hub partitions. // The events processing service will be created with the same number of partitions. EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(parameters.IotHubConnectionString, "messages/events"); EventHubRuntimeInformation eventHubInfo = await eventHubClient.GetRuntimeInformationAsync(); // Application parameters are passed to the Events Processing application instance. NameValueCollection appInstanceParameters = new NameValueCollection(); appInstanceParameters["IotHubConnectionString"] = parameters.IotHubConnectionString; appInstanceParameters["IotHubProcessOnlyFutureEvents"] = parameters.IotHubProcessOnlyFutureEvents; appInstanceParameters["PublishDataServiceURLs"] = parameters.PublishDataServiceURLs; ApplicationDescription application = new ApplicationDescription( new Uri($"{Names.EventsProcessorApplicationPrefix}/{name}"), Names.EventsProcessorApplicationTypeName, parameters.Version, appInstanceParameters); // Create a named application instance await this.fabricClient.ApplicationManager.CreateApplicationAsync(application, this.operationTimeout, this.appLifetime.ApplicationStopping); // Next, create named instances of the services that run in the application. ServiceUriBuilder serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.EventsProcessorRouterServiceName); StatefulServiceDescription service = new StatefulServiceDescription() { ApplicationName = application.ApplicationName, HasPersistedState = true, MinReplicaSetSize = 3, TargetReplicaSetSize = 3, PartitionSchemeDescription = new UniformInt64RangePartitionSchemeDescription(eventHubInfo.PartitionCount, 0, eventHubInfo.PartitionCount - 1), ServiceName = serviceNameUriBuilder.Build(), ServiceTypeName = Names.EventsProcessorRouterServiceTypeName }; await this.fabricClient.ServiceManager.CreateServiceAsync(service, this.operationTimeout, this.appLifetime.ApplicationStopping); if (parameters.PublishDataServiceURLs != null && parameters.PublishDataServiceURLs.Length > 0) { // Next, create named instances of the services that run in the application. serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.EventsProcessorExtenderServiceName); StatelessServiceDescription extenderService = new StatelessServiceDescription() { ApplicationName = application.ApplicationName, InstanceCount = 1, PartitionSchemeDescription = new SingletonPartitionSchemeDescription(), ServiceName = serviceNameUriBuilder.Build(), ServiceTypeName = Names.EventsProcessorExtenderServiceTypeName }; await this.fabricClient.ServiceManager.CreateServiceAsync(extenderService, this.operationTimeout, this.appLifetime.ApplicationStopping); } return(this.Ok()); }
private static async Task <EventHubRuntimeInformation> GetRuntimeInformation() { var eventHubClient = EventHubClient.CreateFromConnectionString(_eventHubConnectionString); EventHubRuntimeInformation runtimeInformation = await eventHubClient.GetRuntimeInformationAsync(); await eventHubClient.CloseAsync(); return(runtimeInformation); }
private async Task <List <EventHubReceiver> > GetReceivers() { EventHubConsumerGroup consumerGroup = eventHubClient.GetConsumerGroup(consumerGroupName); EventHubRuntimeInformation runtimeInfo = await eventHubClient.GetRuntimeInformationAsync(); var receivers = new List <EventHubReceiver>(); foreach (string partition in runtimeInfo.PartitionIds) { receivers.Add(await consumerGroup.CreateReceiverAsync(partition, EndOfStream)); } return(receivers); }
/// <summary> /// Get partition Ids from eventhub /// </summary> /// <returns></returns> protected virtual async Task <string[]> GetPartitionIdsAsync() { #if NETSTANDARD EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync(); return(runtimeInfo.PartitionIds); #else NamespaceManager namespaceManager = NamespaceManager.CreateFromConnectionString(hubSettings.ConnectionString); EventHubDescription hubDescription = await namespaceManager.GetEventHubAsync(hubSettings.Path); return(hubDescription.PartitionIds); #endif }
/// <summary> /// Get runtime info of the fake event hub. /// </summary> /// <returns></returns> public virtual Task <EventHubRuntimeInformation> GetRuntimeInformationAsync() { EventHubRuntimeInformation ehri = new EventHubRuntimeInformation(); ehri.PartitionCount = this.partitionCount; ehri.PartitionIds = new string[this.partitionCount]; for (int i = 0; i < this.partitionCount; i++) { ehri.PartitionIds[i] = i.ToString(); } ehri.Path = this.csb.EntityPath; EventProcessorEventSource.Current.Message($"MOCK GetRuntimeInformationAsync for {ehri.Path}"); return(Task.FromResult <EventHubRuntimeInformation>(ehri)); }
public async Task <PartitionReceiver> GetEventHubReceiverAsync(string name, string consumerGroup = "$default") { EventHubClient client = await GetEventHubClientAsync(name); EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync(); if (runtimeInfo.PartitionCount > 1) { throw new InvalidOperationException("No support for more then 1 partition"); } var partitionId = runtimeInfo.PartitionIds[0]; return(client.CreateReceiver(consumerGroup, partitionId, EventPosition.FromStart())); }
private async Task <PartitionReceiver> CreateEventHubReceiver(string deviceName) { var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString) { EntityPath = Configuration.IoTHub.EventHubCompatibleName }; EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString()); EventHubRuntimeInformation eventHubRuntime = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false); int eventHubPartitionsCount = eventHubRuntime.PartitionCount; string partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount); string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup; return(eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5))); }
/// <summary> /// Retrieves information about an Event Hub, including the number of partitions present /// and their identifiers. /// </summary> /// /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>The set of information for the Event Hub that this client is associated with.</returns> /// public override async Task <EventHubProperties> GetPropertiesAsync(CancellationToken cancellationToken) { try { EventHubRuntimeInformation runtimeInformation = await TrackOneClient.GetRuntimeInformationAsync().ConfigureAwait(false); return(new EventHubProperties ( TrackOneClient.EventHubName, runtimeInformation.CreatedAt, runtimeInformation.PartitionIds )); } catch (TrackOne.EventHubsException ex) { throw ex.MapToTrackTwoException(); } }
private async Task CreateReceivers() { EventHubRuntimeInformation runtimeInfo = await _hubClient.GetRuntimeInformationAsync(); foreach (string partitionId in runtimeInfo.PartitionIds) { if (_partitionIds == null || _partitionIds.Contains(partitionId)) { PartitionReceiver receiver = _hubClient.CreateReceiver( _consumerGroupName ?? PartitionReceiver.DefaultConsumerGroupName, partitionId, (await _state.GetPartitionOffset(partitionId)), false); _partitonReceivers.Add(receiver); } } }
/// <summary> /// Starts receiving messages from Event Hub /// </summary> public async Task StartAsync(Func <IEnumerable <EventMessage>, string, Task> onMessagesReceived, string consumerGroup, CancellationToken token) { _onMessagesReceived = onMessagesReceived; Func <string, string, Task <PartitionReceiver> > setupTask = _checkpointer == null ? (Func <string, string, Task <PartitionReceiver> >)SetupFromEndOfStreamAsync : (Func <string, string, Task <PartitionReceiver> >)SetupWithCheckpointAsync; EventHubRuntimeInformation info = await _client.GetRuntimeInformationAsync(); foreach (string partition in info.PartitionIds) { PartitionReceiver receiver = await setupTask(consumerGroup, partition); AzureEventHubReader reader = new AzureEventHubReader(receiver, RefreshPartitionReaderAsync, _onMessagesReceived); _readers.Add(reader); reader.Open(token); } }
public async Task <EventHubsTriggerMetrics> GetMetricsAsync() { EventHubsTriggerMetrics metrics = new EventHubsTriggerMetrics(); EventHubRuntimeInformation runtimeInfo = null; try { runtimeInfo = await _client.Value.GetRuntimeInformationAsync(); } catch (NotSupportedException e) { _logger.LogWarning($"EventHubs Trigger does not support NotificationHubs. Error: {e.Message}"); return(metrics); } catch (MessagingEntityNotFoundException) { _logger.LogWarning($"EventHub '{_eventHubName}' was not found."); return(metrics); } catch (TimeoutException e) { _logger.LogWarning($"Encountered a timeout while checking EventHub '{_eventHubName}'. Error: {e.Message}"); return(metrics); } catch (Exception e) { _logger.LogWarning($"Encountered an exception while checking EventHub '{_eventHubName}'. Error: {e.Message}"); return(metrics); } // Get the PartitionRuntimeInformation for all partitions _logger.LogInformation($"Querying partition information for {runtimeInfo.PartitionCount} partitions."); var tasks = new Task <EventHubPartitionRuntimeInformation> [runtimeInfo.PartitionCount]; for (int i = 0; i < runtimeInfo.PartitionCount; i++) { tasks[i] = _client.Value.GetPartitionRuntimeInformationAsync(i.ToString()); } await Task.WhenAll(tasks); return(await CreateTriggerMetrics(tasks.Select(t => t.Result).ToList())); }
private async void ControlarServos_Load(object sender, EventArgs e) { HabilitaControles(false); connectionString = new EventHubsConnectionStringBuilder(new Uri(s_eventHubsCompatibleEndpoint), s_eventHubsCompatiblePath, s_iotHubSasKeyName, s_iotHubSasKey); s_eventHubClient = EventHubClient.CreateFromConnectionString(connectionString.ToString()); // Create a PartitionReciever for each partition on the hub. runtimeInfo = await s_eventHubClient.GetRuntimeInformationAsync(); d2cPartitions = runtimeInfo.PartitionIds; cts = new CancellationTokenSource(); HabilitaControles(true); foreach (string partition in d2cPartitions) { await ReceiveMessagesFromDeviceAsync(partition, cts.Token); } }
/// <summary> /// Creates an EventHubReceiver connected though ServiceBus /// with AMQP protocol /// </summary> /// <param name="eventHubClient">Event Hub Client</param> /// <param name="partitionKey">IOT Hub Partition Key</param> /// <param name="offsetDictionary"></param> /// <returns></returns> private async Task <EventHubReceiver> GetEventHubReceiverAsync(EventHubClient eventHubClient, long partitionKey, IReliableDictionary <string, string> offsetDictionary) { //Get the EventHubRuntimeInfo EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync(); // Get an IoT Hub partition ID that corresponds to this partition's low key. // This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1. // For example, given an IoT Hub with 32 partitions, this service should be created with: // partition count = 32 // partition range = 0..31 string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[partitionKey]; EventHubReceiver eventHubReceiver; using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, OffsetKey, LockMode.Default); if (offsetResult.HasValue) { ServiceEventSource.Current.ServiceMessage( this.Context, $"RoutingService partition Key {partitionKey} connecting to IoT Hub partition ID {eventHubPartitionId} with offset {offsetResult.Value}"); eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup() .CreateReceiverAsync(eventHubPartitionId, offsetResult.Value); } else { var dateTimeUtc = DateTime.UtcNow; ServiceEventSource.Current.ServiceMessage( this.Context, $"RoutingService partition Key {partitionKey} connecting to IoT Hub partition ID {eventHubPartitionId} with offset {dateTimeUtc}"); eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup() .CreateReceiverAsync(eventHubPartitionId, dateTimeUtc); } } return(eventHubReceiver); }
private async Task <IEnumerable <PartitionReceiver> > CreateReceiversAsync() { var receivers = new List <PartitionReceiver>(); EventHubRuntimeInformation runtimeInfo = await _hubClient.GetRuntimeInformationAsync(); foreach (string partitionId in runtimeInfo.PartitionIds) { if (_partitionIds == null || _partitionIds.Contains(partitionId)) { PartitionReceiver receiver = _hubClient.CreateReceiver( _consumerGroupName ?? PartitionReceiver.DefaultConsumerGroupName, partitionId, (await _state.GetPartitionPosition(partitionId))); receivers.Add(receiver); } } return(receivers); }
static async Task MonitorEventHub() { EventHubRuntimeInformation runtimeInfo = await _client.GetRuntimeInformationAsync(); Console.WriteLine($"Partition count: {runtimeInfo.PartitionCount}"); long[] sequenceNumbers = new long[runtimeInfo.PartitionCount]; for (;;) { var tasks = new Task <PartitionRuntimeInformation> [runtimeInfo.PartitionCount]; for (int i = 0; i < runtimeInfo.PartitionCount; i++) { tasks[i] = _client.GetPartitionRuntimeInformationAsync(i.ToString()); } // Wait for all of them await Task.WhenAll(tasks); for (int i = 0; i < runtimeInfo.PartitionCount; i++) { PartitionRuntimeInformation part = tasks[i].GetAwaiter().GetResult(); Console.WriteLine($"BeginSequenceNumber: {part.BeginSequenceNumber}, LastEnqueuedOffset: {part.LastEnqueuedOffset}, LastEnqueuedSequenceNumber: {part.LastEnqueuedSequenceNumber}, LastEnqueuedTime: {part.LastEnqueuedTimeUtc}"); if (sequenceNumbers[i] != 0) { if (part.LastEnqueuedSequenceNumber != sequenceNumbers[i]) { Console.WriteLine($"Sequence when up by {part.LastEnqueuedSequenceNumber - sequenceNumbers[i]} for partition {i}!"); } } sequenceNumbers[i] = part.LastEnqueuedSequenceNumber; } System.Threading.Thread.Sleep(5000); } }
private static async Task <IReadOnlyList <PartitionReceiver> > GetReceivers(EventHubClient eventHubClient, string consumerGroupName) { EventHubRuntimeInformation runtimeInformation = await eventHubClient.GetRuntimeInformationAsync(); var receivers = new List <PartitionReceiver>(); foreach (string partitionId in runtimeInformation.PartitionIds) { EventHubPartitionRuntimeInformation partitionRuntimeInformation = await eventHubClient.GetPartitionRuntimeInformationAsync(partitionId); PartitionReceiver receiver = eventHubClient.CreateReceiver( consumerGroupName, partitionId, partitionRuntimeInformation.LastEnqueuedOffset, offsetInclusive: false); receivers.Add(receiver); } return(receivers); }
public async Task <IActionResult> Post([FromRoute] string name, [FromBody] IngestionApplicationParams parameters) { // Determine the number of IoT Hub partitions. // The ingestion service will be created with the same number of partitions. EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(parameters.IotHubConnectionString, "messages/events"); EventHubRuntimeInformation eventHubInfo = await eventHubClient.GetRuntimeInformationAsync(); // Application parameters are passed to the Ingestion application instance. NameValueCollection appInstanceParameters = new NameValueCollection(); appInstanceParameters["IotHubConnectionString"] = parameters.IotHubConnectionString; appInstanceParameters["ReplicaSetSize"] = parameters.ReplicaSetSize.ToString(); ApplicationDescription application = new ApplicationDescription( new Uri($"{Names.IngestionApplicationPrefix}/{name}"), Names.IngestionApplicationTypeName, parameters.Version, appInstanceParameters); // Create a named application instance await this.fabricClient.ApplicationManager.CreateApplicationAsync(application, this.operationTimeout, this.appLifetime.ApplicationStopping); // Next, create named instances of the services that run in the application. ServiceUriBuilder serviceNameUriBuilder = new ServiceUriBuilder(application.ApplicationName.ToString(), Names.IngestionRouterServiceName); StatefulServiceDescription service = new StatefulServiceDescription() { ApplicationName = application.ApplicationName, HasPersistedState = true, MinReplicaSetSize = parameters.ReplicaSetSize, // Previous 3 for 5 node cluster and 1 for 1 node cluster TargetReplicaSetSize = parameters.ReplicaSetSize, PartitionSchemeDescription = new UniformInt64RangePartitionSchemeDescription(eventHubInfo.PartitionCount, 0, eventHubInfo.PartitionCount - 1), ServiceName = serviceNameUriBuilder.Build(), ServiceTypeName = Names.IngestionRouterServiceTypeName }; await this.fabricClient.ServiceManager.CreateServiceAsync(service, this.operationTimeout, this.appLifetime.ApplicationStopping); return(this.Ok()); }
private async Task <IEnumerable <EventData> > GetEventData() { EventHubRuntimeInformation runtimeInformation = _eventHubClient.GetRuntimeInformation(); List <EventData> dataRecieved = new List <EventData>(); foreach (string partitionId in runtimeInformation.PartitionIds) { string partitionOffset = await _stateHandler.GetOffset(partitionId); //EventHubReceiver eventReceiver = _eventHubClient.GetDefaultConsumerGroup().CreateReceiver(partitionId, partitionOffset); EventHubReceiver eventReceiver = _eventHubClient.GetConsumerGroup("mojo").CreateReceiver(partitionId, partitionOffset); EventData eventData = eventReceiver.Receive(); await _stateHandler.SetOffset(eventData.Offset, partitionId); await eventReceiver.CloseAsync(); dataRecieved.Add(eventData); } return(dataRecieved); }
/// <summary> /// Gets the IDs of event hub partitions /// </summary> /// <returns></returns> public async Task <IEnumerable <string> > GetPartitionIdsAsync() { EventHubRuntimeInformation info = await _hubClient.GetRuntimeInformationAsync(); return(info.PartitionIds); }
/// <summary> /// Creates an EventHubReceiver from the given connection sting and partition key. /// The Reliable Dictionaries are used to create a receiver from wherever the service last left off, /// or from the current date/time if it's the first time the service is coming up. /// </summary> /// <param name="connectionString"></param> /// <param name="servicePartitionKey"></param> /// <param name="epochDictionary"></param> /// <param name="offsetDictionary"></param> /// <returns></returns> private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync( string connectionString, long servicePartitionKey, IReliableDictionary <string, long> epochDictionary, IReliableDictionary <string, string> offsetDictionary, string processOnlyFutureEvents) { // EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP. ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString); connectionStringBuilder.TransportType = TransportType.Amqp; ServiceEventSource.Current.ServiceMessage( this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - connecting to IoT Hub at {0}", String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString()))); // A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory. // This gives each partition its own dedicated TCP connection to IoT Hub. MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString()); EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events"); EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync(); EventHubReceiver eventHubReceiver = null; // Get an IoT Hub partition ID that corresponds to this partition's low key. // This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1. // For example, given an IoT Hub with 32 partitions, this service should be created with: // partition count = 32 // partition range = 0..31 string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey]; int retryCount = 1; while (retryCount > 0) { try { using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default); ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update); long newEpoch = epochResult.HasValue ? epochResult.Value + 1 : 0; if (offsetResult.HasValue) { // continue where the service left off before the last failover or restart. ServiceEventSource.Current.ServiceMessage( this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync -Creating EventHub listener on partition {eventHubPartitionId} with offset {offsetResult.Value}"); eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch); } else { // first time this service is running so there is no offset value yet. // start with the current time. ServiceEventSource.Current.ServiceMessage( this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - Creating EventHub listener on partition {eventHubPartitionId} with offset time now{DateTime.UtcNow} - Starting service"); if (processOnlyFutureEvents.Equals("yes")) { eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup() .CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch); } else { eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup() .CreateReceiverAsync(eventHubPartitionId, newEpoch); } } // epoch is recorded each time the service fails over or restarts. await epochDictionary.SetAsync(tx, "epoch", newEpoch); await tx.CommitAsync(); retryCount = 0; } } catch (TimeoutException te) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - TimeoutException Retry Count#{retryCount} : Message=[{te.ToString()}]"); retryCount++; await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills); } catch (FabricTransientException fte) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricTransientException : Message=[{fte.ToString()}]"); retryCount++; await Task.Delay(global::Iot.Common.Names.IoTHubRetryWaitIntervalsInMills); } catch (FabricNotPrimaryException fnpe) { ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - FabricNotPrimaryException Exception - Message=[{fnpe}]"); retryCount = 0; } catch (Exception ex) { ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService - {ServiceUniqueId} - ConnectToIoTHubAsync - General Exception - Message=[{ex}]"); retryCount = 0; } } return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory)); }
/// <summary> /// Creates an EventHubReceiver from the given connection sting and partition key. /// The Reliable Dictionaries are used to create a receiver from wherever the service last left off, /// or from the current date/time if it's the first time the service is coming up. /// </summary> /// <param name="connectionString"></param> /// <param name="servicePartitionKey"></param> /// <param name="epochDictionary"></param> /// <param name="offsetDictionary"></param> /// <returns></returns> private async Task <Tuple <EventHubReceiver, MessagingFactory> > ConnectToIoTHubAsync( string connectionString, long servicePartitionKey, IReliableDictionary <string, long> epochDictionary, IReliableDictionary <string, string> offsetDictionary) { // EventHubs doesn't support NetMessaging, so ensure the transport type is AMQP. ServiceBusConnectionStringBuilder connectionStringBuilder = new ServiceBusConnectionStringBuilder(connectionString); connectionStringBuilder.TransportType = TransportType.Amqp; ServiceEventSource.Current.ServiceMessage( this.Context, "RouterService connecting to IoT Hub at {0}", String.Join(",", connectionStringBuilder.Endpoints.Select(x => x.ToString()))); // A new MessagingFactory is created here so that each partition of this service will have its own MessagingFactory. // This gives each partition its own dedicated TCP connection to IoT Hub. MessagingFactory messagingFactory = MessagingFactory.CreateFromConnectionString(connectionStringBuilder.ToString()); EventHubClient eventHubClient = messagingFactory.CreateEventHubClient("messages/events"); EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync(); EventHubReceiver eventHubReceiver; // Get an IoT Hub partition ID that corresponds to this partition's low key. // This assumes that this service has a partition count 'n' that is equal to the IoT Hub partition count and a partition range of 0..n-1. // For example, given an IoT Hub with 32 partitions, this service should be created with: // partition count = 32 // partition range = 0..31 string eventHubPartitionId = eventHubRuntimeInfo.PartitionIds[servicePartitionKey]; using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <string> offsetResult = await offsetDictionary.TryGetValueAsync(tx, "offset", LockMode.Default); ConditionalValue <long> epochResult = await epochDictionary.TryGetValueAsync(tx, "epoch", LockMode.Update); long newEpoch = epochResult.HasValue ? epochResult.Value + 1 : 0; if (offsetResult.HasValue) { // continue where the service left off before the last failover or restart. ServiceEventSource.Current.ServiceMessage( this.Context, "Creating EventHub listener on partition {0} with offset {1}", eventHubPartitionId, offsetResult.Value); eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup().CreateReceiverAsync(eventHubPartitionId, offsetResult.Value, newEpoch); } else { // first time this service is running so there is no offset value yet. // start with the current time. ServiceEventSource.Current.ServiceMessage( this.Context, "Creating EventHub listener on partition {0} with offset {1} - Starting service", eventHubPartitionId, DateTime.UtcNow); eventHubReceiver = await eventHubClient.GetDefaultConsumerGroup() .CreateReceiverAsync(eventHubPartitionId, DateTime.UtcNow, newEpoch); } // epoch is recorded each time the service fails over or restarts. await epochDictionary.SetAsync(tx, "epoch", newEpoch); await tx.CommitAsync(); } return(new Tuple <EventHubReceiver, MessagingFactory>(eventHubReceiver, messagingFactory)); }
/// <summary> /// Get partition Ids from eventhub /// </summary> /// <returns></returns> protected virtual async Task <string[]> GetPartitionIdsAsync() { EventHubRuntimeInformation runtimeInfo = await client.GetRuntimeInformationAsync(); return(runtimeInfo.PartitionIds); }
private async Task <PartitionReceiver> ConnectToEventHubAsync(string eventHubConnectionString, string hubName, IReliableDictionary <string, string> streamOffsetDictionary) { var eventHubHelper = new EventHubHelper(); var eventHubClient = eventHubHelper.CreatEventHubClientIfExist(eventHubConnectionString, hubName); EventHubRuntimeInformation eventHubRuntimeInfo = await eventHubClient.GetRuntimeInformationAsync(); PartitionReceiver partitionReceiver = null; string[] partitionIds = eventHubRuntimeInfo.PartitionIds; _eventHubPartitionId = await GetMatchingEventHubPartitionId(partitionIds); try { using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <string> offsetResult = await streamOffsetDictionary.TryGetValueAsync(tx, Names.HubStreamOffSetKey); EventPosition eventPosition; if (offsetResult.HasValue) { // continue where the service left off before the last failover or restart. eventPosition = EventPosition.FromSequenceNumber(long.Parse(offsetResult.Value)); } else { // first time this service is running so there is no offset value yet. start with the current time. // Load from database sequence number eventPosition = await LoadEventPositionFromDatabaseAsync() ?? EventPosition.FromEnqueuedTime(DateTime.UtcNow); //EventPosition.FromEnqueuedTime(DateTime.UtcNow.Subtract(TimeSpan.FromHours(5)));//EventPosition.FromEnqueuedTime(DateTime.UtcNow); if (eventPosition.SequenceNumber != null) { _latestSequenceNumber = eventPosition.SequenceNumber.Value; await streamOffsetDictionary.SetAsync(tx, Names.HubStreamOffSetKey, eventPosition.SequenceNumber.ToString()); await tx.CommitAsync(); } } ServiceEventSource.Current.ServiceMessage(this.Context, "Creating EventHub listener on partition {0} with SequenceNumber {1}", _eventHubPartitionId, eventPosition.SequenceNumber); partitionReceiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, _eventHubPartitionId, eventPosition); } } catch (Exception e) { //ServiceEventSource.Current.ServiceMessage(this.Context, $"RouterService ConnectToEventHubAsync met exception= ( {e} )"); string err = $"RouterService ConnectToEventHubAsync met exception, exception type={e.GetType().Name}, exception= {e.Message}, at partition ={Context.PartitionId} ."; //ServiceEventSource.Current.CriticalError("RouterService", err); throw; } return(partitionReceiver); }