public void GetMonitor_ReturnsExpectedValue() { var functionId = "FunctionId"; var eventHubName = "EventHubName"; var consumerGroup = "ConsumerGroup"; var storageUri = new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/"); var testLogger = new TestLogger("Test"); var listener = new EventHubListener( functionId, eventHubName, consumerGroup, "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=", "DefaultEndpointsProtocol=https;AccountName=EventHubScaleMonitorFakeTestAccount;AccountKey=ABCDEFG;EndpointSuffix=core.windows.net", new Mock <ITriggeredFunctionExecutor>(MockBehavior.Strict).Object, null, false, new EventHubOptions(), testLogger, new Mock <CloudBlobContainer>(MockBehavior.Strict, new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/azure-webjobs-eventhub")).Object); IScaleMonitor scaleMonitor = listener.GetMonitor(); Assert.Equal(typeof(EventHubsScaleMonitor), scaleMonitor.GetType()); Assert.Equal($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id); var scaleMonitor2 = listener.GetMonitor(); Assert.Same(scaleMonitor, scaleMonitor2); }
public void GetMonitor_ReturnsExpectedValue() { var functionId = "FunctionId"; var eventHubName = "EventHubName"; var consumerGroup = "ConsumerGroup"; var host = new EventProcessorHost(consumerGroup, "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=", eventHubName, new EventProcessorOptions(), 3, null); var consumerClientMock = new Mock <IEventHubConsumerClient>(); consumerClientMock.SetupGet(c => c.ConsumerGroup).Returns(consumerGroup); consumerClientMock.SetupGet(c => c.EventHubName).Returns(eventHubName); var listener = new EventHubListener( functionId, Mock.Of <ITriggeredFunctionExecutor>(), host, false, consumerClientMock.Object, Mock.Of <BlobsCheckpointStore>(), new EventHubOptions(), Mock.Of <LoggerFactory>()); IScaleMonitor scaleMonitor = listener.GetMonitor(); Assert.AreEqual(typeof(EventHubsScaleMonitor), scaleMonitor.GetType()); Assert.AreEqual($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id); var scaleMonitor2 = listener.GetMonitor(); Assert.AreSame(scaleMonitor, scaleMonitor2); }
public void Dispose_StopsTheProcessor() { var functionId = "FunctionId"; var eventHubName = "EventHubName"; var consumerGroup = "ConsumerGroup"; var host = new Mock <EventProcessorHost>(consumerGroup, "Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=", eventHubName, new EventProcessorOptions(), 3, null); host.Setup(h => h.StopProcessingAsync(CancellationToken.None)).Returns(Task.CompletedTask); var consumerClientMock = new Mock <IEventHubConsumerClient>(); consumerClientMock.SetupGet(c => c.ConsumerGroup).Returns(consumerGroup); consumerClientMock.SetupGet(c => c.EventHubName).Returns(eventHubName); var listener = new EventHubListener( functionId, Mock.Of <ITriggeredFunctionExecutor>(), host.Object, false, consumerClientMock.Object, Mock.Of <BlobsCheckpointStore>(), new EventHubOptions(), Mock.Of <LoggerFactory>()); (listener as IListener).Dispose(); host.Verify(h => h.StopProcessingAsync(CancellationToken.None), Times.Once); (listener as IListener).Cancel(); host.Verify(h => h.StopProcessingAsync(CancellationToken.None), Times.Exactly(2)); }
public Task<ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context) { if (context == null) { throw new ArgumentNullException("context"); } ParameterInfo parameter = context.Parameter; EventHubTriggerAttribute attribute = parameter.GetCustomAttribute<EventHubTriggerAttribute>(inherit: false); if (attribute == null) { return Task.FromResult<ITriggerBinding>(null); } string eventHubName = attribute.EventHubName; string resolvedName = _nameResolver.ResolveWholeString(eventHubName); var eventHostListener = _eventHubConfig.GetEventProcessorHost(resolvedName); var options = _eventHubConfig.GetOptions(); var hooks = new EventHubTriggerBindingStrategy(); Func<ListenerFactoryContext, bool, Task<IListener>> createListener = (factoryContext, singleDispatch) => { IListener listener = new EventHubListener(factoryContext.Executor, eventHostListener, options, singleDispatch); return Task.FromResult(listener); }; ITriggerBinding binding = BindingFactory.GetTriggerBinding<EventData, EventHubTriggerInput>(hooks, parameter, _converterManager, createListener); return Task.FromResult<ITriggerBinding>(binding); }
private static void RunTestSeq(EventHubConfig config, string messageBody, int iterations) { var _eventHubListener = new EventHubListener(config); var _eventHubSender = new EventHubSender(config, messageBody, iterations); _eventHubListener.StartListening().GetAwaiter().GetResult(); _eventHubSender.SendMessages().GetAwaiter().GetResult(); }
/// <summary> /// The method that registers Event Hub listeners and assigns them to a Receive event. When I receive an event from the event hub listener, I trigger the callbackURL /// </summary> /// <param name="triggerId"></param> /// <param name="triggerInput"></param> public async Task RegisterTrigger(EventHubInput input) { var client = EventHubClient.CreateFromConnectionString(input.eventHubConnectionString, input.eventHubName); EventHubConsumerGroup group = String.IsNullOrEmpty(input.consumerGroup) ? client.GetDefaultConsumerGroup() : client.GetConsumerGroup(input.consumerGroup); string[] partitions; //If they specified partitions, iterate over their list to only listen to the partitions they specified if (!String.IsNullOrEmpty(input.eventHubPartitionList)) { partitions = input.eventHubPartitionList.Split(','); } //If they left it blank, create a list to listen to all partitions else { partitions = new string[client.GetRuntimeInformation().PartitionCount]; for (int x = 0; x < partitions.Length; x++) { partitions[x] = x.ToString(); } } List <CancellationTokenSource> tokenSources = new List <CancellationTokenSource>(); //For ever partition I should listen to, create a thread with a listener on it foreach (var p in partitions) { p.Trim(); var Receiver = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow); EventHubListener listener = new EventHubListener(Receiver); //Register the event. When I Receive a message, call the method to trigger the logic app listener.MessageReceived += (sender, e) => TriggerLogicApps(input.callbackUrl, e); var ts = new CancellationTokenSource(); CancellationToken ct = ts.Token; listener.StartListening(ct); tokenSources.Add(ts); } //Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners _store[input.callbackUrl] = tokenSources; }
private async Task EventHubListener_CreatesCheckpointStrategy(int batchCheckpointFrequency, int expected) { var iterations = 100; var strategy = EventHubListener.CreateCheckpointStrategy(batchCheckpointFrequency); var checkpoints = 0; Func <Task> checkpoint = () => { checkpoints++; return(Task.CompletedTask); }; for (int i = 0; i < iterations; i++) { await strategy(checkpoint); } Assert.Equal(expected, checkpoints); }
public void TestMethodTest() { var testClass = new EventHubListener(); var jsonData = new { Name = "Olawale Adewoyin", Address = "Somewhere", Age = 56 }; var testData = JsonConvert.SerializeObject(jsonData); var testArray = Encoding.ASCII.GetBytes(testData); var events = new EventData[1]; events[0] = new EventData(testArray); var result = testClass.EventHubListenerTest(events, out var message, logger.Object); }
/// <summary> /// The method that registers Event Hub listeners and assigns them to a recieve event. When I receive an event from the event hub listener, I trigger the callbackURL /// </summary> /// <param name="triggerId"></param> /// <param name="triggerInput"></param> /// <returns></returns> public async Task RegisterTrigger(string triggerId, TriggerInput <EventHubInput, EventHubMessage> triggerInput) { var client = EventHubClient.CreateFromConnectionString(triggerInput.inputs.eventHubConnectionString, triggerInput.inputs.eventHubName); EventHubConsumerGroup group = client.GetConsumerGroup(triggerInput.inputs.consumerGroup); string[] partitions; //If they specified partitions, iterate over their list to only listen to the partitions they specified if (!String.IsNullOrEmpty(triggerInput.inputs.eventHubPartitionList)) { partitions = triggerInput.inputs.eventHubPartitionList.Split(','); } //If they left it blank, create a list to listen to all partitions else { partitions = new string[client.GetRuntimeInformation().PartitionCount]; for (int x = 0; x < partitions.Length; x++) { partitions[x] = x.ToString(); } } //For ever partition I should listen to, create a thread with a listener on it foreach (var p in partitions) { p.Trim(); var reciever = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow); EventHubListener listener = new EventHubListener(reciever); //Register the event. When I recieve a message, call the method to trigger the logic app listener.MessageReceived += (sender, e) => sendTrigger(sender, e, Runtime.FromAppSettings(), triggerInput.GetCallback()); listener.StartListening(); } //Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners _store[triggerId] = true; }
private void EventHubListener_Throws_IfInvalidCheckpointStrategy(int batchCheckpointFrequency) { var exc = Assert.Throws <InvalidOperationException>(() => EventHubListener.CreateCheckpointStrategy(batchCheckpointFrequency)); Assert.Equal("Batch checkpoint frequency must be larger than 0.", exc.Message); }
/// <summary> /// The method that registers Event Hub listeners and assigns them to a recieve event. When I receive an event from the event hub listener, I trigger the callbackURL /// </summary> /// <param name="triggerId"></param> /// <param name="triggerInput"></param> /// <returns></returns> public async Task RegisterTrigger(string triggerId, TriggerInput<EventHubInput, EventHubMessage> triggerInput) { var client = EventHubClient.CreateFromConnectionString(triggerInput.inputs.eventHubConnectionString, triggerInput.inputs.eventHubName); EventHubConsumerGroup group = client.GetDefaultConsumerGroup(); //client.GetConsumerGroup(triggerInput.inputs.consumerGroup); string[] partitions; //If they specified partitions, iterate over their list to only listen to the partitions they specified if (!String.IsNullOrEmpty(triggerInput.inputs.eventHubPartitionList)) { partitions = triggerInput.inputs.eventHubPartitionList.Split(','); } //If they left it blank, create a list to listen to all partitions else { partitions = new string[client.GetRuntimeInformation().PartitionCount]; for(int x = 0; x < partitions.Length; x++) { partitions[x] = x.ToString(); } } //For ever partition I should listen to, create a thread with a listener on it foreach (var p in partitions) { p.Trim(); var reciever = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow); EventHubListener listener = new EventHubListener(reciever); //Register the event. When I recieve a message, call the method to trigger the logic app listener.MessageReceived += (sender, e) => sendTrigger(sender, e, Runtime.FromAppSettings(), triggerInput.GetCallback()); listener.StartListening(); } //Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners _store[triggerId] = true; }
/// <summary> /// The method that registers Event Hub listeners and assigns them to a Receive event. When I receive an event from the event hub listener, I trigger the callbackURL /// </summary> /// <param name="triggerId"></param> /// <param name="triggerInput"></param> public async Task RegisterTrigger(EventHubInput input) { var client = EventHubClient.CreateFromConnectionString(input.eventHubConnectionString, input.eventHubName); EventHubConsumerGroup group = String.IsNullOrEmpty(input.consumerGroup) ? client.GetDefaultConsumerGroup() : client.GetConsumerGroup(input.consumerGroup); string[] partitions; //If they specified partitions, iterate over their list to only listen to the partitions they specified if (!String.IsNullOrEmpty(input.eventHubPartitionList)) { partitions = input.eventHubPartitionList.Split(','); } //If they left it blank, create a list to listen to all partitions else { partitions = new string[client.GetRuntimeInformation().PartitionCount]; for (int x = 0; x < partitions.Length; x++) { partitions[x] = x.ToString(); } } List<CancellationTokenSource> tokenSources = new List<CancellationTokenSource>(); //For ever partition I should listen to, create a thread with a listener on it foreach (var p in partitions) { p.Trim(); var Receiver = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow); EventHubListener listener = new EventHubListener(Receiver); //Register the event. When I Receive a message, call the method to trigger the logic app listener.MessageReceived += (sender, e) => TriggerLogicApps(input.callbackUrl, e); var ts = new CancellationTokenSource(); CancellationToken ct = ts.Token; listener.StartListening(ct); tokenSources.Add(ts); } //Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners _store[input.callbackUrl] = tokenSources; }
private ICommunicationListener CreateEventHubListener() { /*********************************************************************/ // 1: Create a state factory /*********************************************************************/ /* * as you read events from event hub you need to maintain a state (combination of partition + offset in the partition). this state is * a marker to last event you have read (in order to avoid to duplicates). * * default state factory uses Service Fabric Reliable State to store state as in IReliableDictoinary<string, DefaultPartitionState> * * You can implement different PartitionStateFactory & Partition State (example: using the listener in a stateless service). */ var factory = new DefaultPartitionStateFactory(this.StateManager); // other overloads allows you to override default dictionary name // and/or entries prefix (for example if you want to use one dictionary for multiple listeners). /*********************************************************************/ // 2) listener options /*********************************************************************/ /* * because listener distribute loads among service fabric partitions of the current service * you will need to identify the current partition id + Service name * * or if you intent to manually map a single service fabric service partitions to event hub partition * then you can use this CTOR, the listener will not query other partitions of the service hence it won't need a fabriClient instance * var options = new EventHubListenerOptions(currentSFPartition); */ var currentSFPartition = this.Context.PartitionId.ToString(); var currentServiceName = this.Context.ServiceName; // if you have restricted access to cluster then you will need a to create a fabric client (with security) and pass it to the options var options = new EventHubListenerOptions(currentSFPartition, currentServiceName); // set the processor options.Processor = new myEventProcessor(); // this is a class that implements IEventHubEventsProcessor /* * Supported Modes: * * 1- SafeDistribute: Maps 1..n Event Hub Partitions to 1 Service Fabric partition. * will throw an exception if service fabric partitions are > Event hub partition * * * 2- Distribute: maps 1..n event hub partitions to 1 service fabric partition * if service fabric partitions are > event hub partitions, the remaining partitions will not * get any distribution (i.e Event processor will not be created on them). * * * * 3- OneToOne: maps one to one event hub partition: Service Fabric Partition * Service Fabric partition has to = Event Hub Partitions (or an exception will be thrown) * * * * 4- Single: maps a single event hub partition to a single service fabric partition. * Event Hub communication listener will expect a supplied valid event hub partition id */ options.ListenerMode = EventHubListenerMode.SafeDistribute; // Set the Partition State Factor options.StateFactory = factory; // Set Connection String options.EventHubConnectionString = mEventHubConnectionString; // Set Event Hub Name options.EventHubName = mEventHubName; // optionally set consumer group name (not setting it will default to "default consumer group") //options.EventHubConsumerGroupName = "BE01"; /************************************* * Addtional Options * * 1) * Use event hub Epochs (details: http://blogs.msdn.com/b/gyan/archive/2014/09/02/event-hubs-receiver-epoch.aspx) * options.UseEpoch(myEpochValue) * * * * (the below helps if you switching from receivers running elsewhere to Service Fabric) * (details for the below: https://msdn.microsoft.com/en-us/library/azure/microsoft.servicebus.messaging.eventhubconsumergroup.createreceiver.aspx) * * 2) * Events only newer than UTC Date * options.StartWithEventsNewerThan(DateTime.UtcNow) * * 3) * Specific Offset * options.StartWithOffset("MyOffSet", bInclusive) * * * you can also chain the call * options.UseEpoch(longEpoch) * .StartWithOffset(myoffset); **************************************/ // override default batch size //options.BatchSize = 100; mEventHubListener = new EventHubListener(options); return(mEventHubListener); }