public void GetMonitor_ReturnsExpectedValue()
{
var functionId = "FunctionId";
var eventHubName = "EventHubName";
var consumerGroup = "ConsumerGroup";
var storageUri = new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/");
var testLogger = new TestLogger("Test");
var listener = new EventHubListener(
functionId,
eventHubName,
consumerGroup,
"Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=",
"DefaultEndpointsProtocol=https;AccountName=EventHubScaleMonitorFakeTestAccount;AccountKey=ABCDEFG;EndpointSuffix=core.windows.net",
new Mock <ITriggeredFunctionExecutor>(MockBehavior.Strict).Object,
null,
false,
new EventHubOptions(),
testLogger,
new Mock <CloudBlobContainer>(MockBehavior.Strict, new Uri("https://eventhubsteststorageaccount.blob.core.windows.net/azure-webjobs-eventhub")).Object);
IScaleMonitor scaleMonitor = listener.GetMonitor();
Assert.Equal(typeof(EventHubsScaleMonitor), scaleMonitor.GetType());
Assert.Equal($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id);
var scaleMonitor2 = listener.GetMonitor();
Assert.Same(scaleMonitor, scaleMonitor2);
}
public void GetMonitor_ReturnsExpectedValue()
{
var functionId = "FunctionId";
var eventHubName = "EventHubName";
var consumerGroup = "ConsumerGroup";
var host = new EventProcessorHost(consumerGroup,
"Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=",
eventHubName,
new EventProcessorOptions(),
3, null);
var consumerClientMock = new Mock <IEventHubConsumerClient>();
consumerClientMock.SetupGet(c => c.ConsumerGroup).Returns(consumerGroup);
consumerClientMock.SetupGet(c => c.EventHubName).Returns(eventHubName);
var listener = new EventHubListener(
functionId,
Mock.Of <ITriggeredFunctionExecutor>(),
host,
false,
consumerClientMock.Object,
Mock.Of <BlobsCheckpointStore>(),
new EventHubOptions(),
Mock.Of <LoggerFactory>());
IScaleMonitor scaleMonitor = listener.GetMonitor();
Assert.AreEqual(typeof(EventHubsScaleMonitor), scaleMonitor.GetType());
Assert.AreEqual($"{functionId}-EventHubTrigger-{eventHubName}-{consumerGroup}".ToLower(), scaleMonitor.Descriptor.Id);
var scaleMonitor2 = listener.GetMonitor();
Assert.AreSame(scaleMonitor, scaleMonitor2);
}
public void Dispose_StopsTheProcessor()
{
var functionId = "FunctionId";
var eventHubName = "EventHubName";
var consumerGroup = "ConsumerGroup";
var host = new Mock <EventProcessorHost>(consumerGroup,
"Endpoint=sb://test.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=abc123=",
eventHubName,
new EventProcessorOptions(),
3, null);
host.Setup(h => h.StopProcessingAsync(CancellationToken.None)).Returns(Task.CompletedTask);
var consumerClientMock = new Mock <IEventHubConsumerClient>();
consumerClientMock.SetupGet(c => c.ConsumerGroup).Returns(consumerGroup);
consumerClientMock.SetupGet(c => c.EventHubName).Returns(eventHubName);
var listener = new EventHubListener(
functionId,
Mock.Of <ITriggeredFunctionExecutor>(),
host.Object,
false,
consumerClientMock.Object,
Mock.Of <BlobsCheckpointStore>(),
new EventHubOptions(),
Mock.Of <LoggerFactory>());
(listener as IListener).Dispose();
host.Verify(h => h.StopProcessingAsync(CancellationToken.None), Times.Once);
(listener as IListener).Cancel();
host.Verify(h => h.StopProcessingAsync(CancellationToken.None), Times.Exactly(2));
}
public Task<ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
ParameterInfo parameter = context.Parameter;
EventHubTriggerAttribute attribute = parameter.GetCustomAttribute<EventHubTriggerAttribute>(inherit: false);
if (attribute == null)
{
return Task.FromResult<ITriggerBinding>(null);
}
string eventHubName = attribute.EventHubName;
string resolvedName = _nameResolver.ResolveWholeString(eventHubName);
var eventHostListener = _eventHubConfig.GetEventProcessorHost(resolvedName);
var options = _eventHubConfig.GetOptions();
var hooks = new EventHubTriggerBindingStrategy();
Func<ListenerFactoryContext, bool, Task<IListener>> createListener =
(factoryContext, singleDispatch) =>
{
IListener listener = new EventHubListener(factoryContext.Executor, eventHostListener, options, singleDispatch);
return Task.FromResult(listener);
};
ITriggerBinding binding = BindingFactory.GetTriggerBinding<EventData, EventHubTriggerInput>(hooks, parameter, _converterManager, createListener);
return Task.FromResult<ITriggerBinding>(binding);
}
private static void RunTestSeq(EventHubConfig config, string messageBody, int iterations)
{
var _eventHubListener = new EventHubListener(config);
var _eventHubSender = new EventHubSender(config, messageBody, iterations);
_eventHubListener.StartListening().GetAwaiter().GetResult();
_eventHubSender.SendMessages().GetAwaiter().GetResult();
}
/// <summary>
/// The method that registers Event Hub listeners and assigns them to a Receive event. When I receive an event from the event hub listener, I trigger the callbackURL
/// </summary>
/// <param name="triggerId"></param>
/// <param name="triggerInput"></param>
public async Task RegisterTrigger(EventHubInput input)
{
var client = EventHubClient.CreateFromConnectionString(input.eventHubConnectionString, input.eventHubName);
EventHubConsumerGroup group = String.IsNullOrEmpty(input.consumerGroup) ?
client.GetDefaultConsumerGroup() : client.GetConsumerGroup(input.consumerGroup);
string[] partitions;
//If they specified partitions, iterate over their list to only listen to the partitions they specified
if (!String.IsNullOrEmpty(input.eventHubPartitionList))
{
partitions = input.eventHubPartitionList.Split(',');
}
//If they left it blank, create a list to listen to all partitions
else
{
partitions = new string[client.GetRuntimeInformation().PartitionCount];
for (int x = 0; x < partitions.Length; x++)
{
partitions[x] = x.ToString();
}
}
List <CancellationTokenSource> tokenSources = new List <CancellationTokenSource>();
//For ever partition I should listen to, create a thread with a listener on it
foreach (var p in partitions)
{
p.Trim();
var Receiver = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow);
EventHubListener listener = new EventHubListener(Receiver);
//Register the event. When I Receive a message, call the method to trigger the logic app
listener.MessageReceived += (sender, e) => TriggerLogicApps(input.callbackUrl, e);
var ts = new CancellationTokenSource();
CancellationToken ct = ts.Token;
listener.StartListening(ct);
tokenSources.Add(ts);
}
//Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners
_store[input.callbackUrl] = tokenSources;
}
private async Task EventHubListener_CreatesCheckpointStrategy(int batchCheckpointFrequency, int expected)
{
var iterations = 100;
var strategy = EventHubListener.CreateCheckpointStrategy(batchCheckpointFrequency);
var checkpoints = 0;
Func <Task> checkpoint = () =>
{
checkpoints++;
return(Task.CompletedTask);
};
for (int i = 0; i < iterations; i++)
{
await strategy(checkpoint);
}
Assert.Equal(expected, checkpoints);
}
public void TestMethodTest()
{
var testClass = new EventHubListener();
var jsonData = new
{
Name = "Olawale Adewoyin",
Address = "Somewhere",
Age = 56
};
var testData = JsonConvert.SerializeObject(jsonData);
var testArray = Encoding.ASCII.GetBytes(testData);
var events = new EventData[1];
events[0] = new EventData(testArray);
var result = testClass.EventHubListenerTest(events, out var message, logger.Object);
}
/// <summary>
/// The method that registers Event Hub listeners and assigns them to a recieve event. When I receive an event from the event hub listener, I trigger the callbackURL
/// </summary>
/// <param name="triggerId"></param>
/// <param name="triggerInput"></param>
/// <returns></returns>
public async Task RegisterTrigger(string triggerId, TriggerInput <EventHubInput, EventHubMessage> triggerInput)
{
var client = EventHubClient.CreateFromConnectionString(triggerInput.inputs.eventHubConnectionString, triggerInput.inputs.eventHubName);
EventHubConsumerGroup group = client.GetConsumerGroup(triggerInput.inputs.consumerGroup);
string[] partitions;
//If they specified partitions, iterate over their list to only listen to the partitions they specified
if (!String.IsNullOrEmpty(triggerInput.inputs.eventHubPartitionList))
{
partitions = triggerInput.inputs.eventHubPartitionList.Split(',');
}
//If they left it blank, create a list to listen to all partitions
else
{
partitions = new string[client.GetRuntimeInformation().PartitionCount];
for (int x = 0; x < partitions.Length; x++)
{
partitions[x] = x.ToString();
}
}
//For ever partition I should listen to, create a thread with a listener on it
foreach (var p in partitions)
{
p.Trim();
var reciever = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow);
EventHubListener listener = new EventHubListener(reciever);
//Register the event. When I recieve a message, call the method to trigger the logic app
listener.MessageReceived += (sender, e) => sendTrigger(sender, e, Runtime.FromAppSettings(), triggerInput.GetCallback());
listener.StartListening();
}
//Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners
_store[triggerId] = true;
}
private void EventHubListener_Throws_IfInvalidCheckpointStrategy(int batchCheckpointFrequency)
{
var exc = Assert.Throws <InvalidOperationException>(() => EventHubListener.CreateCheckpointStrategy(batchCheckpointFrequency));
Assert.Equal("Batch checkpoint frequency must be larger than 0.", exc.Message);
}
/// <summary>
/// The method that registers Event Hub listeners and assigns them to a recieve event. When I receive an event from the event hub listener, I trigger the callbackURL
/// </summary>
/// <param name="triggerId"></param>
/// <param name="triggerInput"></param>
/// <returns></returns>
public async Task RegisterTrigger(string triggerId, TriggerInput<EventHubInput, EventHubMessage> triggerInput)
{
var client = EventHubClient.CreateFromConnectionString(triggerInput.inputs.eventHubConnectionString, triggerInput.inputs.eventHubName);
EventHubConsumerGroup group = client.GetDefaultConsumerGroup(); //client.GetConsumerGroup(triggerInput.inputs.consumerGroup);
string[] partitions;
//If they specified partitions, iterate over their list to only listen to the partitions they specified
if (!String.IsNullOrEmpty(triggerInput.inputs.eventHubPartitionList))
{
partitions = triggerInput.inputs.eventHubPartitionList.Split(',');
}
//If they left it blank, create a list to listen to all partitions
else
{
partitions = new string[client.GetRuntimeInformation().PartitionCount];
for(int x = 0; x < partitions.Length; x++)
{
partitions[x] = x.ToString();
}
}
//For ever partition I should listen to, create a thread with a listener on it
foreach (var p in partitions)
{
p.Trim();
var reciever = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow);
EventHubListener listener = new EventHubListener(reciever);
//Register the event. When I recieve a message, call the method to trigger the logic app
listener.MessageReceived += (sender, e) => sendTrigger(sender, e, Runtime.FromAppSettings(), triggerInput.GetCallback());
listener.StartListening();
}
//Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners
_store[triggerId] = true;
}
/// <summary>
/// The method that registers Event Hub listeners and assigns them to a Receive event. When I receive an event from the event hub listener, I trigger the callbackURL
/// </summary>
/// <param name="triggerId"></param>
/// <param name="triggerInput"></param>
public async Task RegisterTrigger(EventHubInput input)
{
var client = EventHubClient.CreateFromConnectionString(input.eventHubConnectionString, input.eventHubName);
EventHubConsumerGroup group = String.IsNullOrEmpty(input.consumerGroup) ?
client.GetDefaultConsumerGroup() : client.GetConsumerGroup(input.consumerGroup);
string[] partitions;
//If they specified partitions, iterate over their list to only listen to the partitions they specified
if (!String.IsNullOrEmpty(input.eventHubPartitionList))
{
partitions = input.eventHubPartitionList.Split(',');
}
//If they left it blank, create a list to listen to all partitions
else
{
partitions = new string[client.GetRuntimeInformation().PartitionCount];
for (int x = 0; x < partitions.Length; x++)
{
partitions[x] = x.ToString();
}
}
List<CancellationTokenSource> tokenSources = new List<CancellationTokenSource>();
//For ever partition I should listen to, create a thread with a listener on it
foreach (var p in partitions)
{
p.Trim();
var Receiver = group.CreateReceiver(client.GetRuntimeInformation().PartitionIds[int.Parse(p)], DateTime.UtcNow);
EventHubListener listener = new EventHubListener(Receiver);
//Register the event. When I Receive a message, call the method to trigger the logic app
listener.MessageReceived += (sender, e) => TriggerLogicApps(input.callbackUrl, e);
var ts = new CancellationTokenSource();
CancellationToken ct = ts.Token;
listener.StartListening(ct);
tokenSources.Add(ts);
}
//Register the triggerID in my store, so on subsequent checks from the logic app I don't spin up a new set of listeners
_store[input.callbackUrl] = tokenSources;
}
private ICommunicationListener CreateEventHubListener()
{
/*********************************************************************/
// 1: Create a state factory
/*********************************************************************/
/*
* as you read events from event hub you need to maintain a state (combination of partition + offset in the partition). this state is
* a marker to last event you have read (in order to avoid to duplicates).
*
* default state factory uses Service Fabric Reliable State to store state as in IReliableDictoinary<string, DefaultPartitionState>
*
* You can implement different PartitionStateFactory & Partition State (example: using the listener in a stateless service).
*/
var factory = new DefaultPartitionStateFactory(this.StateManager); // other overloads allows you to override default dictionary name
// and/or entries prefix (for example if you want to use one dictionary for multiple listeners).
/*********************************************************************/
// 2) listener options
/*********************************************************************/
/*
* because listener distribute loads among service fabric partitions of the current service
* you will need to identify the current partition id + Service name
*
* or if you intent to manually map a single service fabric service partitions to event hub partition
* then you can use this CTOR, the listener will not query other partitions of the service hence it won't need a fabriClient instance
* var options = new EventHubListenerOptions(currentSFPartition);
*/
var currentSFPartition = this.Context.PartitionId.ToString();
var currentServiceName = this.Context.ServiceName;
// if you have restricted access to cluster then you will need a to create a fabric client (with security) and pass it to the options
var options = new EventHubListenerOptions(currentSFPartition, currentServiceName);
// set the processor
options.Processor = new myEventProcessor(); // this is a class that implements IEventHubEventsProcessor
/*
* Supported Modes:
*
* 1- SafeDistribute: Maps 1..n Event Hub Partitions to 1 Service Fabric partition.
* will throw an exception if service fabric partitions are > Event hub partition
*
*
* 2- Distribute: maps 1..n event hub partitions to 1 service fabric partition
* if service fabric partitions are > event hub partitions, the remaining partitions will not
* get any distribution (i.e Event processor will not be created on them).
*
*
*
* 3- OneToOne: maps one to one event hub partition: Service Fabric Partition
* Service Fabric partition has to = Event Hub Partitions (or an exception will be thrown)
*
*
*
* 4- Single: maps a single event hub partition to a single service fabric partition.
* Event Hub communication listener will expect a supplied valid event hub partition id
*/
options.ListenerMode = EventHubListenerMode.SafeDistribute;
// Set the Partition State Factor
options.StateFactory = factory;
// Set Connection String
options.EventHubConnectionString = mEventHubConnectionString;
// Set Event Hub Name
options.EventHubName = mEventHubName;
// optionally set consumer group name (not setting it will default to "default consumer group")
//options.EventHubConsumerGroupName = "BE01";
/*************************************
* Addtional Options
*
* 1)
* Use event hub Epochs (details: http://blogs.msdn.com/b/gyan/archive/2014/09/02/event-hubs-receiver-epoch.aspx)
* options.UseEpoch(myEpochValue)
*
*
*
* (the below helps if you switching from receivers running elsewhere to Service Fabric)
* (details for the below: https://msdn.microsoft.com/en-us/library/azure/microsoft.servicebus.messaging.eventhubconsumergroup.createreceiver.aspx)
*
* 2)
* Events only newer than UTC Date
* options.StartWithEventsNewerThan(DateTime.UtcNow)
*
* 3)
* Specific Offset
* options.StartWithOffset("MyOffSet", bInclusive)
*
*
* you can also chain the call
* options.UseEpoch(longEpoch)
* .StartWithOffset(myoffset);
**************************************/
// override default batch size
//options.BatchSize = 100;
mEventHubListener = new EventHubListener(options);
return(mEventHubListener);
}
