static void SetupAndStartEventHub()
{
const string eventHubConnectionString = "";
const string eventHubName = "";
const string storageAccountName = "";
const string storageAccountKey = "";
const string storageConnectionString = "";
var publisher = new Publisher();
// Init
var data = new List <string> {
"hello", "world"
};
publisher.Publish(data);
var eventProcessorName = Guid.NewGuid().ToString();
var eventProcessorHost = new EventProcessorHost(eventProcessorName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString);
Console.WriteLine("Registering EventProcessor ... ");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
eventProcessorHost.RegisterEventProcessorAsync <Consumer>(options).Wait();
Console.WriteLine("Receiving ");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public static async Task<ColdStorageCoordinator> CreateAsync(string hostName, Configuration configuration)
{
ColdStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName);
var storageAccount = CloudStorageAccount.Parse(configuration.BlobWriterStorageAccount);
Func<string, IBlobWriter> blobWriterFactory =
partitionId =>
new RollingBlobWriter.RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, configuration.BlobPrefix),
storageAccount,
configuration.ContainerName,
configuration.RollSizeForBlobWriterMb);
var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName);
}
catch (Exception e)
{
ColdStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName);
throw;
}
ColdStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName);
var factory = new ColdStorageEventProcessorFactory(
blobWriterFactory,
configuration.CircuitBreakerWarningLevel,
configuration.CircuitBreakerTripLevel,
configuration.CircuitBreakerStallInterval,
configuration.CircuitBreakerLogCooldownInterval,
eventHubId
);
var options = new EventProcessorOptions()
{
MaxBatchSize = configuration.MaxBatchSize,
PrefetchCount = configuration.PreFetchCount,
ReceiveTimeOut = configuration.ReceiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
};
options.ExceptionReceived +=
(s, e) => ColdStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: configuration.ConsumerGroupName,
eventHubPath: configuration.EventHubName,
eventHubConnectionString: configuration.EventHubConnectionString,
storageConnectionString: configuration.CheckpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new ColdStorageCoordinator(host);
}
public ProcessorConstructorMock(int eventBatchMaximumCount,
string consumerGroup,
string connectionString,
string eventHubName,
EventProcessorOptions options = default) : base(eventBatchMaximumCount, consumerGroup, connectionString, eventHubName, options)
{
}
async Task InitialOffsetProviderWithOffset()
{
// Send and receive single message so we can find out offset of the last message.
var lastOffsets = await DiscoverEndOfStream();
// Use a randomly generated container name so that initial offset provider will be respected.
var eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
this.EventHubConnectionString,
this.StorageConnectionString,
Guid.NewGuid().ToString());
var processorOptions = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
InitialOffsetProvider = partitionId => lastOffsets[partitionId].Item1,
MaxBatchSize = 100
};
var receivedEvents = await this.RunGenericScenario(eventProcessorHost, processorOptions);
// We should have received only 1 event from each partition.
Assert.False(receivedEvents.Any(kvp => kvp.Value.Count != 1), "One of the partitions didn't return exactly 1 event");
}
async Task InitialOffsetProviderWithDateTime()
{
// Send and receive single message so we can find out enqueue date-time of the last message.
var partitions = await DiscoverEndOfStream();
// We will use last enqueued message's enqueue date-time so EPH will pick messages only after that point.
var lastEnqueueDateTime = partitions.Max(le => le.Value.Item2);
TestUtility.Log($"Last message enqueued at {lastEnqueueDateTime}");
// Use a randomly generated container name so that initial offset provider will be respected.
var eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
TestUtility.EventHubsConnectionString,
TestUtility.StorageConnectionString,
Guid.NewGuid().ToString());
var processorOptions = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
InitialOffsetProvider = partitionId => lastEnqueueDateTime,
MaxBatchSize = 100
};
var runResult = await this.RunGenericScenario(eventProcessorHost, processorOptions);
// We should have received only 1 event from each partition.
Assert.False(runResult.ReceivedEvents.Any(kvp => kvp.Value.Count != 1), "One of the partitions didn't return exactly 1 event");
}
/// <summary>
/// Initializes a new instance of the <see cref="EventProcessorManager"/> class.
/// </summary>
///
/// <param name="consumerGroup">The name of the consumer group the event processors are associated with. Events are read in the context of this group.</param>
/// <param name="client">The client used to interact with the Azure Event Hubs service.</param>
/// <param name="partitionManager">Interacts with the storage system with responsibility for creation of checkpoints and for ownership claim.</param>
/// <param name="options">The set of options to use for the event processors.</param>
/// <param name="onInitialize">A callback action to be called on <see cref="PartitionProcessor.InitializeAsync" />.</param>
/// <param name="onClose">A callback action to be called on <see cref="PartitionProcessor.CloseAsync" />.</param>
/// <param name="onProcessEvents">A callback action to be called on <see cref="PartitionProcessor.ProcessEventsAsync" />.</param>
/// <param name="onProcessError">A callback action to be called on <see cref="PartitionProcessor.ProcessErrorAsync" />.</param>
///
public EventProcessorManager(string consumerGroup,
EventHubClient client,
PartitionManager partitionManager = null,
EventProcessorOptions options = null,
Action <PartitionContext> onInitialize = null,
Action <PartitionContext, PartitionProcessorCloseReason> onClose = null,
Action <PartitionContext, IEnumerable <EventData>, CancellationToken> onProcessEvents = null,
Action <PartitionContext, Exception, CancellationToken> onProcessError = null)
{
ConsumerGroup = consumerGroup;
InnerClient = client;
PartitionProcessorFactory = partitionContext =>
new PartitionProcessor
(
onInitialize,
onClose,
onProcessEvents,
onProcessError
);
InnerPartitionManager = partitionManager ?? new InMemoryPartitionManager();
// In case it has not been specified, set the maximum receive wait time to 2 seconds because the default
// value (1 minute) would take too much time.
Options = options?.Clone() ?? new EventProcessorOptions();
if (Options.MaximumReceiveWaitTime == null)
{
Options.MaximumReceiveWaitTime = TimeSpan.FromSeconds(2);
}
EventProcessors = new List <EventProcessor <PartitionProcessor> >();
}
/// <summary>
/// Creates an <see cref="EventProcessorClient" /> that uses mock storage and
/// a connection based on an identity credential.
/// </summary>
///
/// <param name="consumerGroup">The consumer group for the processor.</param>
/// <param name="eventHubName">The name of the Event Hub for the processor.</param>
/// <param name="options">The set of client options to pass.</param>
///
/// <returns>The processor instance.</returns>
///
private EventProcessorClient CreateProcessorWithSharedAccessSignature(string consumerGroup,
string eventHubName,
StorageManager storageManager = default,
EventProcessorOptions options = default)
{
var builder = new UriBuilder(EventHubsTestEnvironment.Instance.FullyQualifiedNamespace)
{
Scheme = "amqps://",
Path = eventHubName,
Port = -1,
Fragment = string.Empty,
Password = string.Empty,
UserName = string.Empty,
};
if (builder.Path.EndsWith("/", StringComparison.Ordinal))
{
builder.Path = builder.Path.TrimEnd('/');
}
var resource = builder.Uri.AbsoluteUri.ToLowerInvariant();
var signature = new SharedAccessSignature(resource, EventHubsTestEnvironment.Instance.SharedAccessKeyName, EventHubsTestEnvironment.Instance.SharedAccessKey);
var credential = new AzureSasCredential(signature.Value);
EventHubConnection createConnection() => new EventHubConnection(EventHubsTestEnvironment.Instance.FullyQualifiedNamespace, eventHubName, credential);
storageManager ??= new InMemoryStorageManager(_ => { });
return(new TestEventProcessorClient(storageManager, consumerGroup, EventHubsTestEnvironment.Instance.FullyQualifiedNamespace, eventHubName, credential, createConnection, options));
}
static void Main(string[] args)
{
string eventHubConnectionString = "Endpoint=sb://ehp...";
string eventHubName = "EHPipelineManager";
string storageAccountName = "storageaccountname";
string storageAccountKey = "storageacountkey";
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString);
Console.WriteLine("Registering EventProcessor...");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(options).Wait();
if (AnomallyDetected())
{
SendNotificationAsync();
}
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public Task StartReceiver <T>(string storageConnection) where T : IEventProcessor
{
if (string.IsNullOrWhiteSpace(Listen_EventHubConnectionstring))
{
throw new ArgumentNullException(nameof(Listen_EventHubConnectionstring));
}
var eventhubclient = EventHubClient.CreateFromConnectionString(Listen_EventHubConnectionstring, EventHubPath);
var defaultGroup = eventhubclient.GetDefaultConsumerGroup();
EventProc = new EventProcessorHost(AppDomain.CurrentDomain.FriendlyName, eventhubclient.Path, defaultGroup.GroupName, Listen_EventHubConnectionstring, storageConnection);
var opt = new EventProcessorOptions
{
InitialOffsetProvider = partitionId => DateTime.UtcNow.AddHours(-1),
PrefetchCount = 50,
MaxBatchSize = 50
};
opt.ExceptionReceived += (sender, e) => {
var errorLogger = new FileLogger(filePrefix: "EventProcessorUnhandledExceptions_", async: true);
errorLogger.AddRow(sender?.ToString() ?? "Sender was null" + Environment.NewLine + e?.ToString() ?? "e was null" + Environment.NewLine + e?.Exception?.ToString() ?? "e.exception was null");
};
return(EventProc.RegisterEventProcessorAsync <T>(opt));
}
//Event Hubsの使用
//https://azure.microsoft.com/ja-jp/documentation/articles/event-hubs-csharp-ephcs-getstarted/
static void Main(string[] args)
{
Properties.Settings settings = new Properties.Settings();
//string eventHubConnectionString = "{Event Hub connection string}";
//string eventHubName = "{Event Hub name}";
string eventHubName = settings.EventHubName;
string eventHubConnectionString = settings.EventHubConnectionString;
//string storageAccountName = "{storage account name}";
//string storageAccountKey = "{storage account key}";
string storageAccountName = settings.StorageAccountName;
string storageAccountKey = settings.StorageAccountKey;
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString);
Console.WriteLine("Registering EventProcessor...");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
eventProcessorHost.RegisterEventProcessorAsync<SimpleEventProcessor>(options).Wait();
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public MockCheckpointStoreProcessor(CheckpointStore checkpointStore,
int eventBatchMaximumCount,
string consumerGroup,
string connectionString,
EventProcessorOptions options = default) : base(checkpointStore, eventBatchMaximumCount, consumerGroup, connectionString, options)
{
}
protected void Application_Start()
{
AreaRegistration.RegisterAllAreas();
GlobalConfiguration.Configure(WebApiConfig.Register);
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
BundleConfig.RegisterBundles(BundleTable.Bundles);
string eventHubConnectionString = "connection_string";
string eventHubName = "outputhub";
string storageAccountName = "storage_account_name";
string storageAccountKey = "storage_account_key";
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString);
Debug.WriteLine("Registering EventProcessor...");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Debug.WriteLine(e.Exception); };
eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(options).Wait();
Debug.WriteLine("Receiving...");
}
private async Task RunAsync()
{
Console.WriteLine("Registering EventProcessor...");
var eventProcessorHost = new EventProcessorHost(
EhEntityPath,
PartitionReceiver.DefaultConsumerGroupName,
EhConnectionString,
StorageConnectionString,
StorageContainerName);
var eventProcessorOptions = new EventProcessorOptions();
eventProcessorOptions.SetExceptionHandler(e => {
Console.WriteLine("Error: " + e.Exception);
});
// Registers the Event Processor Host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(eventProcessorOptions);
Console.WriteLine("Receiving. Press ENTER to stop worker.");
Console.ReadLine();
// Disposes of the Event Processor Host
await eventProcessorHost.UnregisterEventProcessorAsync();
}
public EventHubListener(ITriggeredFunctionExecutor executor, EventProcessorHost eventListener, EventProcessorOptions options, bool single)
{
this._executor = executor;
this._eventListener = eventListener;
this._singleDispatch = single;
this._options = options;
}
public ReceiverService(
StatelessServiceContext serviceContext,
ILogger logger,
ReceiverSettings settings,
Action <string, object[]> serviceEventSource,
Func <CancellationToken, Task> @switch,
Func <string, Func <EventContext, Task> > f,
EventProcessorOptions options)
: base(serviceContext)
{
_logger = logger;
_settings = settings;
_serviceEventSource = serviceEventSource;
_f = f;
_switch = @switch;
_options = options;
_host = new EventProcessorHost(
$"{_settings.HostName ?? serviceContext.ServiceTypeName}-{serviceContext.NodeContext.NodeName}",
_settings.EventHubPath,
_settings.ConsumerGroup,
_settings.EventHubConnectionString,
_settings.StorageConnectionString,
_settings.LeaseContainerName);
}
public ReadableOptionsMock(string consumerGroup,
EventHubClient eventHubClient,
Func <PartitionContext, PartitionProcessor> partitionProcessorFactory,
PartitionManager partitionManager,
EventProcessorOptions options) : base(consumerGroup, eventHubClient, partitionProcessorFactory, partitionManager, options)
{
}
private static async Task StartAsync()
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json");
IConfiguration configuration = builder.Build();
var ehConnectionString = configuration["EhConnectionString"];
var eventHubName = configuration["EhEntityPath"];
var storageContainerName = configuration["StorageContainerName"];
var storageAccountName = configuration["StorageAccountName"];
var storageAccountKey = configuration["StorageAccountKey"];
var storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
var eventProcessorHost = new EventProcessorHost(
Guid.NewGuid().ToString(),
eventHubName,
PartitionReceiver.DefaultConsumerGroupName,
ehConnectionString,
storageConnectionString,
storageContainerName
);
var eventProcessorOptions = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow)
};
await eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(eventProcessorOptions);
Console.WriteLine("Receiving. Press Enter to stop worker.");
Console.ReadLine();
await eventProcessorHost.UnregisterEventProcessorAsync();
}
public static async Task Start()
{
var eventHubConnectionString = GetEventHubConnectionString();
var storageConnectionString = ConfigManager.Config.TableConnectionString;
var eventHubName = ConfigManager.Config.EventHubName;
// here it's using eventhub as lease name. but it can be specified as any you want.
// if the host is having same lease name, it will be shared between hosts.
// by default it is using eventhub name as lease name.
host = new EventProcessorHost(
hostName,
eventHubName,
consumerGroupName,
eventHubConnectionString,
storageConnectionString,
eventHubName.ToLowerInvariant());
factory = new EventProcessorFactory(hostName);
try
{
Trace.WriteLine(string.Format("{0} > Registering host: {1}", DateTime.Now.ToString(), hostName), "Information");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory);
}
catch (Exception exception)
{
Trace.WriteLine(string.Format("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message), "Error");
Console.ResetColor();
}
}
async Task InitialOffsetProviderOverrideBehavior()
{
// Generate a new lease container name that will be used through out the test.
string leaseContainerName = Guid.NewGuid().ToString();
// First host will send and receive as usual.
var eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
this.EventHubConnectionString,
this.StorageConnectionString,
leaseContainerName);
await this.RunGenericScenario(eventProcessorHost);
// Second host will use an initial offset provider.
// Since we are still on the same lease container, initial offset provider shouldn't rule.
// We should continue receiving where we left instead if start-of-stream where initial offset provider dictates.
eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
this.EventHubConnectionString,
this.StorageConnectionString,
leaseContainerName);
var processorOptions = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
InitialOffsetProvider = partitionId => PartitionReceiver.StartOfStream
};
var receivedEvents = await this.RunGenericScenario(eventProcessorHost, processorOptions);
// We should have received only 1 event from each partition.
Assert.False(receivedEvents.Any(kvp => kvp.Value.Count != 1), "One of the partitions didn't return exactly 1 event");
}
/// <summary>
/// Initializes a new instance of the <see cref="ShortWaitTimeMock"/> class.
/// </summary>
///
/// <param name="consumerGroup">The name of the consumer group this event processor is associated with. Events are read in the context of this group.</param>
/// <param name="eventHubClient">The client used to interact with the Azure Event Hubs service.</param>
/// <param name="partitionProcessorFactory">Creates an instance of a class implementing the <see cref="IPartitionProcessor" /> interface.</param>
/// <param name="partitionManager">Interacts with the storage system, dealing with ownership and checkpoints.</param>
/// <param name="options">The set of options to use for this event processor.</param>
///
public ShortWaitTimeMock(string consumerGroup,
EventHubClient eventHubClient,
Func <PartitionContext, CheckpointManager, IPartitionProcessor> partitionProcessorFactory,
PartitionManager partitionManager,
EventProcessorOptions options) : base(consumerGroup, eventHubClient, partitionProcessorFactory, partitionManager, options)
{
}
public void RespectsConnectionOptionsForProcessor(string expectedPathName, string connectionString)
{
var testEndpoint = new Uri("http://mycustomendpoint.com");
EventHubOptions options = new EventHubOptions
{
CustomEndpointAddress = testEndpoint,
TransportType = EventHubsTransportType.AmqpWebSockets,
WebProxy = new WebProxy("http://proxyserver/"),
ClientRetryOptions = new EventHubsRetryOptions
{
MaximumRetries = 10
}
};
var configuration = CreateConfiguration(new KeyValuePair <string, string>("connection", connectionString));
var factory = new EventHubClientFactory(configuration, Mock.Of <AzureComponentFactory>(), Options.Create(options), new DefaultNameResolver(configuration), new AzureEventSourceLogForwarder(new NullLoggerFactory()));
var processor = factory.GetEventProcessorHost(expectedPathName, "connection", "consumer");
EventProcessorOptions processorOptions = (EventProcessorOptions)typeof(EventProcessor <EventProcessorHostPartition>)
.GetProperty("Options", BindingFlags.NonPublic | BindingFlags.Instance)
.GetValue(processor);
Assert.AreEqual(testEndpoint, processorOptions.ConnectionOptions.CustomEndpointAddress);
Assert.AreEqual(EventHubsTransportType.AmqpWebSockets, processorOptions.ConnectionOptions.TransportType);
Assert.AreEqual("http://proxyserver/", ((WebProxy)processorOptions.ConnectionOptions.Proxy).Address.AbsoluteUri);
Assert.AreEqual(10, processorOptions.RetryOptions.MaximumRetries);
Assert.AreEqual(expectedPathName, processor.EventHubName);
}
private static void Main(string[] args)
{
const string eventHubConnectionString =
"Endpoint=<EH endpoint>;SharedAccessKeyName=<key name>;SharedAccessKey=<key>";
const string eventHubName = "<event hub name>";
const string storageAccountName = "<storage account name>";
const string storageAccountKey = "<valid storage key>";
var storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
Console.WriteLine("Connecting to storage account with ConnectionString: {0}", storageConnectionString);
var eventProcessorHostName = Guid.NewGuid().ToString();
var eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString);
var epo = new EventProcessorOptions
{
MaxBatchSize = 100,
PrefetchCount = 10,
ReceiveTimeOut = TimeSpan.FromSeconds(20),
InitialOffsetProvider = (name) => DateTime.Now.AddDays(-7)
};
epo.ExceptionReceived += OnExceptionReceived;
eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(epo).Wait();
Console.WriteLine("Receiving. Please enter to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
static void Main(string[] args)
{
string eventHubConnectionString = "[Event Hub Connection String]";
string eventHubName = "[Event Hub Name (Path)]";
string consumerGroupName = EventHubConsumerGroup.DefaultGroupName;
string storageAccountName = "[Storage Account Name]";
string storageAccountKey = "[Storage Account Key]";
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, consumerGroupName, eventHubConnectionString, storageConnectionString);
Console.WriteLine("Registering EventProcessor...");
EventProcessorOptions processorOptions = new EventProcessorOptions()
{
//Start reading messages from now, ingore any previous messages in the hub
InitialOffsetProvider = (name) => DateTime.UtcNow
};
eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(processorOptions).Wait();
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public async Task RegisterHubAsync()
{
if (_isRegistered)
{
return;
}
if (_processorHost == null)
{
_logger.LogInformation("Create event processor host");
_processorHost = _dataEventProcessorFactory.CreateEventProcessorHost();
}
_dataEventProcessorFactory.SetCallback(HandleProcessEventsAsync);
var options = new EventProcessorOptions();
options.SetExceptionHandler(e =>
{
_logger.LogWarning(e.Exception, "Error received from event processor. Action: {0} Hostname: {1} PartitionId: {2}", e.Action, e.Hostname, e.PartitionId);
});
_logger.LogInformation("Connect to azure event hub");
await _processorHost.RegisterEventProcessorFactoryAsync(_dataEventProcessorFactory, options).ConfigureAwait(false);
_isRegistered = true;
}
private static async Task StartHost()
{
//Event Hub
var eventHubConnectionString = GetEventHubConnectionString();
var storageConnectionString = GetStorageConnectionString();
string hostname = Guid.NewGuid().ToString();
host = new EventProcessorHost(
hostname,
Properties.Settings.Default.EventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString, Properties.Settings.Default.EventHubName.ToLowerInvariant());
factory = new HeartbeatEventProcessorFactory(hostname);
try
{
Console.WriteLine("{0} > Registering host: {1}", DateTime.Now.ToString(), hostname);
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory);
}
catch (Exception exception)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message);
Console.ResetColor();
}
}
public async Task Start()
{
_isRunning = true;
_eventProcessorHost = new EventProcessorHost(Name,
Settings.EventHubName,
Settings.EventHubConsumerGroup,
Settings.EventHubConnectionString,
Settings.LeaseConnectionString)
{
PartitionManagerOptions = new PartitionManagerOptions
{
AcquireInterval = Settings.AcquireLeaseEvery,
RenewInterval = Settings.RenewLeaseEvery,
LeaseInterval = Settings.ExpireLeaseEvery,
},
};
EventProcessorOptions options = new EventProcessorOptions
{
MaxBatchSize = Settings.ReadBatchSize,
PrefetchCount = Settings.ReadPrefetchCount,
ReceiveTimeOut = Settings.ReadReceiveTimeout,
EnableReceiverRuntimeMetric = Settings.ReadEnableReceiverRuntimeMetric,
InvokeProcessorAfterReceiveTimeout = false,
};
options.ExceptionReceived += EventProcessorOptions_ExceptionReceivedHandler;
await _eventProcessorHost.RegisterEventProcessorAsync <ServiceBusReader>(options).ConfigureAwait(false);
}
public void RespectsConnectionOptionsForProcessor(string expectedPathName, string connectionString)
{
var testEndpoint = new Uri("http://mycustomendpoint.com");
EventHubOptions options = new EventHubOptions
{
ConnectionOptions = new EventHubConnectionOptions
{
CustomEndpointAddress = testEndpoint
},
RetryOptions = new EventHubsRetryOptions
{
MaximumRetries = 10
}
};
var configuration = CreateConfiguration(new KeyValuePair <string, string>("connection", connectionString));
var factory = new EventHubClientFactory(configuration, Mock.Of <AzureComponentFactory>(), Options.Create(options), new DefaultNameResolver(configuration));
var processor = factory.GetEventProcessorHost(expectedPathName, "connection", "consumer");
EventProcessorOptions processorOptions = (EventProcessorOptions)typeof(EventProcessor <EventProcessorHostPartition>)
.GetProperty("Options", BindingFlags.NonPublic | BindingFlags.Instance)
.GetValue(processor);
Assert.AreEqual(testEndpoint, processorOptions.ConnectionOptions.CustomEndpointAddress);
Assert.AreEqual(10, processorOptions.RetryOptions.MaximumRetries);
Assert.AreEqual(expectedPathName, processor.EventHubName);
}
/// <summary>
/// Initializes the subscriber asynchronously.
/// </summary>
/// <param name="description">
/// The <see cref="SubscriberDescription">description</see>.
/// </param>
/// <param name="handler">
/// The handler callback method.
/// </param>
/// <returns>
/// A <see cref="Task"/> representing the initialization operation.
/// </returns>
public async Task InitializeAsync(SubscriberDescription description, Func <IMessage, Task> handler)
{
using (ActivityMonitor.Instance.SubscriberInitialize(this, description))
{
try
{
await this.EnsureEventHubExistsAsync(description);
await this.EnsureConsumerGroupExistsAsync(description);
this.Callback = handler;
var name = Guid.NewGuid().ToString("N");
var path = description.Entity;
var hcs = description.ConnectionString;
var scs = description.StorageConnectionString;
var group = description.Name;
this.Host = new EventProcessorHost(name, path, group, hcs, scs);
var options = new EventProcessorOptions();
options.ExceptionReceived += this.OnProcessorExceptionReceived;
await this.Host.RegisterEventProcessorFactoryAsync(this, options);
}
catch (Exception e)
{
ActivityMonitor.Instance.ReportSubscriberException(this, e, false);
throw new AzureEventHubException(e.Message, e);
}
}
}
public void RegisterEventProcessor <T>() where T : IEventProcessor
{
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
_eventProcessorHost.RegisterEventProcessorAsync <T>(options).Wait();
}
static void Main(string[] args)
{
string eventProcessorHostName = Guid.NewGuid().ToString();
//messages/events - location of events in EPH
EventProcessorHost eventProcessorHost = new EventProcessorHost
(eventProcessorHostName,
"messages/events",
"code" /*EventHubConsumerGroup.DefaultGroupName*/,
iotHubConnectionString,
storageConnectionString,
"messages-events");
Console.WriteLine("Registering EventProcessor...");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
options.InitialOffsetProvider = (p) =>
{
return(DateTime.Now.AddMinutes(-90));
};
eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(options).Wait();
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
public static void Main()
{
string iotHubConnectionString = ConfigurationManager.ConnectionStrings["Microsoft.Azure.IoT.ConnectionString"].ToString();
string iotHubD2cEndpoint = "messages/events";
string strGroupName = ConfigurationManager.AppSettings["GroupName"].ToString();
IoTProcessor.StorageConnectionString = ConfigurationManager.ConnectionStrings["Microsoft.Azure.IoT.Storage.ConnectionString"].ToString();
string strContainerName = ConfigurationManager.AppSettings["ContainerName"].ToString();
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost =
new EventProcessorHost(eventProcessorHostName, iotHubD2cEndpoint, strGroupName,
iotHubConnectionString, IoTProcessor.StorageConnectionString, strContainerName);
var options = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => DateTime.UtcNow,
};
eventProcessorHost.RegisterEventProcessorAsync <IoTProcessor>(options).Wait();
//eventProcessorHost.RegisterEventProcessorAsync<IoTProcessor>().Wait();
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
private async static void Run(TypeContainer tc)
{
string eventHubConfigFile = _settings.EventHubConfigFile;
var eventHubConfig = ConfigUtility.ReadConfig<Dictionary<string, string>>(eventHubConfigFile);
string eventHubName = eventHubConfig["EventHubName"];
string eventHubConnectionString = eventHubConfig["EventHubConnectionString"];
string azureStorageConnectionString = eventHubConfig["AzureStorageConnectionString"];
string eventProcessorHostName = "IoTEventHubTSDBWriter";
try
{
EventProcessorHost host = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
azureStorageConnectionString,
eventHubName.ToLowerInvariant());
IEventProcessorFactory factory = new TsdbEventProcessorFactory();
IInitializer factoryInitializer = factory as IInitializer;
factoryInitializer.Initialize(_settings);
var options = new EventProcessorOptions();
options.ExceptionReceived += Options_ExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory, options);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}
private static async Task MainAsync(string[] args)
{
AppDomain.CurrentDomain.ProcessExit += OnExit;
AssemblyLoadContext.Default.Unloading += OnExit;
// Setup the Options
var epo = new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow)
};
// Setup the configuration File
var config = new Configuration();
var consumerGroupName = PartitionReceiver.DefaultConsumerGroupName;
if (!string.IsNullOrEmpty(config.ConsumerGroupName))
{
consumerGroupName = config.ConsumerGroupName;
}
else
{
consumerGroupName = PartitionReceiver.DefaultConsumerGroupName;
}
eventProcessorHost = new EventProcessorHost(
config.Hub,
consumerGroupName,
config.EventHubEndpoint,
config.StorageConnectionString,
config.StorageContainer);
await eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(epo);
}
public async Task RegisterProcessorAsync()
{
var processorOps = new EventProcessorOptions()
{
MaxBatchSize=5000,
PrefetchCount=1000
};
await _host.RegisterEventProcessorFactoryAsync(_processorFactory, processorOps);
}
private EventProcessorOptions CreateEventProcessorOptions()
{
var eventProcessorOptions = new EventProcessorOptions
{
MaxBatchSize = MaxBatchSize,
ReceiveTimeOut = ReceiveTimeout
};
eventProcessorOptions.ExceptionReceived += EventProcessorOptionsOnExceptionReceived;
return eventProcessorOptions;
}
public static async Task<WarmStorageCoordinator> CreateAsync(string hostName, Configuration configuration)
{
WarmStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName);
Func<string, IElasticSearchWriter> elasticSearchWriterFactory = partitionId =>
new ElasticSearchWriter.ElasticSearchWriter(
configuration.ElasticSearchUrl,
configuration.ElasticSearchIndexPrefix,
configuration.ElasticSearchIndexType,
configuration.RetryCount
);
var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName);
}
catch (Exception e)
{
WarmStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName);
throw;
}
WarmStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName);
var buildingLookupService = new BuildingLookupService();
await buildingLookupService.InitializeAsync();
var factory = new WarmStorageEventProcessorFactory(elasticSearchWriterFactory, eventHubId, buildingLookupService);
var options = new EventProcessorOptions()
{
MaxBatchSize = configuration.MaxBatchSize,
PrefetchCount = configuration.PreFetchCount,
ReceiveTimeOut = configuration.ReceiveTimeout
};
options.ExceptionReceived +=
(s, e) => WarmStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: configuration.ConsumerGroupName,
eventHubPath: configuration.EventHubName,
eventHubConnectionString: configuration.EventHubConnectionString,
storageConnectionString: configuration.CheckpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new WarmStorageCoordinator(host);
}
public void ProcessMessages(ref EventProcessorHost host, ref object thelock, ref bool registered, bool active)
{
ServicePointManager.DefaultConnectionLimit = 12;
lock (thelock)
{
if (active)
{
if (!registered)
{
try
{
EventProcessorOptions options = new EventProcessorOptions();
options.InitialOffsetProvider = (partitionId) => DateTime.UtcNow;
host.RegisterEventProcessorAsync<MessageProcessor>(options).Wait();
registered = true;
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
registered = false;
}
}
}
else
{
if (registered)
{
try
{
host.UnregisterEventProcessorAsync().Wait();
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
finally
{
registered = false;
}
}
}
}
}
private async Task StartProcessor(CancellationToken token)
{
try
{
string hostName = Environment.MachineName;
string eventHubPath = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.Name");
string consumerGroup = EventHubConsumerGroup.DefaultGroupName;
string eventHubConnectionString = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.ConnectionString");
string storageConnectionString = _configurationProvider.GetConfigurationSettingValue("device.StorageConnectionString");
_eventProcessorHost = new EventProcessorHost(
hostName,
eventHubPath,
consumerGroup,
eventHubConnectionString,
storageConnectionString);
_factory = new ActionProcessorFactory(
_actionLogic,
_actionMappingLogic,
_configurationProvider);
Trace.TraceInformation("ActionEventProcessor: Registering host...");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory);
// processing loop
while (!token.IsCancellationRequested)
{
Trace.TraceInformation("ActionEventProcessor: Processing...");
await Task.Delay(TimeSpan.FromMinutes(5), token);
}
// cleanup
await _eventProcessorHost.UnregisterEventProcessorAsync();
}
catch (Exception e)
{
Trace.TraceError("Error in ActionProcessor.StartProcessor, Exception: {0}", e.ToString());
}
_isRunning = false;
}
static void Main(string[] args)
{
string eventHubConnectionString = ConfigurationManager.AppSettings["eventHubReceiveRuleConnectionString"];
string eventHubName = ConfigurationManager.AppSettings["eventHubName"];
string storageAccountName = ConfigurationManager.AppSettings["storageAccountName"];
string storageAccountKey = ConfigurationManager.AppSettings["storageAccountKey"];
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString);
Console.WriteLine("Registering EventProcessor...");
var options = new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => { Console.WriteLine(e.Exception); };
eventProcessorHost.RegisterEventProcessorAsync<SimpleEventProcessor>(options).Wait();
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
/// <summary>
/// Starts the specified host control.
/// </summary>
/// <param name="hostControl">The host control.</param>
/// <returns></returns>
public override bool Start(HostControl hostControl)
{
var processorOptions = new EventProcessorOptions
{
MaxBatchSize = 5000, // default is 300
PrefetchCount = 1000
};
_processorHost = new EventProcessorHost(
Environment.MachineName,
eventHubPath: ConfigurationManager.AppSettings["eventHubPath"],
consumerGroupName: ConfigurationManager.AppSettings["consumerGroupName"],
storageConnectionString: ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString,
eventHubConnectionString: ConfigurationManager.ConnectionStrings["EventHubConnectionString"].ConnectionString);
using (Logger.BeginTimedOperation("RegisterEventProcessorAsync"))
_processorHost.RegisterEventProcessorFactoryAsync(this, processorOptions).Wait();
return base.Start(hostControl);
}
async Task StartProcessor(CancellationToken token)
{
try
{
// Initialize
_eventProcessorHost = new EventProcessorHost(
Environment.MachineName,
_configurationProvider.GetConfigurationSettingValue("eventHub.HubName").ToLowerInvariant(),
EventHubConsumerGroup.DefaultGroupName,
_configurationProvider.GetConfigurationSettingValue("eventHub.ConnectionString"),
_configurationProvider.GetConfigurationSettingValue("eventHub.StorageConnectionString"));
_factory = new DeviceAdministrationProcessorFactory(_deviceLogic, _configurationProvider);
Trace.TraceInformation("DeviceEventProcessor: Registering host...");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory);
// processing loop
while (!token.IsCancellationRequested)
{
Trace.TraceInformation("DeviceEventProcessor: Processing...");
await Task.Delay(TimeSpan.FromMinutes(5), token);
// Any additional incremental processing can be done here (like checking states, etc).
}
// cleanup
await _eventProcessorHost.UnregisterEventProcessorAsync();
}
catch (Exception e)
{
Trace.TraceInformation("Error in DeviceEventProcessor.StartProcessor, Exception: {0}", e.Message);
}
_running = false;
}
// ReSharper disable once FunctionComplexityOverflow
private async void btnStart_Click(object sender, EventArgs e)
{
try
{
Cursor.Current = Cursors.WaitCursor;
if (string.Compare(btnStart.Text, Start, StringComparison.OrdinalIgnoreCase) == 0)
{
// Validate parameters
if (!ValidateParameters())
{
return;
}
btnStart.Enabled = false;
var eventHubClient = EventHubClient.CreateFromConnectionString(txtServiceBusConnectionString.Text,
cboEventHub.Text);
// Get the default Consumer Group
eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(),
eventHubClient.Path.ToLower(),
txtConsumerGroup.Text.ToLower(),
txtServiceBusConnectionString.Text,
txtStorageAccountConnectionString.Text)
{
PartitionManagerOptions = new PartitionManagerOptions
{
AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds
}
};
WriteToLog(RegisteringEventProcessor);
var eventProcessorOptions = new EventProcessorOptions
{
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = 100,
PrefetchCount = 100,
ReceiveTimeOut = TimeSpan.FromSeconds(30),
};
eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived;
var eventProcessorFactoryConfiguration = new EventProcessorFactoryConfiguration
{
TrackEvent = a => Invoke(new Action<Alert>(i => alertBindingList.Add(i)), a),
WriteToLog = m => Invoke(new Action<string>(WriteToLog), m)
};
await
eventProcessorHost.RegisterEventProcessorFactoryAsync(
new EventProcessorFactory<EventProcessor>(eventProcessorFactoryConfiguration),
eventProcessorOptions);
WriteToLog(EventProcessorRegistered);
// Change button text
btnStart.Text = Stop;
}
else
{
// Stop Event Processor
if (eventProcessorHost != null)
{
await eventProcessorHost.UnregisterEventProcessorAsync();
}
// Change button text
btnStart.Text = Start;
}
}
catch (Exception ex)
{
HandleException(ex);
}
finally
{
Cursor.Current = Cursors.Default;
btnStart.Enabled = true;
}
}
private async Task StartEventProcessorAsync()
{
try
{
var eventHubClient = EventHubClient.CreateFromConnectionString(serviceBusConnectionString, eventHubName);
// Get the default Consumer Group
eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(),
eventHubClient.Path.ToLower(),
consumerGroupName.ToLower(),
serviceBusConnectionString,
storageAccountConnectionString)
{
PartitionManagerOptions = new PartitionManagerOptions
{
AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds
}
};
ServiceEventSource.Current.Message(RegisteringEventProcessor);
var eventProcessorOptions = new EventProcessorOptions
{
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = 100,
PrefetchCount = 100,
ReceiveTimeOut = TimeSpan.FromSeconds(30),
};
eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived;
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory<EventProcessor>(deviceActorServiceUri),
eventProcessorOptions);
ServiceEventSource.Current.Message(EventProcessorRegistered);
}
catch (Exception ex)
{
// Trace Error
ServiceEventSource.Current.Message(ex.Message);
throw;
}
}
EventProcessorOptions IEventHubProvider.GetOptions()
{
EventProcessorOptions options = new EventProcessorOptions
{
MaxBatchSize = 1000
};
return options;
}
/// <summary>
/// Initializes the subscriber asynchronously.
/// </summary>
/// <param name="description">
/// The <see cref="SubscriberDescription">description</see>.
/// </param>
/// <param name="handler">
/// The handler callback method.
/// </param>
/// <returns>
/// A <see cref="Task"/> representing the initialization operation.
/// </returns>
public async Task InitializeAsync(SubscriberDescription description, Func<IMessage, Task> handler)
{
using (ActivityMonitor.Instance.SubscriberInitialize(this, description))
{
try
{
await this.EnsureEventHubExistsAsync(description);
await this.EnsureConsumerGroupExistsAsync(description);
this.Callback = handler;
var name = Guid.NewGuid().ToString("N");
var path = description.Entity;
var hcs = description.ConnectionString;
var scs = description.StorageConnectionString;
var group = description.Name;
this.Host = new EventProcessorHost(name, path, group, hcs, scs);
var options = new EventProcessorOptions();
options.ExceptionReceived += this.OnProcessorExceptionReceived;
await this.Host.RegisterEventProcessorFactoryAsync(this, options);
}
catch (Exception e)
{
ActivityMonitor.Instance.ReportSubscriberException(this, e, false);
throw new AzureEventHubException(e.Message, e);
}
}
}
// ReSharper disable once FunctionComplexityOverflow
private async void btnStart_Click(object sender, EventArgs e)
{
if (string.Compare(btnStart.Text, Start, StringComparison.OrdinalIgnoreCase) == 0)
{
if (startLog != null && checkBoxLogging.Checked)
{
startLog();
}
receiverEventDataInspector = cboReceiverInspector.SelectedIndex > 0
? Activator.CreateInstance(serviceBusHelper.EventDataInspectors[cboReceiverInspector.Text]) as IEventDataInspector
: null;
cancellationTokenSource = new CancellationTokenSource();
btnStart.Text = Stop;
blockingCollection = new BlockingCollection<Tuple<long, long, long>>();
timer = new System.Timers.Timer
{
AutoReset = true,
Enabled = true,
Interval = 1000 * txtRefreshInformation.IntegerValue
};
timer.Elapsed += timer_Elapsed;
var ns = serviceBusHelper != null && !string.IsNullOrWhiteSpace(serviceBusHelper.Namespace) ? serviceBusHelper.Namespace : iotHubConnectionString;
var eventHub = eventHubClient.Path;
var maxBatchSize = txtMaxBatchSize.IntegerValue > 0 ? txtMaxBatchSize.IntegerValue : 1;
var receiveTimeout = TimeSpan.FromSeconds(txtReceiveTimeout.IntegerValue);
try
{
registeredDictionary = new Dictionary<string, bool>(partitionRuntumeInformationList.Count);
var startDateTimeEnabled = pickerStartingDateTimeUtc.Checked;
var startDateTimeValue = DateTime.SpecifyKind(pickerStartingDateTimeUtc.Value, DateTimeKind.Utc);
var eventProcessorOptions = new EventProcessorOptions
{
InitialOffsetProvider = partitionId =>
{
if (startDateTimeEnabled)
{
return startDateTimeValue;
}
var lease = EventProcessorCheckpointHelper.GetLease(ns,
eventHub,
consumerGroup.GroupName,
partitionId);
return lease != null ? lease.Offset : "-1";
},
MaxBatchSize = maxBatchSize,
ReceiveTimeOut = receiveTimeout
};
eventProcessorOptions.ExceptionReceived += (s, ex) => HandleException(ex.Exception);
var checkpointManager = new EventProcessorCheckpointManager
{
Namespace = ns,
EventHub = eventHub,
ConsumerGroup = consumerGroup.GroupName
};
var eventProcessorFactoryConfiguration = new EventProcessorFactoryConfiguration(checkBoxLogging,
checkBoxTrackMessages,
checkBoxVerbose,
checkBoxOffsetInclusive,
checkBoxCheckpoint,
cancellationTokenSource.Token)
{
TrackEvent = ev => Invoke(new Action<EventData>(m => eventDataCollection.Add(m)), ev),
GetElapsedTime = GetElapsedTime,
UpdateStatistics = UpdateStatistics,
WriteToLog = writeToLog,
MessageInspector = receiverEventDataInspector,
ServiceBusHelper = serviceBusHelper
};
foreach (var partitionRuntimeInformation in partitionRuntumeInformationList)
{
#pragma warning disable 4014
consumerGroup.RegisterProcessorFactoryAsync(
#pragma warning restore 4014
EventProcessorCheckpointHelper.GetLease(ns, eventHub, consumerGroup.GroupName, partitionRuntimeInformation.PartitionId),
checkpointManager,
new EventProcessorFactory<EventProcessor>(eventProcessorFactoryConfiguration),
eventProcessorOptions);
registeredDictionary.Add(partitionRuntimeInformation.PartitionId, true);
}
#pragma warning disable 4014
Task.Run(new Action(RefreshGraph));
#pragma warning restore 4014
}
catch (Exception ex)
{
HandleException(ex);
StopListenerAsync().Wait();
btnStart.Text = Start;
}
}
else
{
try
{
await StopListenerAsync();
btnStart.Text = Start;
}
catch (Exception ex)
{
HandleException(ex);
}
}
}
protected async Task InitializeAsync(
string hostName,
string eventHubName,
string consumerGroupName,
string eventHubConnectionString,
string checkpointStorageAccount,
int maxBatchSize,
int prefetchCount,
TimeSpan receiveTimeout,
int maxConcurrencyPerProcessor,
IEnumerable<Type> typesToSearch,
Func<string, string, ICircuitBreaker> circuitBreakerFactory,
IDispatcherInstrumentationPublisher instrumentationPublisher)
{
Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName);
// Get the consumer group via the Service Bus namespace (identifies the
// consumer)
var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName);
Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName);
}
catch (Exception e)
{
Logger.Error(e, "Could not establish connection to {0} in event hub {1}", consumerGroupName, eventHubName);
throw;
}
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName);
// Use a custom event processor factory to pass parameters to the
// event host processor
var factory = new EventProcessorFactory(
handlerResolver: new MessageHandlerResolver(typesToSearch, DependencyResolverFactory.GetResolver()),
maxConcurrency: maxConcurrencyPerProcessor,
circuitBreakerFactory: circuitBreakerFactory,
eventHubName: eventHubId,
instrumentationPublisher: instrumentationPublisher);
var options = new EventProcessorOptions
{
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeOut = receiveTimeout
};
options.ExceptionReceived += options_ExceptionReceived;
// Create the event processor host and register via the factory
_host = new EventProcessorHost(
hostName,
consumerGroupName: consumerGroupName,
eventHubPath: eventHubName,
eventHubConnectionString: eventHubConnectionString,
storageConnectionString: checkpointStorageAccount
);
await _host.RegisterEventProcessorFactoryAsync(factory, options);
Logger.Info("Event processor registered for {0} ({1})", eventHubName, consumerGroupName);
}
public static async Task<ColdStorageCoordinator> CreateAsync(
string hostName,
string eventHubName,
string consumerGroupName,
string eventHubConnectionString,
string checkpointStorageAccount,
int maxBatchSize,
int prefetchCount,
TimeSpan receiveTimeout,
IReadOnlyList<string> blobWriterStorageAccounts,
string containerName,
int rollSizeMb,
string blobPrefix,
int warningLevel,
int tripLevel,
TimeSpan stallInterval,
TimeSpan logCooldownInterval,
IColdStorageInstrumentationPublisher instrumentationPublisher)
{
Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName);
var storageAccounts = blobWriterStorageAccounts
.Select(CloudStorageAccount.Parse)
.ToList();
Func<string, IBlobWriter> blobWriterFactory =
partitionId =>
new RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, blobPrefix),
instrumentationPublisher,
storageAccounts[Int32.Parse(partitionId) % storageAccounts.Count],
containerName,
rollSizeMb);
var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName);
}
catch (Exception e)
{
Logger.Error(e, "Invalid consumer group name {0} in event hub {1}", consumerGroupName, eventHubName);
throw;
}
Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName);
var factory = new ColdStorageEventProcessorFactory(
blobWriterFactory,
instrumentationPublisher,
CancellationToken.None,
warningLevel,
tripLevel,
stallInterval,
logCooldownInterval,
eventHubId
);
var options = new EventProcessorOptions()
{
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeOut = receiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
};
options.ExceptionReceived +=
(s, e) => Logger.Error(
e.Exception,
"Error on message processing, action {0}",
e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: consumerGroupName,
eventHubPath: eventHubName,
eventHubConnectionString: eventHubConnectionString,
storageConnectionString: checkpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new ColdStorageCoordinator(host);
}