private IConsumer <TKey, TValue> CreateConsumer()
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.logger);
var builder = this.CreateConsumerBuilder(GetConsumerConfiguration());
builder.SetErrorHandler((_, e) =>
{
logger.LogError(e.Reason);
})
.SetPartitionsAssignedHandler((_, e) =>
{
logger.LogInformation($"Assigned partitions: [{string.Join(", ", e)}]");
})
.SetPartitionsRevokedHandler((_, e) =>
{
logger.LogInformation($"Revoked partitions: [{string.Join(", ", e)}]");
});
if (ValueDeserializer != null)
{
builder.SetValueDeserializer(ValueDeserializer);
}
builder.SetLogHandler((_, m) =>
{
logger.Log((LogLevel)m.LevelAs(LogLevelType.MicrosoftExtensionsLogging), $"Libkafka: {m?.Message}");
});
return(builder.Build());
}
private IConsumer <TKey, TValue> CreateConsumer()
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.logger);
var builder = this.CreateConsumerBuilder(GetConsumerConfiguration());
builder.SetErrorHandler((_, e) =>
{
logger.LogError(e.Reason);
})
.SetPartitionsAssignedHandler((_, e) =>
{
logger.LogInformation($"Assigned partitions: [{string.Join(", ", e)}]");
})
.SetPartitionsRevokedHandler((_, e) =>
{
logger.LogInformation($"Revoked partitions: [{string.Join(", ", e)}]");
});
if (ValueDeserializer != null)
{
builder.SetValueDeserializer(ValueDeserializer);
}
return(builder.Build());
}
public IKafkaProducer Create(KafkaProducerEntity entity)
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka")));
// Goal is to create as less producers as possible
// We can group producers based on following criterias
// - Broker List
// - Configuration
var producerConfig = this.GetProducerConfig(entity);
var producerKey = CreateKeyForConfig(producerConfig);
var baseProducer = baseProducers.GetOrAdd(producerKey, (k) => CreateBaseProducer(producerConfig));
return(Create(baseProducer.Handle, entity));
}
public Task StartAsync(CancellationToken cancellationToken)
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.logger);
var builder = this.CreateConsumerBuilder(GetConsumerConfiguration());
builder.SetErrorHandler((_, e) =>
{
logger.LogError(e.Reason);
})
.SetPartitionsAssignedHandler((_, e) =>
{
logger.LogInformation($"Assigned partitions: [{string.Join(", ", e)}]");
})
.SetPartitionsRevokedHandler((_, e) =>
{
logger.LogInformation($"Revoked partitions: [{string.Join(", ", e)}]");
});
if (ValueDeserializer != null)
{
builder.SetValueDeserializer(ValueDeserializer);
}
this.consumer = builder.Build();
var commitStrategy = new AsyncCommitStrategy <TKey, TValue>(consumer, this.logger);
functionExecutor = singleDispatch ?
(FunctionExecutorBase <TKey, TValue>) new SingleItemFunctionExecutor <TKey, TValue>(executor, consumer, this.options.ExecutorChannelCapacity, this.options.ChannelFullRetryIntervalInMs, commitStrategy, logger) :
new MultipleItemFunctionExecutor <TKey, TValue>(executor, consumer, this.options.ExecutorChannelCapacity, this.options.ChannelFullRetryIntervalInMs, commitStrategy, logger);
consumer.Subscribe(this.listenerConfiguration.Topic);
// Using a thread as opposed to a task since this will be long running
var thread = new Thread(ProcessSubscription)
{
IsBackground = true,
};
thread.Start(cancellationTokenSource.Token);
return(Task.CompletedTask);
}