public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_name = name;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();
if (options.Topics != null && options.Topics.Count == 0)
{
throw new ArgumentNullException(nameof(options.Topics));
}
_adapterCache = new SimpleQueueAdapterCache(
cacheOptions,
name,
loggerFactory
);
_queueProperties = GetQueuesProperties();
_streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
}
/// <summary>
/// Add Kafka Topic Stream from configuration.
/// </summary>
/// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
/// <param name="domain">The domain of the stream.</param>
/// <param name="streamType">The stream type.</param>
/// <param name="version">The version of the stream.</param>
/// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Topic configuration options.</param>
/// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
[NotNull] string domain, [NotNull] string streamType, uint version, [NotNull] IConfiguration configuration)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (string.IsNullOrWhiteSpace(domain))
{
throw new ArgumentException($"'{nameof(domain)}' must not be null, empty or whitespace.",
nameof(domain));
}
if (string.IsNullOrWhiteSpace(streamType))
{
throw new ArgumentException($"'{nameof(streamType)}' must not be null, empty or whitespace.",
nameof(streamType));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
return(AddTopicStream(options, StreamHelper.GetNamespace(domain, streamType, version),
configuration));
}
/// <summary>
/// Add Kafka Topic Stream from configuration section.
/// </summary>
/// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
/// <param name="name">The name of the topic.</param>
/// <param name="configurationSection">The <see cref="IConfigurationSection" /> which contains Kafka Topic configuration options.</param>
/// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
[NotNull] string name, [NotNull] IConfigurationSection configurationSection)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (string.IsNullOrWhiteSpace(name))
{
throw new ArgumentException($"'{nameof(name)}' must not be null, empty or whitespace.", nameof(name));
}
var kafkaTopicOptions = configurationSection?.Get <KafkaTopicOptions>() ?? new KafkaTopicOptions();
if (kafkaTopicOptions.Partitions < 1)
{
throw new ConfigurationErrorsException("Number of topic partitions cannot be less than 1.");
}
if (kafkaTopicOptions.ReplicationFactor < 1)
{
throw new ConfigurationErrorsException("Number of topic replicas cannot be less than 1.");
}
options.AddTopic(name, new TopicCreationConfig
{
Partitions = kafkaTopicOptions.Partitions,
AutoCreate = kafkaTopicOptions.IsTopicCreationEnabled,
ReplicationFactor = kafkaTopicOptions.ReplicationFactor,
RetentionPeriodInMs = kafkaTopicOptions.RetentionPeriodInMs
});
return(options);
}
public void FromConfiguration_CorrectSetup_ReturnsKafkaStreamOptions()
{
var config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var kafkaStreamOptions = new KafkaStreamOptions().FromConfiguration(config);
Assert.NotNull(kafkaStreamOptions);
}
public KafkaAdapterReceiver(
QueueProperties queueProperties,
KafkaStreamOptions options,
SerializationManager serializationManager,
ILoggerFactory loggerFactory
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_queueProperties = queueProperties;
_serializationManager = serializationManager;
_logger = loggerFactory.CreateLogger <KafkaAdapterReceiver>();
}
/// <summary>
/// Use KafkaStreamOptions from configuration.
/// </summary>
/// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
/// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Streams provider's configuration options.</param>
/// <returns>The <see cref="KafkaStreamOptions"/> </returns>
public static KafkaStreamOptions FromConfiguration([NotNull] this KafkaStreamOptions options,
[NotNull] IConfiguration configuration)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
return(options.FromConfiguration(configuration.GetSection(KafkaOptions.DefaultSectionName)));
}
public void AddTopicStream_KafkaStreamsConfigurationSectionNotNull_ShouldSetValuesOnKafkaStreamOptions()
{
var config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var expectedConfig = config.GetSection(KafkaTopicOptions.DefaultSectionName).Get <KafkaTopicOptions>();
var kafkaStreamOptions = new KafkaStreamOptions().AddTopicStream("Test", config.GetSection(KafkaTopicOptions.DefaultSectionName));
Assert.NotNull(kafkaStreamOptions);
Assert.Equal("Test", kafkaStreamOptions.Topics[0].Name);
Assert.Equal(expectedConfig.Partitions, kafkaStreamOptions.Topics[0].Partitions);
Assert.Equal(expectedConfig.ReplicationFactor, kafkaStreamOptions.Topics[0].ReplicationFactor);
Assert.Equal(expectedConfig.RetentionPeriodInMs, kafkaStreamOptions.Topics[0].RetentionPeriodInMs);
Assert.Equal(expectedConfig.IsTopicCreationEnabled, kafkaStreamOptions.Topics[0].AutoCreate);
}
public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory
) : this(name, options, cacheOptions, serializationManager, loggerFactory, grainFactory, null)
{
if (options.Topics.Any(topic => topic.IsExternal))
{
throw new InvalidOperationException(
"Cannot have external topic with no 'IExternalDeserializer' defined. Use 'AddJson' or 'AddAvro'"
);
}
}
/// <summary>
/// Use KafkaStreamOptions from configuration section.
/// </summary>
/// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
/// <param name="configurationSection">The <see cref="IConfigurationSection" /> which contains Kafka Stream options.</param>
/// <returns>The <see cref="KafkaStreamOptions"/> </returns>
public static KafkaStreamOptions FromConfiguration([NotNull] this KafkaStreamOptions options,
[NotNull] IConfigurationSection configurationSection)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (configurationSection?.Exists() != true)
{
throw new ConfigurationErrorsException(
$"Configuration section '{configurationSection?.Path}' is incorrect.");
}
var kafkaOptions = configurationSection.Get <KafkaOptions>();
if (kafkaOptions == null)
{
throw new ConfigurationErrorsException(
$"Configuration section '{configurationSection.Path}' is incorrect.");
}
if (kafkaOptions.Brokers != null)
{
options.BrokerList = kafkaOptions.Brokers.Split(';');
}
options.ConsumerGroupId = kafkaOptions.ConsumerGroupId;
if (kafkaOptions.Security.IsEnabled)
{
if (string.IsNullOrWhiteSpace(kafkaOptions.Security.SaslUsername))
{
throw new ConfigurationErrorsException($"Security is enabled but username is empty");
}
if (string.IsNullOrEmpty(kafkaOptions.Security.SaslPassword))
{
throw new ConfigurationErrorsException($"Security is enabled but password is empty");
}
options.SecurityProtocol = kafkaOptions.Security.SecurityProtocol;
options.SaslUserName = kafkaOptions.Security.SaslUsername;
options.SaslPassword = kafkaOptions.Security.SaslPassword;
options.SaslMechanism = kafkaOptions.Security.SaslMechanism;
}
return(options);
}
/// <summary>
/// Add Kafka Topic Stream from configuration.
/// </summary>
/// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
/// <param name="name">The name of the topic.</param>
/// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Topic configuration options.</param>
/// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
[NotNull] string name, [NotNull] IConfiguration configuration)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (string.IsNullOrWhiteSpace(name))
{
throw new ArgumentException($"'{nameof(name)}' must not be null, empty or whitespace.", nameof(name));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
return(options.AddTopicStream(name, configuration.GetSection(KafkaTopicOptions.DefaultSectionName)));
}
public void FromConfiguration_SaslEnabled_ShouldSetValuesOnKafkaStreamOptions()
{
//Arrange
var config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
var expectedConfig = config.GetSection(KafkaOptions.DefaultSectionName).Get <KafkaOptions>();
//Act
var kafkaStreamOptions = new KafkaStreamOptions().FromConfiguration(config);
//Assert
Assert.NotNull(kafkaStreamOptions);
Assert.Equal(expectedConfig.Security.SaslUsername, kafkaStreamOptions.SaslUserName);
Assert.Equal(expectedConfig.Security.SaslPassword, kafkaStreamOptions.SaslPassword);
Assert.Equal(expectedConfig.Security.SaslMechanism, kafkaStreamOptions.SaslMechanism);
Assert.Equal(expectedConfig.Security.SecurityProtocol, kafkaStreamOptions.SecurityProtocol);
}
public KafkaAdapterReceiver(
string providerName,
QueueProperties queueProperties,
KafkaStreamOptions options,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory,
IExternalStreamDeserializer externalDeserializer
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_providerName = providerName;
_queueProperties = queueProperties;
_serializationManager = serializationManager;
_grainFactory = grainFactory;
_externalDeserializer = externalDeserializer;
_logger = loggerFactory.CreateLogger <KafkaAdapterReceiver>();
}
public static KafkaBatchContainer ToBatchContainer(
this ConsumeResult <byte[], byte[]> result,
SerializationManager serializationManager,
KafkaStreamOptions options,
string streamNamespace
)
{
var externalHeader = result.Headers.FirstOrDefault(header => header.Key == options.ExternalMessageIdentifier);
var sequence = new EventSequenceTokenV2(result.Offset.Value);
if (externalHeader != null)
{
var isExternal = BitConverter.ToBoolean(externalHeader.GetValueBytes(), 0);
if (isExternal)
{
var key = Encoding.UTF8.GetString(result.Key);
return(new KafkaBatchContainer(
StreamProviderUtils.GenerateStreamGuid(key),
streamNamespace,
new List <object> {
Encoding.UTF8.GetString(result.Value)
},
null,
isExternalBatch: true,
sequence,
result.TopicPartitionOffset
));
}
}
var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value);
if (batchContainer.SequenceToken == null)
{
batchContainer.SequenceToken = sequence;
}
batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset;
return(batchContainer);
}
public KafkaAdapter(
string providerName,
KafkaStreamOptions options,
IDictionary <string, QueueProperties> queueProperties,
SerializationManager serializationManager,
ILoggerFactory loggerFactory
)
{
_options = options;
_queueProperties = queueProperties;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_logger = _loggerFactory.CreateLogger <KafkaAdapter>();
Name = providerName;
_producer = new Producer <byte[], KafkaBatchContainer>(
options.ToProducerProperties(),
valueSerializer: (topic, container) => container.ToByteArray(_serializationManager)
);
}
public KafkaAdapter(
string providerName,
KafkaStreamOptions options,
IDictionary <string, QueueProperties> queueProperties,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory
)
{
_options = options;
_queueProperties = queueProperties;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_logger = _loggerFactory.CreateLogger <KafkaAdapter>();
Name = providerName;
_producer = new ProducerBuilder <byte[], KafkaBatchContainer>(options.ToProducerProperties())
.SetValueSerializer(new KafkaBatchContainerSerializer(serializationManager))
.Build();
}
public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory,
IExternalStreamDeserializer externalDeserializer
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_name = name;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_externalDeserializer = externalDeserializer;
_logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();
_adminConfig = new AdminClientBuilder(options.ToAdminProperties());
if (options.Topics != null && options.Topics.Count == 0)
{
throw new ArgumentNullException(nameof(options.Topics));
}
_adapterCache = new SimpleQueueAdapterCache(
cacheOptions,
name,
loggerFactory
);
_queueProperties = GetQueuesProperties().ToDictionary(q => q.QueueName);
_streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
_config = _options.ToAdminProperties();
}
public void AddTopicStream_KafkaStreamsConfigurationSectionIsNull_ShouldSetDefaultValuesOnKafkaStreamOptions()
{
var kafkaStreamOptions = new KafkaStreamOptions().AddTopicStream("Test", null);
Assert.NotNull(kafkaStreamOptions);
}
