public KafkaAdapterFactory(
string name,
KafkaStreamOptions options,
SimpleQueueCacheOptions cacheOptions,
SerializationManager serializationManager,
ILoggerFactory loggerFactory,
IGrainFactory grainFactory,
IExternalStreamDeserializer externalDeserializer
)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_name = name;
_serializationManager = serializationManager;
_loggerFactory = loggerFactory;
_grainFactory = grainFactory;
_externalDeserializer = externalDeserializer;
_logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();
_adminConfig = new AdminClientBuilder(options.ToAdminProperties());
if (options.Topics != null && options.Topics.Count == 0)
{
throw new ArgumentNullException(nameof(options.Topics));
}
_adapterCache = new SimpleQueueAdapterCache(
cacheOptions,
name,
loggerFactory
);
_queueProperties = GetQueuesProperties().ToDictionary(q => q.QueueName);
_streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
_config = _options.ToAdminProperties();
}
/// <summary>
/// Remove all topics used by this example if they exist.
/// </summary>
static async Task DeleteTopics(string brokerList)
{
var config = new AdminClientConfig {
BootstrapServers = brokerList
};
using (var adminClent = new AdminClientBuilder(config).Build())
{
try
{
await adminClent.DeleteTopicsAsync(new List <string> {
Topic_InputLines, Topic_Words, Topic_Counts
});
}
catch (DeleteTopicsException e)
{
// propagate the exception unless the error was that one or more of the topics didn't exist.
if (e.Results.Select(r => r.Error.Code).Where(el => el != ErrorCode.UnknownTopicOrPart && el != ErrorCode.NoError).Count() > 0)
{
throw new Exception("Unable to delete topics", e);
}
}
}
}
public ICollection <string> FetchTopics(IEnumerable <string> topicNames)
{
if (topicNames == null)
{
throw new ArgumentNullException(nameof(topicNames));
}
try
{
var config = new AdminClientConfig(_kafkaOptions.MainConfig)
{
BootstrapServers = _kafkaOptions.Servers
};
using var adminClient = new AdminClientBuilder(config).Build();
adminClient.CreateTopicsAsync(topicNames.Select(x => new TopicSpecification
{
Name = x
})).GetAwaiter().GetResult();
}
catch (CreateTopicsException ex) when(ex.Message.Contains("already exists"))
{
}
catch (Exception ex)
{
var logArgs = new LogMessageEventArgs
{
LogType = MqLogType.ConsumeError,
Reason = $"An error was encountered when automatically creating topic! -->" + ex.Message
};
OnLog?.Invoke(null, logArgs);
}
return(topicNames.ToList());
}
// Use for testing (TaskSynchronousTopologyDriver & ClusterInMemoryTopologyDriver) to create source topics before repartition & changelog topcis
internal InternalTopicManagerUtils CreateSourceTopics(InternalTopologyBuilder builder, IKafkaSupplier supplier)
{
var adminConfig = new AdminClientConfig();
adminConfig.ClientId = "internal-admin-create-soure-topic";
var sourceTopics = builder.BuildTopology().GetSourceTopics().ToList();
var globalTopo = builder.BuildGlobalStateTopology();
if (globalTopo != null)
{
sourceTopics.AddRange(globalTopo.StoresToTopics.Values);
}
supplier
.GetAdmin(adminConfig)
.CreateTopicsAsync(sourceTopics.Select(s => new TopicSpecification()
{
Name = s,
NumPartitions = -1
})).GetAwaiter().GetResult();
return(this);
}
public static async Task CreateTopic(string host, string topic)
{
var adminClientConfig = new AdminClientConfig();
adminClientConfig.BootstrapServers = host;
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
try
{
var topicSpecification = new TopicSpecification()
{
Name = topic,
ReplicationFactor = 1,
NumPartitions = 1
};
await adminClient.CreateTopicsAsync(new TopicSpecification[] { topicSpecification });
}
catch (Exception ex)
{
throw new Exception($"{nameof(KafkaCommon)} failed to create topic {topic}", ex);
}
}
}
private async Task <bool> connectToKafka()
{
try
{
//var log = new BlockingCollection<string>();
int port = this.Kafka.Ports.Single();
var config = new AdminClientConfig
{
BootstrapServers = this.Ip + ":" + port,
}; // + ports.First()};
using (var client = new AdminClientBuilder(config).Build())
//.SetLogHandler((adminClient, message) =>log.Add(message.Message)).Build()
{
await this.createMissingTopics(client);
}
return(true);
}
catch
{
return(false);
}
}
public KafkaTopicScalerForTest(string topic, string consumerGroup, string functionId, IConsumer <TKey, TValue> consumer, AdminClientConfig adminClientConfig, long lagThreshold, ILogger logger)
: base(topic, consumerGroup, functionId, consumer, adminClientConfig, lagThreshold, logger)
{
}
public KafkaTopicCreationService(IOptions <KafkaOptions> optionsProvider, ILogger <KafkaTopicCreationService> logger, AdminClientConfig adminClientConfig)
{
_optionsProvider = optionsProvider;
_logger = logger;
_adminClientConfig = adminClientConfig;
}
public async Task Should_bypass_if_created()
{
var services = new ServiceCollection();
const ushort partitionCount = 2;
const short replicaCount = BrokersCount;
const string topicName = "do-not-create-topic";
TaskCompletionSource <ConsumeContext <KafkaMessage> > taskCompletionSource = GetTask <ConsumeContext <KafkaMessage> >();
services.TryAddSingleton <ILoggerFactory>(LoggerFactory);
services.TryAddSingleton(typeof(ILogger <>), typeof(Logger <>));
services.AddSingleton(taskCompletionSource);
services.AddMassTransit(x =>
{
x.UsingInMemory((context, cfg) => cfg.ConfigureEndpoints(context));
x.AddRider(rider =>
{
rider.AddConsumer <KafkaMessageConsumer>();
rider.AddProducer <KafkaMessage>(topicName);
rider.UsingKafka((context, k) =>
{
k.Host(Host);
k.TopicEndpoint <KafkaMessage>(topicName, nameof(ConfigureTopology_Specs), c =>
{
c.AutoOffsetReset = AutoOffsetReset.Earliest;
c.ConfigureConsumer <KafkaMessageConsumer>(context);
c.CreateIfMissing(t =>
{
t.NumPartitions = partitionCount;
t.ReplicationFactor = replicaCount;
});
});
});
});
});
var provider = services.BuildServiceProvider();
var busControl = provider.GetRequiredService <IBusControl>();
var config = new AdminClientConfig {
BootstrapServers = Host
};
var client = new AdminClientBuilder(config).Build();
var specification = new TopicSpecification
{
Name = topicName,
NumPartitions = partitionCount,
ReplicationFactor = replicaCount
};
await client.CreateTopicsAsync(new[] { specification });
await busControl.StartAsync(TestCancellationToken);
var serviceScope = provider.CreateScope();
var producer = serviceScope.ServiceProvider.GetRequiredService <ITopicProducer <KafkaMessage> >();
try
{
await producer.Produce(new { Text = "text" }, TestCancellationToken);
ConsumeContext <KafkaMessage> result = await taskCompletionSource.Task;
Assert.NotNull(result);
}
finally
{
await busControl.StopAsync(TestCancellationToken);
await provider.DisposeAsync();
try
{
await client.DeleteTopicsAsync(new[] { topicName });
}
catch (DeleteTopicsException)
{
//suppress
}
finally
{
client.Dispose();
}
}
}
public IAdminClient GetAdmin(AdminClientConfig config)
{
return(new MockAdminClient());
}
internal void UseConfig(AdminClientConfig config)
{
this.config = config;
}
public IAdminClient GetAdmin(AdminClientConfig config)
{
return(new MockAdminClient(cluster, config.ClientId));
}
public void Subscribe <TData>(string topic, Action <MessageData <TData> > action)
{
if (_consumers.ContainsKey(topic))
{
_logger?.LogError($"Already subscribe {topic}");
return;
}
if (_options.PartitionTopics.Contains(topic))
{
var adminClientConfig = new AdminClientConfig();
SetClientConfig(adminClientConfig);
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
PrepareTopic(adminClient, topic);
}
}
var config = new ConsumerConfig
{
GroupId = _options.ConsumerGroup,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
SetClientConfig(config);
var consumer = new ConsumerBuilder <Null, byte[]>(config).Build();
consumer.Subscribe(topic);
_consumers.TryAdd(topic, consumer);
Task.Factory.StartNew(() =>
{
_logger.LogInformation("Subscribe: " + topic);
while (_consumers.ContainsKey(topic))
{
TransferMessage msg = null;
try
{
var value = consumer.Consume().Value;
msg = LZ4MessagePackSerializer.Deserialize <TransferMessage>(value,
TypelessContractlessStandardResolver.Instance);
}
catch (ObjectDisposedException)
{
_logger?.LogDebug("Kafka handler is disposed");
}
catch (Exception e)
{
_logger?.LogError($"Consume kafka message failed on topic {topic}: {e}");
}
if (msg != null)
{
try
{
action?.Invoke(new MessageData <TData>
{
Timestamp = msg.Timestamp,
Type = msg.Type,
Data = LZ4MessagePackSerializer.Deserialize <TData>(msg.Data,
TypelessContractlessStandardResolver.Instance)
});
}
catch (Exception e)
{
_logger?.LogError($"Handle kafka message failed on topic {topic}: {e}");
}
}
else
{
_logger?.LogWarning($"Ignore empty kafka message on topic {topic}");
}
}
_logger?.LogWarning($"Exit consume kafka topic {topic}");
}).ConfigureAwait(false).GetAwaiter();
}
public KafkaClient(ProducerConfig producerConfig, AdminClientConfig adminClientConfig)
{
_producer = new ProducerBuilder <string, string>(producerConfig).Build();
_adminClient = new AdminClientBuilder(adminClientConfig).Build();
}
public static void CancellationDelayMax(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start CancellationDelayMax");
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
EnablePartitionEof = false,
CancellationDelayMaxMs = 2
};
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
CancellationDelayMaxMs = 2
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers,
CancellationDelayMaxMs = 2
};
using (var topic = new TemporaryTopic(bootstrapServers, 3))
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
consumer.Subscribe(topic.Name);
// for the consumer, check that the cancellation token is honored.
for (int i = 0; i < 20; ++i)
{
var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(2));
var sw = Stopwatch.StartNew();
try
{
var record = consumer.Consume(cts.Token);
}
catch (OperationCanceledException)
{
// expected.
}
// 2ms + 2ms + quite a bit of leeway (but still much less than the default of 50).
// in practice this should 4 almost all of the time.
var elapsed = sw.ElapsedMilliseconds;
Skip.If(elapsed > 8);
}
consumer.Close();
// for the producer, make do with just a simple check that this does not throw or hang.
var dr = producer.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 255 }
}).Result;
// for the admin client, make do with just simple check that this does not throw or hang.
var cr = new Confluent.Kafka.Admin.ConfigResource {
Type = ResourceType.Topic, Name = topic.Name
};
var configs = adminClient.DescribeConfigsAsync(new ConfigResource[] { cr }).Result;
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end CancellationDelayMax");
}
public async Task Should_create_on_start()
{
var services = new ServiceCollection();
const ushort partitionCount = 2;
const short replicaCount = BrokersCount;
const string topicName = "create-topic";
services.TryAddSingleton <ILoggerFactory>(LoggerFactory);
services.TryAddSingleton(typeof(ILogger <>), typeof(Logger <>));
services.AddMassTransit(x =>
{
x.UsingInMemory((context, cfg) => cfg.ConfigureEndpoints(context));
x.AddRider(rider =>
{
rider.UsingKafka((context, k) =>
{
k.Host(Host);
k.TopicEndpoint <KafkaMessage>(topicName, nameof(ConfigureTopology_Specs), c =>
{
c.CreateIfMissing(t =>
{
t.NumPartitions = partitionCount;
t.ReplicationFactor = replicaCount;
});
});
});
});
});
var provider = services.BuildServiceProvider();
var busControl = provider.GetRequiredService <IBusControl>();
var config = new AdminClientConfig {
BootstrapServers = Host
};
var client = new AdminClientBuilder(config).Build();
await busControl.StartAsync(TestCancellationToken);
try
{
var meta = client.GetMetadata(topicName, TimeSpan.FromSeconds(10));
Assert.AreEqual(1, meta.Topics.Count);
foreach (var topic in meta.Topics)
{
Assert.AreEqual(partitionCount, topic.Partitions.Count);
foreach (var partition in topic.Partitions)
{
Assert.AreEqual(replicaCount, partition.Replicas.Length);
}
}
await Task.Delay(1000);
}
finally
{
await busControl.StopAsync(TestCancellationToken);
await provider.DisposeAsync();
try
{
await client.DeleteTopicsAsync(new[] { topicName });
}
catch (DeleteTopicsException)
{
//suppress
}
finally
{
client.Dispose();
}
}
}
public static void AddEventBus(this IServiceCollection services, ProducerConfig producerConfig, ConsumerConfig consumerConfig, AdminClientConfig adminCientConfig)
{
services.AddSingleton <IEventBusSubscriptionManager, EventBusSubscriptionManager>();
//注册EventBus
services.AddSingleton <IEventBus, KafkaEventBus>(sp =>
{
var eventBus = new KafkaEventBus(
producerConfig,
consumerConfig,
adminCientConfig,
sp.GetRequiredService <ILogger <KafkaEventBus> >(),
sp.GetRequiredService <IEventBusSubscriptionManager>(),
sp
);
eventBus.SubscribeAll();
return(eventBus);
});
//注册EventHandlers
var eventHandlers = GetEventHandlers();
if (eventHandlers != null && eventHandlers.Any())
{
eventHandlers.ToList().ForEach(x => services.AddTransient(x));
}
}
public void LogDelegate(string bootstrapServers)
{
LogToFile("start LogDelegate");
var logCount = 0;
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
Debug = "all"
};
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers,
Debug = "all"
};
var adminConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers,
Debug = "all"
};
DeliveryResult <byte[], byte[]> dr;
using (var producer =
new ProducerBuilder <byte[], byte[]>(producerConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> {
Value = Serializers.Utf8.Serialize("test value", SerializationContext.Empty)
}).Result;
producer.Flush(TimeSpan.FromSeconds(10));
}
Assert.True(logCount > 0);
logCount = 0;
using (var consumer =
new ConsumerBuilder <byte[], byte[]>(consumerConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
consumer.Consume(TimeSpan.FromSeconds(10));
}
Assert.True(logCount > 0);
logCount = 0;
using (var adminClient =
new AdminClientBuilder(adminConfig)
.SetLogHandler((_, m) => logCount += 1)
.Build())
{
adminClient.GetMetadata(TimeSpan.FromSeconds(1));
}
Assert.True(logCount > 0);
Assert.Equal(0, Library.HandleCount);
LogToFile("end LogDelegate");
}
public static void ProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
SchemaRegistryUrl = schemaRegistryServers
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, User>(producerConfig)
.SetKeySerializer(new AsyncAvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AsyncAvroSerializer <User>(schemaRegistry))
.Build())
{
for (int i = 0; i < 100; ++i)
{
var user = new User
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, User> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, User>(consumerConfig)
.SetKeyDeserializer(new AsyncAvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AsyncAvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
}
public ConfigureTopologyFilter(IReadOnlyDictionary <string, string> clientConfig, KafkaTopicOptions options)
{
_options = options;
_specification = _options.ToSpecification();
_config = new AdminClientConfig(clientConfig.ToDictionary(x => x.Key, x => x.Value));
}
private static void BMSpecificProduceConsume(string bootstrapServers, string schemaRegistryServers)
{
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
var topic = Guid.NewGuid().ToString();
Console.Error.WriteLine($"topic: {topic}");
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
var srClient = new CachedSchemaRegistryClient(schemaRegistryConfig);
Schema schema1 = new Schema(EventA._SCHEMA.ToString(), SchemaType.Avro);
Schema schema2 = new Schema(EventB._SCHEMA.ToString(), SchemaType.Avro);
var id1 = srClient.RegisterSchemaAsync("events-a", schema1).Result;
var id2 = srClient.RegisterSchemaAsync("events-b", schema2).Result;
var avroUnion = @"[""Confluent.Kafka.Examples.AvroSpecific.EventA"",""Confluent.Kafka.Examples.AvroSpecific.EventB""]";
Schema unionSchema = new Schema(avroUnion, SchemaType.Avro);
SchemaReference reference = new SchemaReference(
"Confluent.Kafka.Examples.AvroSpecific.EventA",
"events-a",
srClient.GetLatestSchemaAsync("events-a").Result.Version);
unionSchema.References.Add(reference);
reference = new SchemaReference(
"Confluent.Kafka.Examples.AvroSpecific.EventB",
"events-b",
srClient.GetLatestSchemaAsync("events-b").Result.Version);
unionSchema.References.Add(reference);
var id3 = srClient.RegisterSchemaAsync($"{topic}-value", unionSchema).Result;
AvroSerializerConfig avroSerializerConfig = new AvroSerializerConfig {
AutoRegisterSchemas = false, UseLatestSchemaVersion = true
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ISpecificRecord>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new BmSpecificSerializerImpl <ISpecificRecord>(
schemaRegistry,
false,
1024,
SubjectNameStrategy.Topic.ToDelegate(),
true))
.Build())
{
for (int i = 0; i < 3; ++i)
{
var eventA = new EventA
{
A = "I'm event A",
EventId = Guid.NewGuid().ToString(),
EventType = "EventType-A",
OccuredOn = DateTime.UtcNow.Ticks,
};
producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
Key = "DomainEvent", Value = eventA
}).Wait();
}
for (int i = 0; i < 3; ++i)
{
var eventB = new EventB
{
B = 123456987,
EventId = Guid.NewGuid().ToString(),
EventType = "EventType-B",
OccuredOn = DateTime.UtcNow.Ticks,
};
producer.ProduceAsync(topic, new Message <string, ISpecificRecord> {
Key = "DomainEvent", Value = eventB
}).Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new BmGenericDeserializerImpl(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Console.WriteLine(record.Message.Value["EventType"]);
i += 1;
}
Assert.Equal(6, i);
consumer.Close();
}
}
private static void ProduceConsumeNull(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
};
var avroSerializerConfig = new AvroSerializerConfig
{
SubjectNameStrategy = nameStrategy
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
// Test ValueSubjectNameStrategy here,
// and KeySubjectNameStrategy in ProduceConsumeGeneric.
.SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry, avroSerializerConfig))
.Build())
{
for (int i = 0; i < 100; ++i)
{
producer
.ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
Key = null, Value = null
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Null(record.Message.Key);
Assert.Null(record.Message.Value);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
}
public KafkaAdmin(AdminClientConfig config)
{
_innerClient = new AdminClientBuilder(config).Build();
}
public KafkaTopicScaler(string topic, string consumerGroup, string functionId, IConsumer <TKey, TValue> consumer, AdminClientConfig adminClientConfig, long lagThreshold, ILogger logger)
{
if (string.IsNullOrWhiteSpace(topic))
{
throw new ArgumentException("Invalid topic", nameof(topic));
}
if (string.IsNullOrWhiteSpace(consumerGroup))
{
throw new ArgumentException("Invalid consumer group", nameof(consumerGroup));
}
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.adminClientConfig = adminClientConfig ?? throw new ArgumentNullException(nameof(adminClientConfig));
this.consumer = consumer ?? throw new ArgumentNullException(nameof(consumer));
this.topicName = topic;
this.Descriptor = new ScaleMonitorDescriptor($"{functionId}-kafkatrigger-{topicName}-{consumerGroup}".ToLower());
this.topicPartitions = new Lazy <List <TopicPartition> >(LoadTopicPartitions);
this.consumerGroup = consumerGroup;
this.lagThreshold = lagThreshold;
}
public void Subscribe(string topic, Action <Event> action)
{
if (_consumers.ContainsKey(topic))
{
_logger?.LogError($"Already subscribe {topic}");
return;
}
if (_options.PartitionTopics.Contains(topic))
{
var adminClientConfig = new AdminClientConfig();
SetClientConfig(adminClientConfig);
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
PrepareTopic(adminClient, topic);
}
}
var config = new ConsumerConfig
{
GroupId = _options.ConsumerGroup,
// Note: The AutoOffsetReset property determines the start offset in the event
// there are not yet any committed offsets for the consumer group for the
// topic/partitions of interest. By default, offsets are committed
// automatically, so in this example, consumption will only start from the
// earliest message in the topic 'my-topic' the first time you run the program.
AutoOffsetReset = AutoOffsetReset.Earliest
};
SetClientConfig(config);
var consumer = new ConsumerBuilder <Null, Event>(config)
.SetValueDeserializer(new ProtobufDeserializer <Event>()).Build();
consumer.Subscribe(topic);
_consumers.TryAdd(topic, consumer);
Task.Factory.StartNew(() =>
{
_logger.LogInformation("Subscribe: " + topic);
while (_consumers.ContainsKey(topic))
{
Event msg = null;
try
{
msg = consumer.Consume().Value;
}
catch (Exception e)
{
_logger?.LogError($"Consume kafka message failed on topic {topic}: {e}");
}
if (msg != null)
{
try
{
action(msg);
}
catch (Exception e)
{
_logger?.LogError($"Handle kafka message failed on topic {topic}: {e}");
}
}
else
{
_logger?.LogWarning($"Ignore empty kafka message on topic {topic}");
}
}
_logger?.LogWarning($"Exit consume kafka topic {topic}");
}).ConfigureAwait(false).GetAwaiter();
}
public KafkaHealthChecker(AdminClientConfig configuration, string name = "ApacheKafka") : base(name)
{
this.Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
this.Name = name;
this.AdminClientBuilder = new AdminClientBuilder(Configuration);
}
public IAdminClient GetAdmin(AdminClientConfig config) => new SyncAdminClient();
static async Task Main()
{
const string KAFKA_BOOTSTRAP_SERVER = "KAFKA_BOOTSTRAP_SERVER";
const string KAFKA_TOPIC = "KAFKA_TOPIC";
const string METRIC_ENDPOINT = "METRIC_ENDPOINT";
const string METRIC_METHOD = "METRIC_METHOD";
const string METRIC_INTERVAL = "METRIC_INTERVAL";
const string METRIC_HEADERS = "METRIC_HEADERS";
string GetEnvironmentOrThrow(string env, string info)
{
var value = Environment.GetEnvironmentVariable(env);
if (string.IsNullOrEmpty(value))
{
throw new Exception($"Expected {info} in ENV {env}!");
}
return(value);
}
#region Setup
var server = GetEnvironmentOrThrow(KAFKA_BOOTSTRAP_SERVER, "a Kafka bootstrap server connection string");
var topic = GetEnvironmentOrThrow(KAFKA_TOPIC, "a Kafka topic");
var metricEndpoint = GetEnvironmentOrThrow(METRIC_ENDPOINT, "a metric endpoint URL");
var metricMethod = GetEnvironmentOrThrow(METRIC_METHOD, "a HTTP method").ToUpper();
var metricInterval = int.Parse(GetEnvironmentOrThrow(METRIC_INTERVAL, "an interval in ms"));
var metricHeader = GetEnvironmentOrThrow(METRIC_HEADERS, "HTTP headers in JSON format");
var producerConfig = new ProducerConfig {
BootstrapServers = server
};
var producer = new ProducerBuilder <Null, string>(producerConfig).Build();
#region Try to connect to Kafka
var adminClientConfig = new AdminClientConfig {
BootstrapServers = server
};
var adminClient = new AdminClientBuilder(adminClientConfig).Build();
Metadata metadata = null;
while (metadata == null)
{
try
{
metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(1));
}
catch (Exception)
{
adminClientConfig = new AdminClientConfig {
BootstrapServers = server
};
adminClient = new AdminClientBuilder(adminClientConfig).Build();
}
}
Console.WriteLine($"Connected to Kafka broker {metadata.OriginatingBrokerName}!");
#endregion
var client = new HttpClient();
//Manual switch since Enum.parse sometimes returns a default value when it really shouldn't
var method = metricMethod switch
{
"GET" => HttpMethod.Get,
"PUT" => HttpMethod.Put,
"POST" => HttpMethod.Post,
"DELETE" => HttpMethod.Delete,
"PATCH" => HttpMethod.Patch,
"OPTIONS" => HttpMethod.Options,
"HEAD" => HttpMethod.Head,
_ => throw new Exception($"Invalid HTTP method {metricMethod}")
};
var headers = JsonConvert.DeserializeObject <Dictionary <string, string> >(metricHeader).ToList();
#endregion
#region Data load
while (true)
{
var request = new HttpRequestMessage
{
RequestUri = new Uri(metricEndpoint),
Method = method,
};
headers.ForEach(entry =>
{
request.Headers.Add(entry.Key, entry.Value);
});
var response = await client.SendAsync(request);
var responseBody = await response.Content.ReadAsStringAsync();
producer.Produce(topic, new Message <Null, string> {
Value = responseBody
});
Thread.Sleep(metricInterval);
}
#endregion
}
}
public AdminClient(AdminClientConfig config)
{
client = new AdminClientBuilder(config).Build();
}
private static void ProduceConsume(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var producerConfig = new ProducerConfig
{
BootstrapServers = bootstrapServers
};
var consumerConfig = new ConsumerConfig
{
BootstrapServers = bootstrapServers,
GroupId = Guid.NewGuid().ToString(),
SessionTimeoutMs = 6000,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnablePartitionEof = true
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
// Test ValueSubjectNameStrategy here,
// and KeySubjectNameStrategy in ProduceConsumeGeneric.
ValueSubjectNameStrategy = nameStrategy
};
var adminClientConfig = new AdminClientConfig
{
BootstrapServers = bootstrapServers
};
string topic = Guid.NewGuid().ToString();
using (var adminClient = new AdminClientBuilder(adminClientConfig).Build())
{
adminClient.CreateTopicsAsync(
new List <TopicSpecification> {
new TopicSpecification {
Name = topic, NumPartitions = 1, ReplicationFactor = 1
}
}).Wait();
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var producer =
new ProducerBuilder <string, ProduceConsumeUser>(producerConfig)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <ProduceConsumeUser>(schemaRegistry))
.Build())
{
for (int i = 0; i < 100; ++i)
{
var user = new ProduceConsumeUser
{
name = i.ToString(),
favorite_number = i,
favorite_color = "blue"
};
producer
.ProduceAsync(topic, new Message <string, ProduceConsumeUser> {
Key = user.name, Value = user
})
.Wait();
}
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10)));
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <string, ProduceConsumeUser>(consumerConfig)
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(new AvroDeserializer <ProduceConsumeUser>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Assert.True(false, e.Reason))
.Build())
{
consumer.Subscribe(topic);
int i = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
Assert.Equal(i.ToString(), record.Message.Key);
Assert.Equal(i.ToString(), record.Message.Value.name);
Assert.Equal(i, record.Message.Value.favorite_number);
Assert.Equal("blue", record.Message.Value.favorite_color);
i += 1;
}
Assert.Equal(100, i);
consumer.Close();
}
// Check that what's in schema registry is what's expected.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
if (nameStrategy == SubjectNameStrategy.TopicRecord)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
if (nameStrategy == SubjectNameStrategy.Topic)
{
Assert.Equal(2, (int)subjects.Where(s => s.Contains(topic)).Count());
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{topic}-value"));
}
if (nameStrategy == SubjectNameStrategy.Record)
{
Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser._SCHEMA).Fullname}"));
}
}
}
