public void ApplyInternalChangelogTopicsParrallel()
{
AdminClientConfig config = new AdminClientConfig();
config.BootstrapServers = "localhost:9092";
StreamConfig config2 = new StreamConfig();
DefaultTopicManager manager = new DefaultTopicManager(config2, kafkaSupplier.GetAdmin(config));
IDictionary <string, InternalTopicConfig> topics = new Dictionary <string, InternalTopicConfig>();
topics.Add("topic", new UnwindowedChangelogTopicConfig
{
Name = "topic",
NumberPartitions = 1
});
var r = Parallel.ForEach(new List <int> {
1, 2, 3, 4
},
(i) => manager.ApplyAsync(0, topics).GetAwaiter().GetResult());
Assert.IsTrue(r.IsCompleted);
}
public void ApplyInternalChangelogTopicsWithExistingTopics()
{
AdminClientConfig config = new AdminClientConfig();
config.BootstrapServers = "localhost:9092";
StreamConfig config2 = new StreamConfig();
DefaultTopicManager manager = new DefaultTopicManager(config2, kafkaSupplier.GetAdmin(config));
((SyncProducer)kafkaSupplier.GetProducer(new ProducerConfig())).CreateTopic("topic");
IDictionary <string, InternalTopicConfig> topics = new Dictionary <string, InternalTopicConfig>();
topics.Add("topic", new UnwindowedChangelogTopicConfig
{
Name = "topic",
NumberPartitions = 1
});
topics.Add("topic1", new UnwindowedChangelogTopicConfig
{
Name = "topic1",
NumberPartitions = 1
});
var r = manager.ApplyAsync(0, topics).GetAwaiter().GetResult().ToList();
Assert.AreEqual(1, r.Count);
Assert.AreEqual("topic1", r[0]);
}
private async Task InitializeInternalTopicManagerAsync()
{
// Create internal topics (changelogs & repartition) if need
var adminClientInternalTopicManager = kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId($"{configuration.ApplicationId.ToLower()}-admin-internal-topic-manager")));
using (var internalTopicManager = new DefaultTopicManager(configuration, adminClientInternalTopicManager))
await InternalTopicManagerUtils.New().CreateInternalTopicsAsync(internalTopicManager, topology.Builder);
}
public void Init()
{
token1 = new System.Threading.CancellationTokenSource();
token2 = new System.Threading.CancellationTokenSource();
config.ApplicationId = "test-stream-thread";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(2, 0);
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled());
var topo = builder.Build();
topo.Builder.RewriteTopology(config);
topo.Builder.BuildTopology();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
var internalTopicManager =
new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")));
InternalTopicManagerUtils
.New()
.CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter();
}
public void ApplyInternalChangelogTopicsInvalidDifferentPartition()
{
AdminClientConfig config = new AdminClientConfig();
config.BootstrapServers = "localhost:9092";
StreamConfig config2 = new StreamConfig();
DefaultTopicManager manager = new DefaultTopicManager(config2, kafkaSupplier.GetAdmin(config));
// Create topic with just one partition
((SyncProducer)kafkaSupplier.GetProducer(new ProducerConfig())).CreateTopic("topic");
IDictionary <string, InternalTopicConfig> topics = new Dictionary <string, InternalTopicConfig>();
topics.Add("topic", new UnwindowedChangelogTopicConfig
{
Name = "topic",
NumberPartitions = 4
});
Assert.Throws <StreamsException>(() => manager.ApplyAsync(0, topics).GetAwaiter().GetResult());
}