public void TestConsume()
{
var consumerConfig = new ConsumerConfig();
consumerConfig.GroupId = "cg";
consumerConfig.ClientId = "cg-0";
var supplier = new MockKafkaSupplier(2);
var producer = supplier.GetProducer(new ProducerConfig());
producer.Produce("topic", new Message <byte[], byte[]> {
Key = new byte[1] {
42
}, Value = new byte[1] {
12
}
});
var c1 = supplier.GetConsumer(consumerConfig, null);
c1.Subscribe(new List <string> {
"topic"
});
var item1 = c1.Consume();
Assert.IsNotNull(item1);
var item2 = c1.Consume();
Assert.IsNull(item2);
}
public void TestAssignment()
{
var consumerConfig = new ConsumerConfig();
var consumerConfig2 = new ConsumerConfig();
consumerConfig.GroupId = "cg";
consumerConfig2.GroupId = "cg";
consumerConfig.ClientId = "cg-0";
consumerConfig2.ClientId = "cg-1";
var supplier = new MockKafkaSupplier(2);
var c1 = supplier.GetConsumer(consumerConfig, null);
var c2 = supplier.GetConsumer(consumerConfig2, null);
c1.Subscribe(new List <string> {
"topic1", "topic2"
});
c2.Subscribe(new List <string> {
"topic1", "topic2"
});
c1.Consume();
Assert.AreEqual(2, c1.Assignment.Count);
Assert.AreEqual(2, c2.Assignment.Count);
}
public void TestConsumeRecords2()
{
var consumerConfig = new ConsumerConfig();
consumerConfig.GroupId = "cg";
consumerConfig.ClientId = "cg-0";
var supplier = new MockKafkaSupplier(2);
var producer = supplier.GetProducer(new ProducerConfig());
producer.Produce("topic", new Message <byte[], byte[]> {
Key = new byte[1] {
42
}, Value = new byte[1] {
12
}
});
var c1 = supplier.GetConsumer(consumerConfig, null);
c1.Subscribe(new List <string> {
"topic"
});
var item = c1.ConsumeRecords(TimeSpan.FromSeconds(1)).ToList();
Assert.IsNotNull(item);
Assert.AreEqual(1, item.Count);
}
public void Init()
{
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(4);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
public void Initialize()
{
streamMetricsRegistry
= new StreamMetricsRegistry(Guid.NewGuid().ToString(),
MetricsRecordingLevel.INFO);
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
config.MaxPollRecords = 10;
config.CommitIntervalMs = 10;
config.MetricsRecording = MetricsRecordingLevel.INFO;
mockKafkaSupplier = new MockKafkaSupplier(numberPartitions);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread = StreamThread.Create(
threadId, "c1",
topo.Builder, streamMetricsRegistry, config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
public void TestPauseResumeAssign()
{
var consumerConfig = new ConsumerConfig();
consumerConfig.GroupId = "cg";
consumerConfig.ClientId = "cg-0";
var supplier = new MockKafkaSupplier(1);
var producer = supplier.GetProducer(new ProducerConfig());
producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
Key = new byte[1] {
1
}, Value = new byte[1] {
1
}
});
producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
Key = new byte[1] {
2
}, Value = new byte[1] {
2
}
});
var c1 = supplier.GetConsumer(consumerConfig, null);
c1.Assign(new TopicPartition("topic", 0).ToSingle());
var r1 = c1.Consume();
var r2 = c1.Consume();
var r3 = c1.Consume();
Assert.IsNotNull(r1);
Assert.IsNotNull(r2);
Assert.IsNull(r3);
c1.Pause(c1.Assignment);
c1.Assign(new List <TopicPartition>());
c1.Assign(new TopicPartition("topic", 0).ToSingle());
producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
Key = new byte[1] {
3
}, Value = new byte[1] {
3
}
});
c1.Resume(new TopicPartition("topic", 0).ToSingle());
var r4 = c1.Consume();
Assert.IsNotNull(r4);
}
public void RepartitionTestJoinTopology()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-repartition-processor"
};
var supplier = new MockKafkaSupplier(10);
StreamBuilder builder = new StreamBuilder();
IKStream <string, string> stream1 = builder
.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v));
IKStream <string, string> stream2 = builder.Stream <string, string>("topic2");
stream1.Join(stream2,
(v1, v2) => $"{v1}-{v2}",
JoinWindowOptions.Of(TimeSpan.FromMinutes(1)),
StreamJoinProps.As <string, string, string>("join-store"))
.To("output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
var inputTopic2 = driver.CreateInputTopic <string, string>("topic2");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTopic.PipeInput("test", "coucou");
inputTopic2.PipeInput("TEST", "sylvain");
inputTopic2.PipeInput("TEST2", "antoine");
inputTopic.PipeInput("test2", "test");
var records = IntegrationTestUtils
.WaitUntilMinKeyValueRecordsReceived(outputTopic, 2)
.ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value);
Assert.IsNotNull(records);
Assert.AreEqual(2, records.Count);
Assert.AreEqual("coucou-sylvain", records["TEST"]);
Assert.AreEqual("test-antoine", records["TEST2"]);
}
}
public void RepartitionInternalTopicOnCascace()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-repartition-processor-on-cascade"
};
StreamBuilder builder = new StreamBuilder();
var table = builder.Table <string, string>("input-table");
builder
.Stream <string, string>("topic")
.SelectKey((k, v) => v.ToUpper())
.Join(table, (v, t) => $"{v}:{t}")
.To("output");
Topology t = builder.Build();
MockKafkaSupplier supplier = new MockKafkaSupplier(4);
using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
var inputTableTopic = driver.CreateInputTopic <string, string>("input-table");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTableTopic.PipeInput("PRODUCT1", "P1");
inputTableTopic.PipeInput("PRODUCT2", "P2");
inputTopic.PipeInput("test", "product1");
inputTopic.PipeInput("test", "product2");
var records = IntegrationTestUtils
.WaitUntilMinKeyValueRecordsReceived(outputTopic, 2)
.ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value);
Assert.IsNotNull(records);
Assert.AreEqual(2, records.Count);
Assert.AreEqual("product2:P2", records["PRODUCT2"]);
Assert.AreEqual("product1:P1", records["PRODUCT1"]);
}
}
public void Init()
{
token1 = new System.Threading.CancellationTokenSource();
token2 = new System.Threading.CancellationTokenSource();
config.ApplicationId = "test-stream-thread";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(2, 0);
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled());
var topo = builder.Build();
topo.Builder.RewriteTopology(config);
topo.Builder.BuildTopology();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
var internalTopicManager =
new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")));
InternalTopicManagerUtils
.New()
.CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter();
}
public void RepartitionTestTopologyAsyncHighNumberPartititon()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-repartition-processor"
};
var supplier = new MockKafkaSupplier(30);
StreamBuilder builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.GroupByKey()
.Count()
.ToStream()
.To("output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
var outputTopic = driver.CreateOuputTopic <string, long, StringSerDes, Int64SerDes>("output");
inputTopic.PipeInput("test", "test1");
inputTopic.PipeInput("test", "test2");
inputTopic.PipeInput("test", "test3");
inputTopic.PipeInput("test", "test4");
inputTopic.PipeInput("test", "test5");
var records = IntegrationTestUtils
.WaitUntilMinKeyValueRecordsReceived(outputTopic, 5)
.ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value);
Assert.IsNotNull(records);
Assert.AreEqual(1, records.Count);
Assert.AreEqual(5, records["TEST"]);
}
}
public void StreamThreadNormalWorkflowWithRebalancing()
{
List <ThreadState> allStates = new List <ThreadState>();
var expectedStates = new List <ThreadState>
{
ThreadState.CREATED,
ThreadState.STARTING,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PARTITIONS_REVOKED,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PENDING_SHUTDOWN,
ThreadState.DEAD
};
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
var consumeConfig = config.Clone();
consumeConfig.ApplicationId = "consume-test";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new MockKafkaSupplier(4);
var producer = supplier.GetProducer(consumeConfig.ToProducerConfig());
var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
allStates.Add(thread.State);
thread.StateChanged += (t, o, n) =>
{
Assert.IsInstanceOf <ThreadState>(n);
allStates.Add(n as ThreadState);
};
thread.Start(source.Token);
// WAIT PARTITONS ASSIGNED
System.Threading.Thread.Sleep(50);
var thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
thread2.Start(source.Token);
// WAIT PARTITONS REBALANCING
System.Threading.Thread.Sleep(50);
producer.Produce("topic", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
//WAIT STREAMTHREAD PROCESS MESSAGE
System.Threading.Thread.Sleep(100);
var message = consumer.Consume(100);
// 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS
Assert.AreEqual(2, thread.ActiveTasks.Count());
Assert.AreEqual(2, thread2.ActiveTasks.Count());
source.Cancel();
thread.Dispose();
thread2.Dispose();
Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
Assert.AreEqual(expectedStates, allStates);
// Destroy in memory cluster
supplier.Destroy();
}
public void TestConsumeSaveOffset()
{
var consumerConfig = new ConsumerConfig();
consumerConfig.GroupId = "cg";
consumerConfig.ClientId = "cg-0";
var consumerConfig1 = new ConsumerConfig();
consumerConfig1.GroupId = "cg";
consumerConfig1.ClientId = "cg-1";
var supplier = new MockKafkaSupplier(2);
var producer = supplier.GetProducer(new ProducerConfig());
producer.Produce(new TopicPartition("topic", 0), new Message <byte[], byte[]> {
Key = new byte[1] {
42
}, Value = new byte[1] {
12
}
});
producer.Produce(new TopicPartition("topic", 1), new Message <byte[], byte[]> {
Key = new byte[1] {
43
}, Value = new byte[1] {
13
}
});
var c1 = supplier.GetConsumer(consumerConfig, null);
var c2 = supplier.GetConsumer(consumerConfig1, null);
c1.Subscribe(new List <string> {
"topic"
});
c1.Consume();
c2.Subscribe(new List <string> {
"topic"
});
c2.Consume();
c1.Consume();
c2.Consume();
Assert.AreEqual(1, c1.Assignment.Count);
Assert.AreEqual(1, c2.Assignment.Count);
c1.Unsubscribe();
c2.Unsubscribe();
c1.Subscribe(new List <string> {
"topic"
});
c1.Consume();
c2.Subscribe(new List <string> {
"topic"
});
c2.Consume();
c2.Consume();
c1.Consume();
c2.Consume();
Assert.AreEqual(1, c1.Assignment.Count);
Assert.AreEqual(1, c2.Assignment.Count);
}