public void Init()
{
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(4);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
public void Initialize()
{
streamMetricsRegistry
= new StreamMetricsRegistry(Guid.NewGuid().ToString(),
MetricsRecordingLevel.INFO);
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
config.MaxPollRecords = 10;
config.CommitIntervalMs = 10;
config.MetricsRecording = MetricsRecordingLevel.INFO;
mockKafkaSupplier = new MockKafkaSupplier(numberPartitions);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread = StreamThread.Create(
threadId, "c1",
topo.Builder, streamMetricsRegistry, config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
public void Init()
{
token1 = new System.Threading.CancellationTokenSource();
token2 = new System.Threading.CancellationTokenSource();
config.ApplicationId = "test-stream-thread";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(2, 0);
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled());
var topo = builder.Build();
topo.Builder.RewriteTopology(config);
topo.Builder.BuildTopology();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
var internalTopicManager =
new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")));
InternalTopicManagerUtils
.New()
.CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter();
}
public void StreamThreadNormalWorkflowWithRebalancing()
{
List <ThreadState> allStates = new List <ThreadState>();
var expectedStates = new List <ThreadState>
{
ThreadState.CREATED,
ThreadState.STARTING,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PARTITIONS_REVOKED,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PENDING_SHUTDOWN,
ThreadState.DEAD
};
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
var consumeConfig = config.Clone();
consumeConfig.ApplicationId = "consume-test";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new MockKafkaSupplier(4);
var producer = supplier.GetProducer(consumeConfig.ToProducerConfig());
var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
allStates.Add(thread.State);
thread.StateChanged += (t, o, n) =>
{
Assert.IsInstanceOf <ThreadState>(n);
allStates.Add(n as ThreadState);
};
thread.Start(source.Token);
// WAIT PARTITONS ASSIGNED
System.Threading.Thread.Sleep(50);
var thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
thread2.Start(source.Token);
// WAIT PARTITONS REBALANCING
System.Threading.Thread.Sleep(50);
producer.Produce("topic", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
//WAIT STREAMTHREAD PROCESS MESSAGE
System.Threading.Thread.Sleep(100);
var message = consumer.Consume(100);
// 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS
Assert.AreEqual(2, thread.ActiveTasks.Count());
Assert.AreEqual(2, thread2.ActiveTasks.Count());
source.Cancel();
thread.Dispose();
thread2.Dispose();
Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
Assert.AreEqual(expectedStates, allStates);
// Destroy in memory cluster
supplier.Destroy();
}