public void Initialize()
{
streamMetricsRegistry
= new StreamMetricsRegistry(Guid.NewGuid().ToString(),
MetricsRecordingLevel.INFO);
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
config.MaxPollRecords = 10;
config.CommitIntervalMs = 10;
config.MetricsRecording = MetricsRecordingLevel.INFO;
mockKafkaSupplier = new MockKafkaSupplier(numberPartitions);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread = StreamThread.Create(
threadId, "c1",
topo.Builder, streamMetricsRegistry, config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
threads = new IThread[configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < configuration.NumStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config)
{
this.topologyBuilder = builder;
this.configuration = config;
// ONLY 1 thread for test driver
this.configuration.NumStreamThreads = 1;
this.configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
this.topicConfiguration = config.Clone();
this.topicConfiguration.ApplicationId = $"test-driver-{this.configuration.ApplicationId}";
var processID = Guid.NewGuid();
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
this.configuration.ClientId = clientId;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new PipeBuilder(kafkaSupplier);
this.processorTopology = this.topologyBuilder.BuildTopology();
this.threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
builder,
config,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
RunDriver();
}
public void CheckSetStateStartingWithDeadThread()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
// MUST BE IN CREATED STATE
Assert.AreEqual(ThreadState.CREATED, thread.State);
thread.SetState(ThreadState.STARTING);
thread.SetState(ThreadState.PENDING_SHUTDOWN);
thread.SetState(ThreadState.DEAD);
thread.Start(default);
private async Task CreatePostAsync(string streamId, string threadId, string url)
{
if (url.IsNullOrWhiteSpace())
{
Log.Debug($"{nameof(CreatePostAsync)} url is missing");
await Task.CompletedTask;
}
else
{
try {
var streamResponse = await _codeStreamAgent.GetStreamAsync(streamId);
if (streamResponse != null)
{
var streamThread = new StreamThread(threadId, streamResponse.Stream);
await _codeStreamAgent.CreatePostAsync(streamThread.Stream.Id,
streamThread.Id, $"Join my Live Share session: {url}");
_sessionService.LiveShareUrl = url;
}
}
catch (Exception ex) {
Log.Warning(ex, "Could not post Live Share url");
}
}
}
public void GetCurrentPartitionMetadataTests()
{
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
config.FollowMetadata = true;
var configConsumer = config.Clone();
configConsumer.ApplicationId = "test-consumer";
int?h = null;
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.MapValues((v) =>
{
h = StreamizMetadata.GetCurrentPartitionMetadata();
return(v);
})
.To("output");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(configConsumer.ToConsumerConfig(), null);
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread.Start(source.Token);
producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
consumer.Subscribe("output");
ConsumeResult <byte[], byte[]> result = null;
do
{
result = consumer.Consume(100);
} while (result == null);
source.Cancel();
thread.Dispose();
Assert.NotNull(h);
Assert.AreEqual(0, h);
}
public void Init()
{
config.ApplicationId = "test-stream-thread";
config.StateDir = Guid.NewGuid().ToString();
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(4);
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
}
public void GetTaskProducerClientIdTest()
{
var taskId = new TaskId {
Id = 0, Partition = 0
};
var result = StreamThread.GetTaskProducerClientId("thread-client", taskId);
Assert.AreEqual($"thread-client-0-0-producer", result);
}
public void StreamThreadCommitIntervalWorkflow()
{
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
config.CommitIntervalMs = 1;
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread.Start(source.Token);
producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
//WAIT STREAMTHREAD PROCESS MESSAGE
System.Threading.Thread.Sleep(100);
var message = consumer.Consume(100);
Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
var offsets = thread.GetCommittedOffsets(new List <TopicPartition> {
new TopicPartition("topic", 0)
},
TimeSpan.FromSeconds(10)).ToList();
Assert.AreEqual(1, offsets.Count);
Assert.AreEqual(1, offsets[0].Offset.Value);
Assert.AreEqual(0, offsets[0].TopicPartition.Partition.Value);
Assert.AreEqual("topic", offsets[0].Topic);
source.Cancel();
thread.Dispose();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = kafkaSupplier;
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(this.threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
this.queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
private void InitializeInternalTopicManager()
{
// Create internal topics (changelogs & repartition) if need
var adminClientInternalTopicManager = supplier.GetAdmin(configuration.ToAdminConfig(
StreamThread.GetSharedAdminClientId(
$"{configuration.ApplicationId.ToLower()}-admin-internal-topic-manager")));
internalTopicManager = new DefaultTopicManager(configuration, adminClientInternalTopicManager);
InternalTopicManagerUtils
.New()
.CreateSourceTopics(builder, supplier)
.CreateInternalTopicsAsync(internalTopicManager, builder)
.GetAwaiter()
.GetResult();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance.
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
public KafkaStream(Topology topology, IStreamConfig configuration)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = new DefaultKafkaClientSupplier(new KafkaLoggerAdapter(configuration));
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
StreamState = State.CREATED;
}
public void CreateStreamThread()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
Assert.AreEqual("thread-0", thread.Name);
}
public void Init()
{
token1 = new System.Threading.CancellationTokenSource();
token2 = new System.Threading.CancellationTokenSource();
config.ApplicationId = "test-stream-thread";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(2, 0);
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled());
var topo = builder.Build();
topo.Builder.RewriteTopology(config);
topo.Builder.BuildTopology();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
var internalTopicManager =
new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")));
InternalTopicManagerUtils
.New()
.CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter();
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
{
this.startTimeout = startTimeout;
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new KafkaPipeBuilder(kafkaSupplier);
// ONLY FOR CHECK IF TOLOGY IS CORRECT
topologyBuilder.BuildTopology();
threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
topologyBuilder,
this.configuration,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
}
public void CheckIncorrectStateTransition()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
// MUST BE IN CREATED STATE
Assert.AreEqual(ThreadState.CREATED, thread.State);
Assert.Throws <StreamsException>(() => thread.SetState(ThreadState.DEAD));
}
public void CheckSetStateWithoutStateChangedHandler()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
// MUST BE IN CREATED STATE
Assert.AreEqual(ThreadState.CREATED, thread.State);
thread.SetState(ThreadState.STARTING);
Assert.AreEqual(ThreadState.STARTING, thread.State);
}
public void GetSharedAdminClientIdTest()
{
var result = StreamThread.GetSharedAdminClientId("thread-client");
Assert.AreEqual($"thread-client-admin", result);
}
//[Test]
// TODO : fix that
public void WorkflowCompleteBufferedRecordsTest()
{
int maxBuffered = 10;
var token = new System.Threading.CancellationTokenSource();
var serdes = new StringSerDes();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-group";
config.MaxTaskIdleMs = (long)TimeSpan.FromSeconds(100).TotalMilliseconds;
config.BufferedRecordsPerPartition = maxBuffered;
config.PollMs = 10;
var builder = new StreamBuilder();
var stream1 = builder.Stream <string, string>("topic1");
var stream2 = builder.Stream <string, string>("topic2");
stream1
.Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10)))
.To("output");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("output");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread.Start(token.Token);
for (int i = 0; i < maxBuffered + 1; ++i)
{
producer.Produce("topic1", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize($"coucou{i}", new SerializationContext())
});
}
// CONSUME PAUSE AFTER maxBuffered + 1 messages
System.Threading.Thread.Sleep(50);
// Add one message more with consumer in stream thread in pause
producer.Produce("topic1", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize($"coucou{maxBuffered+1}", new SerializationContext())
});
Assert.AreEqual(1, thread.ActiveTasks.Count());
var task = thread.ActiveTasks.ToArray()[0];
Assert.IsNotNull(task.Grouper);
Assert.IsFalse(task.Grouper.AllPartitionsBuffered);
Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered());
Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered(new TopicPartition("topic1", 0)));
Assert.AreEqual(0, task.Grouper.NumBuffered(new TopicPartition("topic2", 0)));
producer.Produce("topic2", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize($"test", new SerializationContext())
});
List <ConsumeResult <byte[], byte[]> > records = new List <ConsumeResult <byte[], byte[]> >();
do
{
records.AddRange(consumer.ConsumeRecords(TimeSpan.FromMilliseconds(100)).ToList());
} while (records.Count() <= 12);
Assert.AreEqual(maxBuffered + 2, records.Count());
for (int i = 0; i < maxBuffered + 2; ++i)
{
var message = records.ToArray()[i];
Assert.AreEqual("key", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.IsTrue(serdes.Deserialize(message.Message.Value, new SerializationContext()).Contains($"coucou{i}-"));
}
token.Cancel();
thread.Dispose();
}
public void StreamThreadNormalWorkflowWithRebalancing()
{
List <ThreadState> allStates = new List <ThreadState>();
var expectedStates = new List <ThreadState>
{
ThreadState.CREATED,
ThreadState.STARTING,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PARTITIONS_REVOKED,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PENDING_SHUTDOWN,
ThreadState.DEAD
};
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
var consumeConfig = config.Clone();
consumeConfig.ApplicationId = "consume-test";
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new MockKafkaSupplier(4);
var producer = supplier.GetProducer(consumeConfig.ToProducerConfig());
var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
allStates.Add(thread.State);
thread.StateChanged += (t, o, n) =>
{
Assert.IsInstanceOf <ThreadState>(n);
allStates.Add(n as ThreadState);
};
thread.Start(source.Token);
// WAIT PARTITONS ASSIGNED
System.Threading.Thread.Sleep(50);
var thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
thread2.Start(source.Token);
// WAIT PARTITONS REBALANCING
System.Threading.Thread.Sleep(50);
producer.Produce("topic", new Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
//WAIT STREAMTHREAD PROCESS MESSAGE
System.Threading.Thread.Sleep(100);
var message = consumer.Consume(100);
// 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS
Assert.AreEqual(2, thread.ActiveTasks.Count());
Assert.AreEqual(2, thread2.ActiveTasks.Count());
source.Cancel();
thread.Dispose();
thread2.Dispose();
Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
Assert.AreEqual(expectedStates, allStates);
// Destroy in memory cluster
supplier.Destroy();
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads;
threads = new IThread[numStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology();
bool hasGlobalTopology = globalTaskTopology != null;
if (numStreamThreads == 0 && !hasGlobalTopology)
{
throw new TopologyException("Topology has no stream threads and no global threads, " +
"must subscribe to at least one source topic or global table.");
}
GlobalThreadState globalThreadState = null;
if (hasGlobalTopology)
{
string globalThreadId = $"{clientId}-GlobalStreamThread";
GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology,
globalThreadId,
kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)),
configuration,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId)));
globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread();
globalThreadState = globalStreamThread.State;
}
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < numStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState, globalThreadState);
if (hasGlobalTopology)
{
globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange;
}
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores);
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider);
StreamState = State.CREATED;
}
private async Task InitializeInternalTopicManagerAsync()
{
// Create internal topics (changelogs & repartition) if need
var adminClientInternalTopicManager = kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId($"{configuration.ApplicationId.ToLower()}-admin-internal-topic-manager")));
using (var internalTopicManager = new DefaultTopicManager(configuration, adminClientInternalTopicManager))
await InternalTopicManagerUtils.New().CreateInternalTopicsAsync(internalTopicManager, topology.Builder);
}
public void GetRestoreConsumerClientIdTest()
{
var result = StreamThread.GetRestoreConsumerClientId("thread-client");
Assert.AreEqual($"thread-client-restore-consumer", result);
}
public void StreamThreadNormalWorkflow()
{
bool metricsReporterCalled = false;
List <ThreadState> allStates = new List <ThreadState>();
var expectedStates = new List <ThreadState>
{
ThreadState.CREATED,
ThreadState.STARTING,
ThreadState.PARTITIONS_ASSIGNED,
ThreadState.RUNNING,
ThreadState.PENDING_SHUTDOWN,
ThreadState.DEAD
};
var source = new System.Threading.CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 1;
config.MetricsReporter = (sensor) => { metricsReporterCalled = true; };
config.AddOrUpdate(StreamConfig.metricsIntervalMsCst, 10);
var serdes = new StringSerDes();
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var topo = builder.Build();
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null);
consumer.Subscribe("topic2");
var thread = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
supplier, supplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
allStates.Add(thread.State);
thread.StateChanged += (t, o, n) =>
{
Assert.IsInstanceOf <ThreadState>(n);
allStates.Add(n as ThreadState);
};
thread.Start(source.Token);
producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
//WAIT STREAMTHREAD PROCESS MESSAGE
System.Threading.Thread.Sleep(100);
var message = consumer.Consume(100);
source.Cancel();
thread.Dispose();
Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext()));
Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext()));
Assert.AreEqual(expectedStates, allStates);
Assert.IsTrue(metricsReporterCalled);
}
public void GetThreadProducerClientIdTest()
{
var result = StreamThread.GetThreadProducerClientId("thread-client");
Assert.AreEqual($"thread-client-producer", result);
}
