public override void WriteToTopology(InternalTopologyBuilder builder)
{
if (ProcessorParameters != null)
{
builder.AddInternalTopic(RepartitionTopic, NumberOfPartition);
builder.AddProcessor(ProcessorParameters.ProcessorName, ProcessorParameters.Processor,
ParentNodeNames());
builder.AddSinkOperator(
new StaticTopicNameExtractor <K, V>(RepartitionTopic),
SinkName,
Produced <K, V> .Create(KeySerdes, ValueSerdes).WithPartitioner(StreamPartitioner),
ProcessorParameters.ProcessorName);
builder.AddSourceOperator(
RepartitionTopic,
SourceName,
new ConsumedInternal <K, V>(SourceName, KeySerdes, ValueSerdes, new FailOnInvalidTimestamp()));
}
else
{
builder.AddInternalTopic(RepartitionTopic, NumberOfPartition);
builder.AddSinkOperator(
new StaticTopicNameExtractor <K, V>(RepartitionTopic),
SinkName,
Produced <K, V> .Create(KeySerdes, ValueSerdes).WithPartitioner(StreamPartitioner),
ParentNodeNames());
builder.AddSourceOperator(
RepartitionTopic,
SourceName,
new ConsumedInternal <K, V>(SourceName, KeySerdes, ValueSerdes, new FailOnInvalidTimestamp()));
}
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
builder = topologyBuilder;
this.supplier = supplier ?? new SyncKafkaSupplier();
producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
foreach (var sourceTopic in builder.GetSourceTopics().Union(builder.GetGlobalTopics()))
{
var part = new TopicPartition(sourceTopic, 0);
var taskId = builder.GetTaskIdFromPartition(part);
if (partitionsByTaskId.ContainsKey(taskId))
{
partitionsByTaskId[taskId].Add(part);
}
else
{
partitionsByTaskId.Add(taskId, new List <TopicPartition> {
part
});
}
}
}
private StreamThread(string threadId, string clientId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, IChangelogReader storeChangelogReader, StreamMetricsRegistry streamMetricsRegistry, TimeSpan timeSpan, long commitInterval)
{
this.manager = manager;
this.consumer = consumer;
this.builder = builder;
consumeTimeout = timeSpan;
this.threadId = threadId;
this.clientId = clientId;
logPrefix = $"stream-thread[{threadId}] ";
commitTimeMs = commitInterval;
changelogReader = storeChangelogReader;
this.streamMetricsRegistry = streamMetricsRegistry;
thread = new Thread(Run);
thread.Name = this.threadId;
Name = this.threadId;
State = ThreadState.CREATED;
commitSensor = ThreadMetrics.CommitSensor(threadId, streamMetricsRegistry);
pollSensor = ThreadMetrics.PollSensor(threadId, streamMetricsRegistry);
pollRecordsSensor = ThreadMetrics.PollRecordsSensor(threadId, streamMetricsRegistry);
pollRatioSensor = ThreadMetrics.PollRatioSensor(threadId, streamMetricsRegistry);
processLatencySensor = ThreadMetrics.ProcessLatencySensor(threadId, streamMetricsRegistry);
processRecordsSensor = ThreadMetrics.ProcessRecordsSensor(threadId, streamMetricsRegistry);
processRateSensor = ThreadMetrics.ProcessRateSensor(threadId, streamMetricsRegistry);
processRatioSensor = ThreadMetrics.ProcessRatioSensor(threadId, streamMetricsRegistry);
commitRatioSensor = ThreadMetrics.CommitRatioSensor(threadId, streamMetricsRegistry);
}
private StreamThread(string threadId, string clientId, TaskManager manager, IConsumer <byte[], byte[]> consumer,
InternalTopologyBuilder builder, IChangelogReader storeChangelogReader,
StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration)
: this(threadId, clientId, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs)
{
streamConfig = configuration;
}
private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config)
{
this.topologyBuilder = builder;
this.configuration = config;
// ONLY 1 thread for test driver
this.configuration.NumStreamThreads = 1;
this.configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
this.topicConfiguration = config.Clone();
this.topicConfiguration.ApplicationId = $"test-driver-{this.configuration.ApplicationId}";
var processID = Guid.NewGuid();
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
this.configuration.ClientId = clientId;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new PipeBuilder(kafkaSupplier);
this.processorTopology = this.topologyBuilder.BuildTopology();
this.threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
builder,
config,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
RunDriver();
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
// TODO: we assume source KTables can only be timestamped-key-value stores for now.
// should be expanded for other types of stores as well.
StoreBuilder <State.TimestampedKeyValueStore <K, V> > storeBuilder = new TimestampedKeyValueStoreMaterializer <K, V>(materialized as Materialized <K, V, IKeyValueStore <Bytes, byte[]> >).Materialize();
if (isGlobalKTable)
{
builder.AddGlobalStore(topicName, storeBuilder, sourceName, consumed, processorParameters);
}
else
{
builder.AddSourceOperator(this.topicName, sourceName, consumed);
builder.AddProcessor(processorParameters.ProcessorName, processorParameters.Processor, sourceName);
//// only add state store if the source KTable should be materialized
KTableSource <K, V> ktableSource = (KTableSource <K, V>)processorParameters.Processor;
if (ktableSource.QueryableName != null)
{
builder.AddStateStore(storeBuilder, this.streamGraphNode);
// TODO :
//if (shouldReuseSourceTopicForChangelog)
//{
// storeBuilder.withLoggingDisabled();
// topologyBuilder.connectSourceStoreAndTopic(storeBuilder.name(), topicName);
//}
}
}
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
builder = topologyBuilder;
supplier = new SyncKafkaSupplier();
producer = supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
// Stream - Table join (Global or KTable)
builder.AddProcessor(processorParameters.ProcessorName, processorParameters.Processor, ParentNodeNames());
// Steam - KTable join only
if (!string.IsNullOrEmpty(otherJoinSideNodeName))
{
builder.ConnectProcessorAndStateStore(processorParameters.ProcessorName, storeNames);
}
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
string leftProcessorName = JoinLeftParams.ProcessorName;
string rightProcessorName = JoinRightParams.ProcessorName;
string windowedLeftProcessorName = windowedLeftParams.ProcessorName;
string windowedRightProcessorName = windowedRightParams.ProcessorName;
builder.AddProcessor(leftProcessorName, JoinLeftParams.Processor, windowedLeftProcessorName);
builder.AddProcessor(rightProcessorName, JoinRightParams.Processor, windowedRightProcessorName);
builder.AddProcessor(JoinMergeParams.ProcessorName, JoinMergeParams.Processor, leftProcessorName, rightProcessorName);
builder.AddStateStore(windowedLeftStoreBuilder, windowedLeftProcessorName, rightProcessorName);
builder.AddStateStore(windowedRightStoreBuilder, windowedRightProcessorName, leftProcessorName);
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
builder.AddProcessor(ProcessorParameters.ProcessorName, ProcessorParameters.Processor, ParentNodeNames());
if (storeNames != null && storeNames.Length > 0)
{
builder.ConnectProcessorAndStateStore(ProcessorParameters.ProcessorName, storeNames);
}
if (storeBuilder != null)
{
builder.AddStateStore(storeBuilder, ProcessorParameters.ProcessorName);
}
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
builder.AddProcessor(JoinLeftParams.ProcessorName, JoinLeftParams.Processor, LeftJoinSideName);
builder.AddProcessor(JoinRightParams.ProcessorName, JoinRightParams.Processor, RightJoinSideName);
builder.AddProcessor(JoinMergeParams.ProcessorName, JoinMergeProcessorSupplier, JoinLeftParams.ProcessorName, JoinRightParams.ProcessorName);
builder.ConnectProcessorAndStateStore(JoinLeftParams.ProcessorName, leftStoreNames);
builder.ConnectProcessorAndStateStore(JoinRightParams.ProcessorName, rightStoreNames);
if (storeBuilder != null)
{
builder.AddStateStore(storeBuilder, JoinMergeParams.ProcessorName);
}
}
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder,
StreamMetricsRegistry streamMetricsRegistry, IStreamConfig configuration, IKafkaSupplier kafkaSupplier,
IAdminClient adminClient, int threadInd)
{
string logPrefix = $"stream-thread[{threadId}] ";
var log = Logger.GetLogger(typeof(StreamThread));
var customerID = $"{clientId}-StreamThread-{threadInd}";
IProducer <byte[], byte[]> producer = null;
// TODO : remove this limitations depends version of Kafka Cluster
// Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is
// currently necessary to use a separate producer per input partition. The
// producerState dictionary is used to keep track of these, and the current
// consumed offset.
// https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics
// IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread,
// ELSE one producer by StreamTask.
if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE)
{
log.LogInformation("{LogPrefix}Creating shared producer client", logPrefix);
producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)).Wrap(threadId));
}
var restoreConfig = configuration.ToConsumerConfig(GetRestoreConsumerClientId(customerID));
restoreConfig.GroupId = $"{configuration.ApplicationId}-restore-group";
var restoreConsumer = kafkaSupplier.GetRestoreConsumer(restoreConfig);
var storeChangelogReader = new StoreChangelogReader(
configuration,
restoreConsumer,
threadId,
streamMetricsRegistry);
var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer, storeChangelogReader, streamMetricsRegistry);
var manager = new TaskManager(builder, taskCreator, adminClient, storeChangelogReader);
var listener = new StreamsRebalanceListener(manager);
log.LogInformation("{LogPrefix}Creating consumer client", logPrefix);
var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(GetConsumerClientId(customerID)).Wrap(threadId), listener);
manager.Consumer = consumer;
var thread = new StreamThread(threadId, customerID, manager, consumer, builder, storeChangelogReader, streamMetricsRegistry, configuration);
listener.Thread = thread;
return(thread);
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder,
IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier,
CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
metricsRegistry = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG);
this.token = token;
builder = topologyBuilder;
this.supplier = supplier ?? new SyncKafkaSupplier();
this.supplier.MetricsRegistry = metricsRegistry;
producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
foreach (var sourceTopic in builder
.GetSourceTopics())
{
var part = new TopicPartition(sourceTopic, 0);
var taskId = builder.GetTaskIdFromPartition(part);
if (partitionsByTaskId.ContainsKey(taskId))
{
partitionsByTaskId[taskId].Add(part);
}
else
{
partitionsByTaskId.Add(taskId, new List <TopicPartition> {
part
});
}
}
ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology();
hasGlobalTopology = globalTaskTopology != null;
if (hasGlobalTopology)
{
var globalConsumer =
this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer"));
var adminClient = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin"));
var stateManager =
new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration);
globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry);
stateManager.SetGlobalProcessorContext(globalProcessorContext);
globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext);
globalTask.Initialize();
}
}
private StreamThread(string threadId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, TimeSpan timeSpan, long commitInterval)
{
this.manager = manager;
this.consumer = consumer;
this.builder = builder;
consumeTimeout = timeSpan;
logPrefix = $"stream-thread[{threadId}] ";
commitTimeMs = commitInterval;
thread = new Thread(Run);
thread.Name = threadId;
Name = threadId;
State = ThreadState.CREATED;
}
public void WriteTopologyTest()
{
var builder = new InternalTopologyBuilder();
List <StreamGraphNode> nodes = new List <StreamGraphNode>();
RootNode root = new RootNode();
var source = new StreamSourceNode <string, string>(
"topic",
"source-01",
new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(),
new StringSerDes(), null));
root.AppendChild(source);
nodes.Add(source);
var filterParameters =
new ProcessorParameters <string, string>(
new KStreamFilter <string, string>((k, v) => true, false), "filter-02");
var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters);
source.AppendChild(filter);
nodes.Add(filter);
var to = new StreamSinkNode <string, string>(
new StaticTopicNameExtractor <string, string>("topic2"), "to-03",
new Stream.Internal.Produced <string, string>(
new StringSerDes(),
new StringSerDes())
);
filter.AppendChild(to);
nodes.Add(to);
builder.BuildTopology(root, nodes);
Assert.IsTrue(root.AllParentsWrittenToTopology);
Assert.IsTrue(source.AllParentsWrittenToTopology);
Assert.IsTrue(filter.AllParentsWrittenToTopology);
Assert.IsTrue(to.AllParentsWrittenToTopology);
var topology = builder.BuildTopology();
Assert.IsTrue(topology.SourceOperators.ContainsKey("topic"));
Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02"));
Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2"));
}
internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier)
{
topologyBuilder = builder;
configuration = config;
// ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY)
configuration.NumStreamThreads = 1;
configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}";
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId;
// sanity check
topologyBuilder.BuildTopology();
topologyBuilder.RewriteTopology(configuration);
switch (mode)
{
case Mode.SYNC_TASK:
behavior = new TaskSynchronousTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
case Mode.ASYNC_CLUSTER_IN_MEMORY:
behavior = new ClusterInMemoryTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
}
behavior.StartDriver();
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
{
this.startTimeout = startTimeout;
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new KafkaPipeBuilder(kafkaSupplier);
// ONLY FOR CHECK IF TOLOGY IS CORRECT
topologyBuilder.BuildTopology();
threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
topologyBuilder,
this.configuration,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
}
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd)
{
string logPrefix = $"stream-thread[{threadId}] ";
var log = Logger.GetLogger(typeof(StreamThread));
var customerID = $"{clientId}-StreamThread-{threadInd}";
IProducer <byte[], byte[]> producer = null;
// Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is
// currently necessary to use a separate producer per input partition. The
// producerState dictionary is used to keep track of these, and the current
// consumed offset.
// https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics
// IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread,
// ELSE one producer by StreamTask.
if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE)
{
log.Info($"{logPrefix}Creating shared producer client");
producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)));
}
var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer);
var manager = new TaskManager(builder, taskCreator, adminClient);
var listener = new StreamsRebalanceListener(manager);
log.Info($"{logPrefix}Creating consumer client");
var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(customerID), listener);
manager.Consumer = consumer;
var thread = new StreamThread(threadId, customerID, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs);
listener.Thread = thread;
return(thread);
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token)
: this(clientId, topologyBuilder, configuration, topicConfiguration, TimeSpan.FromSeconds(30), token)
{
}
private StreamThread(string threadId, TaskManager manager, IConsumer <byte[], byte[]> consumer, InternalTopologyBuilder builder, IStreamConfig configuration)
: this(threadId, manager, consumer, builder, TimeSpan.FromMilliseconds(configuration.PollMs), configuration.CommitIntervalMs)
{
streamConfig = configuration;
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder,
IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token)
: this(clientId, topologyBuilder, configuration, topicConfiguration, null, token)
{
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
}
public override void WriteToTopology(InternalTopologyBuilder builder)
{
builder.AddSourceOperator(topicName, this.streamGraphNode, consumed);
}
private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode)
: this(builder, config, mode, null)
{
}
internal InternalStreamBuilder(InternalTopologyBuilder internalTopologyBuilder)
{
this.internalTopologyBuilder = internalTopologyBuilder;
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
: this(clientId, topologyBuilder, configuration, topicConfiguration, startTimeout, null, token)
{
}
/// <summary>
/// Constructor without arguments
/// </summary>
public StreamBuilder()
{
internalTopologyBuilder = topology.Builder;
internalStreamBuilder = new InternalStreamBuilder(internalTopologyBuilder);
}
public StreamThreadStateStoreProvider(IThread streamThread, InternalTopologyBuilder internalTopologyBuilder)
{
this.streamThread = streamThread;
this.internalTopologyBuilder = internalTopologyBuilder;
}
public abstract void WriteToTopology(InternalTopologyBuilder builder);
