private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config)
{
this.topologyBuilder = builder;
this.configuration = config;
// ONLY 1 thread for test driver
this.configuration.NumStreamThreads = 1;
this.configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
this.topicConfiguration = config.Clone();
this.topicConfiguration.ApplicationId = $"test-driver-{this.configuration.ApplicationId}";
var processID = Guid.NewGuid();
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
this.configuration.ClientId = clientId;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new PipeBuilder(kafkaSupplier);
this.processorTopology = this.topologyBuilder.BuildTopology();
this.threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
builder,
config,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
RunDriver();
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
builder = topologyBuilder;
this.supplier = supplier ?? new SyncKafkaSupplier();
producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
foreach (var sourceTopic in builder.GetSourceTopics().Union(builder.GetGlobalTopics()))
{
var part = new TopicPartition(sourceTopic, 0);
var taskId = builder.GetTaskIdFromPartition(part);
if (partitionsByTaskId.ContainsKey(taskId))
{
partitionsByTaskId[taskId].Add(part);
}
else
{
partitionsByTaskId.Add(taskId, new List <TopicPartition> {
part
});
}
}
}
private IEnumerable <ConsumeResult <string, string> > SetupTest(Action <StreamBuilder> builderFunction, TopologyTestDriver.Mode mode = TopologyTestDriver.Mode.SYNC_TASK, int numberPartition = 1)
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-repartition-processor"
};
StreamBuilder builder = new StreamBuilder();
builderFunction(builder);
Topology t = builder.Build();
IKafkaSupplier supplier = mode == TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY
? new MockKafkaSupplier(numberPartition)
: null;
using (var driver = new TopologyTestDriver(t.Builder, config, mode, supplier))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTopic.PipeInput("test", "test1");
inputTopic.PipeInput("test", "test2");
inputTopic.PipeInput("test", "test3");
var records = IntegrationTestUtils.WaitUntilMinKeyValueRecordsReceived(outputTopic, 3);
var recordsMap = records.ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value);
Assert.IsNotNull(recordsMap);
Assert.AreEqual(1, recordsMap.Count);
Assert.AreEqual("test3", recordsMap["test"]);
return(records);
}
}
public StreamTask(string threadId, TaskId id, TopicPartition partition, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(id, partition, processorTopology, consumer, configuration)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.consumedOffsets = new Dictionary <TopicPartition, long>();
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
this.collector = new RecordCollector(logPrefix);
collector.Init(ref this.producer);
var sourceTimestampExtractor = (processorTopology.GetSourceProcessor(id.Topic) as ISourceProcessor).Extractor;
Context = new ProcessorContext(configuration, stateMgr).UseRecordCollector(collector);
processor = processorTopology.GetSourceProcessor(partition.Topic);
queue = new RecordQueue <ConsumeResult <byte[], byte[]> >(
100,
logPrefix,
$"record-queue-{id.Topic}-{id.Partition}",
sourceTimestampExtractor == null ? configuration.DefaultTimestampExtractor : sourceTimestampExtractor);
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
threads = new IThread[configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < configuration.NumStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base()
{
this.builder = builder;
this.configuration = configuration;
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.producer = producer;
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
builder = topologyBuilder;
supplier = new SyncKafkaSupplier();
producer = supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = kafkaSupplier;
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(this.threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
this.queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders);
StreamState = State.CREATED;
}
public TaskCreator(InternalTopologyBuilder builder, IStreamConfig configuration, string threadId,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, StoreChangelogReader storeChangelogReader,
StreamMetricsRegistry streamMetricsRegistry)
{
this.builder = builder;
this.configuration = configuration;
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.producer = producer;
this.storeChangelogReader = storeChangelogReader;
this.streamMetricsRegistry = streamMetricsRegistry;
createTaskSensor = ThreadMetrics.CreateTaskSensor(threadId, streamMetricsRegistry);
}
public TaskSynchronousTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder,
IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier,
CancellationToken token)
{
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
metricsRegistry = new StreamMetricsRegistry(clientId, MetricsRecordingLevel.DEBUG);
this.token = token;
builder = topologyBuilder;
this.supplier = supplier ?? new SyncKafkaSupplier();
this.supplier.MetricsRegistry = metricsRegistry;
producer = this.supplier.GetProducer(configuration.ToProducerConfig()) as SyncProducer;
foreach (var sourceTopic in builder
.GetSourceTopics())
{
var part = new TopicPartition(sourceTopic, 0);
var taskId = builder.GetTaskIdFromPartition(part);
if (partitionsByTaskId.ContainsKey(taskId))
{
partitionsByTaskId[taskId].Add(part);
}
else
{
partitionsByTaskId.Add(taskId, new List <TopicPartition> {
part
});
}
}
ProcessorTopology globalTaskTopology = topologyBuilder.BuildGlobalStateTopology();
hasGlobalTopology = globalTaskTopology != null;
if (hasGlobalTopology)
{
var globalConsumer =
this.supplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig($"{clientId}-global-consumer"));
var adminClient = this.supplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin"));
var stateManager =
new GlobalStateManager(globalConsumer, globalTaskTopology, adminClient, configuration);
globalProcessorContext = new GlobalProcessorContext(configuration, stateManager, metricsRegistry);
stateManager.SetGlobalProcessorContext(globalProcessorContext);
globalTask = new GlobalStateUpdateTask(stateManager, globalTaskTopology, globalProcessorContext);
globalTask.Initialize();
}
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(id, partitions, processorTopology, consumer, configuration)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
collector = new RecordCollector(logPrefix, configuration, id);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr).UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance.
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
public KafkaStream(Topology topology, IStreamConfig configuration)
{
this.topology = topology;
this.configuration = configuration;
this.kafkaSupplier = new DefaultKafkaClientSupplier(new KafkaLoggerAdapter(configuration));
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
// sanity check
this.processorTopology = topology.Builder.BuildTopology();
this.threads = new IThread[this.configuration.NumStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
for (int i = 0; i < this.configuration.NumStreamThreads; ++i)
{
var threadId = $"{this.configuration.ApplicationId.ToLower()}-stream-thread-{i}";
adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
this.threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(this.threads[i].Id, this.threads[i].State);
}
var manager = new StreamStateManager(this, threadState);
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
StreamState = State.CREATED;
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
{
this.startTimeout = startTimeout;
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new KafkaPipeBuilder(kafkaSupplier);
// ONLY FOR CHECK IF TOLOGY IS CORRECT
topologyBuilder.BuildTopology();
threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
topologyBuilder,
this.configuration,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
}
UnassignedStreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(threadId, id, partitions, processorTopology, consumer, configuration, kafkaSupplier, producer, null, new StreamMetricsRegistry())
{
config = configuration;
}
public PipeBuilder(IKafkaSupplier kafkaSupplier)
{
this.kafkaSupplier = kafkaSupplier;
}
public SyncPipeBuilder(StreamTask task, IKafkaSupplier kafkaSupplier)
{
this.kafkaSupplier = kafkaSupplier;
this.task = task;
}
internal static IThread Create(string threadId, string clientId, InternalTopologyBuilder builder, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IAdminClient adminClient, int threadInd)
{
string logPrefix = $"stream-thread[{threadId}] ";
var log = Logger.GetLogger(typeof(StreamThread));
var customerID = $"{clientId}-StreamThread-{threadInd}";
IProducer <byte[], byte[]> producer = null;
// Due to limitations outlined in KIP-447 (which KIP-447 overcomes), it is
// currently necessary to use a separate producer per input partition. The
// producerState dictionary is used to keep track of these, and the current
// consumed offset.
// https://cwiki.apache.org/confluence/display/KAFKA/KIP-447%3A+Producer+scalability+for+exactly+once+semantics
// IF Guarantee is AT_LEAST_ONCE, producer is the same of all StreamTasks in this thread,
// ELSE one producer by StreamTask.
if (configuration.Guarantee == ProcessingGuarantee.AT_LEAST_ONCE)
{
log.Info($"{logPrefix}Creating shared producer client");
producer = kafkaSupplier.GetProducer(configuration.ToProducerConfig(GetThreadProducerClientId(threadId)));
}
var taskCreator = new TaskCreator(builder, configuration, threadId, kafkaSupplier, producer);
var manager = new TaskManager(builder, taskCreator, adminClient);
var listener = new StreamsRebalanceListener(manager);
log.Info($"{logPrefix}Creating consumer client");
var consumer = kafkaSupplier.GetConsumer(configuration.ToConsumerConfig(customerID), listener);
manager.Consumer = consumer;
var thread = new StreamThread(threadId, customerID, manager, consumer, builder, configuration);
listener.Thread = thread;
return(thread);
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads;
threads = new IThread[numStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology();
bool hasGlobalTopology = globalTaskTopology != null;
if (numStreamThreads == 0 && !hasGlobalTopology)
{
throw new TopologyException("Topology has no stream threads and no global threads, " +
"must subscribe to at least one source topic or global table.");
}
GlobalThreadState globalThreadState = null;
if (hasGlobalTopology)
{
string globalThreadId = $"{clientId}-GlobalStreamThread";
GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology,
globalThreadId,
kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)),
configuration,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId)));
globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread();
globalThreadState = globalStreamThread.State;
}
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < numStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState, globalThreadState);
if (hasGlobalTopology)
{
globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange;
}
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores);
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider);
StreamState = State.CREATED;
}
public SyncPipeBuilder(StreamTask task, IKafkaSupplier kafkaSupplier, SyncProducer mockProducer)
{
this.kafkaSupplier = kafkaSupplier;
this.mockProducer = mockProducer;
this.task = task;
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
StreamMetricsRegistry streamMetricsRegistry)
: base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.streamMetricsRegistry = streamMetricsRegistry;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);
collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
.UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
sourceProcessor.SetTaskId(id);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor,
droppedRecordsSensor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, IKafkaSupplier supplier, CancellationToken token)
: this(clientId, topologyBuilder, configuration, topicConfiguration, TimeSpan.FromSeconds(30), supplier, token)
{
}
// Use for testing (TaskSynchronousTopologyDriver & ClusterInMemoryTopologyDriver) to create source topics before repartition & changelog topcis
internal InternalTopicManagerUtils CreateSourceTopics(InternalTopologyBuilder builder, IKafkaSupplier supplier)
{
var adminConfig = new AdminClientConfig();
adminConfig.ClientId = "internal-admin-create-soure-topic";
var sourceTopics = builder.BuildTopology().GetSourceTopics().ToList();
var globalTopo = builder.BuildGlobalStateTopology();
if (globalTopo != null)
{
sourceTopics.AddRange(globalTopo.StoresToTopics.Values);
}
supplier
.GetAdmin(adminConfig)
.CreateTopicsAsync(sourceTopics.Select(s => new TopicSpecification()
{
Name = s,
NumPartitions = -1
})).GetAwaiter().GetResult();
return(this);
}
internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier)
{
topologyBuilder = builder;
configuration = config;
// ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY)
configuration.NumStreamThreads = 1;
configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}";
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId;
// sanity check
topologyBuilder.BuildTopology();
topologyBuilder.RewriteTopology(configuration);
switch (mode)
{
case Mode.SYNC_TASK:
behavior = new TaskSynchronousTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
case Mode.ASYNC_CLUSTER_IN_MEMORY:
behavior = new ClusterInMemoryTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
}
behavior.StartDriver();
}
