internal StreamTask GetTask(string topicName)
{
StreamTask task = null;
var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0));
if (tasks.ContainsKey(id))
{
task = tasks[id];
}
else
{
if (!builder.GetGlobalTopics().Contains(topicName))
{
task = new StreamTask("thread-0",
id,
partitionsByTaskId[id],
builder.BuildTopology(id),
supplier.GetConsumer(configuration.ToConsumerConfig(), null),
configuration,
supplier,
producer,
new MockChangelogRegister(),
metricsRegistry);
task.InitializeStateStores();
task.InitializeTopology();
task.RestorationIfNeeded();
task.CompleteRestoration();
tasks.Add(id, task);
}
}
return(task);
}
internal StreamTask GetTask(string topicName)
{
StreamTask task;
var id = builder.GetTaskIdFromPartition(new Confluent.Kafka.TopicPartition(topicName, 0));
if (tasks.ContainsKey(id))
{
task = tasks[id];
}
else
{
task = new StreamTask("thread-0",
id,
partitionsByTaskId[id],
builder.BuildTopology(id),
supplier.GetConsumer(configuration.ToConsumerConfig(), null),
configuration,
supplier,
producer);
task.InitializeStateStores();
task.InitializeTopology();
tasks.Add(id, task);
}
return(task);
}
internal StreamTask GetTask(string topicName)
{
StreamTask task;
if (tasks.ContainsKey(topicName))
{
task = tasks[topicName];
}
else
{
task = new StreamTask("thread-0",
new TaskId {
Id = id++, Partition = 0, Topic = topicName
},
new Confluent.Kafka.TopicPartition(topicName, 0),
builder.BuildTopology(topicName),
supplier.GetConsumer(configuration.ToConsumerConfig(), null),
configuration,
supplier,
producer);
task.InitializeStateStores();
task.InitializeTopology();
tasks.Add(topicName, task);
}
return(task);
}
public void WriteTopologyTest()
{
var builder = new InternalTopologyBuilder();
List <StreamGraphNode> nodes = new List <StreamGraphNode>();
RootNode root = new RootNode();
var source = new StreamSourceNode <string, string>(
"topic",
"source-01",
new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(),
new StringSerDes(), null));
root.AppendChild(source);
nodes.Add(source);
var filterParameters =
new ProcessorParameters <string, string>(
new KStreamFilter <string, string>((k, v) => true, false), "filter-02");
var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters);
source.AppendChild(filter);
nodes.Add(filter);
var to = new StreamSinkNode <string, string>(
new StaticTopicNameExtractor <string, string>("topic2"), "to-03",
new Stream.Internal.Produced <string, string>(
new StringSerDes(),
new StringSerDes())
);
filter.AppendChild(to);
nodes.Add(to);
builder.BuildTopology(root, nodes);
Assert.IsTrue(root.AllParentsWrittenToTopology);
Assert.IsTrue(source.AllParentsWrittenToTopology);
Assert.IsTrue(filter.AllParentsWrittenToTopology);
Assert.IsTrue(to.AllParentsWrittenToTopology);
var topology = builder.BuildTopology();
Assert.IsTrue(topology.SourceOperators.ContainsKey("topic"));
Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02"));
Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2"));
}
internal TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config, Mode mode, IKafkaSupplier supplier)
{
topologyBuilder = builder;
configuration = config;
// ONLY 1 thread for test driver (use only for ASYNC_CLUSTER_IN_MEMORY)
configuration.NumStreamThreads = 1;
configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{configuration.ApplicationId}";
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{Guid.NewGuid()}" : configuration.ClientId;
// sanity check
topologyBuilder.BuildTopology();
topologyBuilder.RewriteTopology(configuration);
switch (mode)
{
case Mode.SYNC_TASK:
behavior = new TaskSynchronousTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
case Mode.ASYNC_CLUSTER_IN_MEMORY:
behavior = new ClusterInMemoryTopologyDriver(
clientId,
topologyBuilder,
configuration,
topicConfiguration,
supplier,
tokenSource.Token);
break;
}
behavior.StartDriver();
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
{
this.startTimeout = startTimeout;
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new KafkaPipeBuilder(kafkaSupplier);
// ONLY FOR CHECK IF TOLOGY IS CORRECT
topologyBuilder.BuildTopology();
threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
topologyBuilder,
this.configuration,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
}
internal void Build()
{
internalTopologyBuilder.BuildTopology(root, nodes);
}