private TopologyTestDriver(InternalTopologyBuilder builder, IStreamConfig config)
{
this.topologyBuilder = builder;
this.configuration = config;
// ONLY 1 thread for test driver
this.configuration.NumStreamThreads = 1;
this.configuration.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
this.topicConfiguration = config.Clone();
this.topicConfiguration.ApplicationId = $"test-driver-{this.configuration.ApplicationId}";
var processID = Guid.NewGuid();
var clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{this.configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
this.configuration.ClientId = clientId;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new PipeBuilder(kafkaSupplier);
this.processorTopology = this.topologyBuilder.BuildTopology();
this.threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
builder,
config,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
RunDriver();
}
private async Task InitializeInternalTopicManagerAsync()
{
// Create internal topics (changelogs & repartition) if need
var adminClientInternalTopicManager = kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId($"{configuration.ApplicationId.ToLower()}-admin-internal-topic-manager")));
using (var internalTopicManager = new DefaultTopicManager(configuration, adminClientInternalTopicManager))
await InternalTopicManagerUtils.New().CreateInternalTopicsAsync(internalTopicManager, topology.Builder);
}
private void InitializeInternalTopicManager()
{
// Create internal topics (changelogs & repartition) if need
var adminClientInternalTopicManager = supplier.GetAdmin(configuration.ToAdminConfig(
StreamThread.GetSharedAdminClientId(
$"{configuration.ApplicationId.ToLower()}-admin-internal-topic-manager")));
internalTopicManager = new DefaultTopicManager(configuration, adminClientInternalTopicManager);
InternalTopicManagerUtils
.New()
.CreateSourceTopics(builder, supplier)
.CreateInternalTopicsAsync(internalTopicManager, builder)
.GetAwaiter()
.GetResult();
}
public ClusterInMemoryTopologyDriver(string clientId, InternalTopologyBuilder topologyBuilder, IStreamConfig configuration, IStreamConfig topicConfiguration, TimeSpan startTimeout, CancellationToken token)
{
this.startTimeout = startTimeout;
this.configuration = configuration;
this.configuration.ClientId = clientId;
this.topicConfiguration = topicConfiguration;
this.token = token;
kafkaSupplier = new MockKafkaSupplier();
pipeBuilder = new KafkaPipeBuilder(kafkaSupplier);
// ONLY FOR CHECK IF TOLOGY IS CORRECT
topologyBuilder.BuildTopology();
threadTopology = StreamThread.Create(
$"{this.configuration.ApplicationId.ToLower()}-stream-thread-0",
clientId,
topologyBuilder,
this.configuration,
kafkaSupplier,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig($"{clientId}-admin")),
0);
}
// Use for testing (TaskSynchronousTopologyDriver & ClusterInMemoryTopologyDriver) to create source topics before repartition & changelog topcis
internal InternalTopicManagerUtils CreateSourceTopics(InternalTopologyBuilder builder, IKafkaSupplier supplier)
{
var adminConfig = new AdminClientConfig();
adminConfig.ClientId = "internal-admin-create-soure-topic";
var sourceTopics = builder.BuildTopology().GetSourceTopics().ToList();
var globalTopo = builder.BuildGlobalStateTopology();
if (globalTopo != null)
{
sourceTopics.AddRange(globalTopo.StoresToTopics.Values);
}
supplier
.GetAdmin(adminConfig)
.CreateTopicsAsync(sourceTopics.Select(s => new TopicSpecification()
{
Name = s,
NumPartitions = -1
})).GetAwaiter().GetResult();
return(this);
}
/// <summary>
/// Create a <see cref="KafkaStream"/> instance with your own <see cref="IKafkaSupplier" />
/// Please DO NOT FORGET to call Close to avoid resources leak !
/// </summary>
/// <param name="topology">the topology specifying the computational logic</param>
/// <param name="configuration">configuration about this stream</param>
/// <param name="kafkaSupplier">the Kafka clients supplier which provides underlying producer and consumer clients for the new <see cref="KafkaStream"/> instance</param>
public KafkaStream(Topology topology, IStreamConfig configuration, IKafkaSupplier kafkaSupplier)
{
this.topology = topology;
this.kafkaSupplier = kafkaSupplier;
// check if ApplicationId & BootstrapServers has been set
if (string.IsNullOrEmpty(configuration.ApplicationId) || string.IsNullOrEmpty(configuration.BootstrapServers))
{
throw new StreamConfigException($"Stream configuration is not correct. Please set ApplicationId and BootstrapServers as minimal.");
}
var processID = Guid.NewGuid();
clientId = string.IsNullOrEmpty(configuration.ClientId) ? $"{configuration.ApplicationId.ToLower()}-{processID}" : configuration.ClientId;
logPrefix = $"stream-application[{configuration.ApplicationId}] ";
logger.Info($"{logPrefix} Start creation of the stream application with this configuration: {configuration}");
// re-write the physical topology according to the config
topology.Builder.RewriteTopology(configuration);
// sanity check
var processorTopology = topology.Builder.BuildTopology();
int numStreamThreads = topology.Builder.HasNoNonGlobalTopology ? 0 : configuration.NumStreamThreads;
threads = new IThread[numStreamThreads];
var threadState = new Dictionary <long, Processors.ThreadState>();
ProcessorTopology globalTaskTopology = topology.Builder.BuildGlobalStateTopology();
bool hasGlobalTopology = globalTaskTopology != null;
if (numStreamThreads == 0 && !hasGlobalTopology)
{
throw new TopologyException("Topology has no stream threads and no global threads, " +
"must subscribe to at least one source topic or global table.");
}
GlobalThreadState globalThreadState = null;
if (hasGlobalTopology)
{
string globalThreadId = $"{clientId}-GlobalStreamThread";
GlobalStreamThreadFactory globalStreamThreadFactory = new GlobalStreamThreadFactory(globalTaskTopology,
globalThreadId,
kafkaSupplier.GetGlobalConsumer(configuration.ToGlobalConsumerConfig(globalThreadId)),
configuration,
kafkaSupplier.GetAdmin(configuration.ToAdminConfig(clientId)));
globalStreamThread = globalStreamThreadFactory.GetGlobalStreamThread();
globalThreadState = globalStreamThread.State;
}
List <StreamThreadStateStoreProvider> stateStoreProviders = new List <StreamThreadStateStoreProvider>();
for (int i = 0; i < numStreamThreads; ++i)
{
var threadId = $"{configuration.ApplicationId.ToLower()}-stream-thread-{i}";
var adminClient = this.kafkaSupplier.GetAdmin(configuration.ToAdminConfig(StreamThread.GetSharedAdminClientId(clientId)));
threads[i] = StreamThread.Create(
threadId,
clientId,
this.topology.Builder,
configuration,
this.kafkaSupplier,
adminClient,
i);
threadState.Add(threads[i].Id, threads[i].State);
stateStoreProviders.Add(new StreamThreadStateStoreProvider(threads[i], this.topology.Builder));
}
var manager = new StreamStateManager(this, threadState, globalThreadState);
if (hasGlobalTopology)
{
globalStreamThread.StateChanged += manager.OnGlobalThreadStateChange;
}
foreach (var t in threads)
{
t.StateChanged += manager.OnChange;
}
var globalStateStoreProvider = new GlobalStateStoreProvider(topology.Builder.GlobalStateStores);
queryableStoreProvider = new QueryableStoreProvider(stateStoreProviders, globalStateStoreProvider);
StreamState = State.CREATED;
}