private ProcessorTopology BuildTopology(ISet<string> nodeGroup)
{
IProcessor rootProcessor = new RootProcessor();
IDictionary<string, IProcessor> sources = new Dictionary<string, IProcessor>();
IDictionary<string, IProcessor> sinks = new Dictionary<string, IProcessor>();
IDictionary<string, IProcessor> processors = new Dictionary<string, IProcessor>();
IDictionary<string, IStateStore> stateStores = new Dictionary<string, IStateStore>();
foreach (var nodeFactory in nodeFactories.Values)
{
if(nodeGroup == null || nodeGroup.Contains(nodeFactory.Name))
{
var processor = nodeFactory.Build();
processors.Add(nodeFactory.Name, processor);
if (nodeFactory is IProcessorNodeFactory)
BuildProcessorNode(processors, stateStores, nodeFactory as IProcessorNodeFactory, processor);
else if (nodeFactory is ISourceNodeFactory)
BuildSourceNode(sources, nodeFactory as ISourceNodeFactory, processor);
else if (nodeFactory is ISinkNodeFactory)
BuildSinkNode(processors, sinks, nodeFactory as ISinkNodeFactory, processor);
else
throw new TopologyException($"Unknown definition class: {nodeFactory.GetType().Name}");
}
}
foreach (var sourceProcessor in sources.Values)
rootProcessor.AddNextProcessor(sourceProcessor);
return new ProcessorTopology(rootProcessor, sources, sinks, processors, stateStores);
}
private ProcessorTopology BuildTopology(ISet <string> nodeGroup, TaskId taskId)
{
// need refactor a little for repartition topic/processor source & sink etc .. change topic name
IProcessor rootProcessor = new RootProcessor();
IDictionary <string, IProcessor> sources = new Dictionary <string, IProcessor>();
IDictionary <string, IProcessor> sinks = new Dictionary <string, IProcessor>();
IDictionary <string, IProcessor> processors = new Dictionary <string, IProcessor>();
IDictionary <string, IStateStore> stateStores = new Dictionary <string, IStateStore>();
IList <string> repartitionTopics = new List <string>();
foreach (var nodeFactory in nodeFactories.Values)
{
if (nodeGroup == null || nodeGroup.Contains(nodeFactory.Name))
{
var processor = nodeFactory.Build();
processors.Add(nodeFactory.Name, processor);
if (nodeFactory is IProcessorNodeFactory)
{
BuildProcessorNode(processors, stateStores, nodeFactory as IProcessorNodeFactory, processor, taskId);
}
else if (nodeFactory is ISourceNodeFactory)
{
BuildSourceNode(sources, repartitionTopics, nodeFactory as ISourceNodeFactory, processor);
}
else if (nodeFactory is ISinkNodeFactory)
{
BuildSinkNode(processors, repartitionTopics, sinks, nodeFactory as ISinkNodeFactory, processor);
}
else
{
throw new TopologyException($"Unknown definition class: {nodeFactory.GetType().Name}");
}
}
}
foreach (var sourceProcessor in sources.Values)
{
rootProcessor.AddNextProcessor(sourceProcessor);
}
var storesToChangelog = new Dictionary <string, string>(
storesToTopics
.Where(e => stateStores.ContainsKey(e.Key) ||
GlobalStateStores.ContainsKey(e.Key)));
return(new ProcessorTopology(
rootProcessor,
sources,
sinks,
processors,
stateStores,
GlobalStateStores,
storesToChangelog,
repartitionTopics.Distinct().ToList()));
}
private ProcessorTopology BuildTopology(ISet <string> nodeGroup, TaskId taskId)
{
IProcessor rootProcessor = new RootProcessor();
IDictionary <string, IProcessor> sources = new Dictionary <string, IProcessor>();
IDictionary <string, IProcessor> sinks = new Dictionary <string, IProcessor>();
IDictionary <string, IProcessor> processors = new Dictionary <string, IProcessor>();
IDictionary <string, IStateStore> stateStores = new Dictionary <string, IStateStore>();
foreach (var nodeFactory in nodeFactories.Values)
{
if (nodeGroup == null || nodeGroup.Contains(nodeFactory.Name))
{
var processor = nodeFactory.Build();
processors.Add(nodeFactory.Name, processor);
switch (nodeFactory)
{
case IProcessorNodeFactory factory:
BuildProcessorNode(processors, stateStores, factory, processor, taskId);
break;
case ISourceNodeFactory sourceNodeFactory:
BuildSourceNode(sources, sourceNodeFactory, processor);
break;
case ISinkNodeFactory sinkNodeFactory:
BuildSinkNode(processors, sinks, sinkNodeFactory, processor);
break;
default:
throw new TopologyException($"Unknown definition class: {nodeFactory.GetType().Name}");
}
}
}
foreach (var sourceProcessor in sources.Values)
{
rootProcessor.AddNextProcessor(sourceProcessor);
}
return(new ProcessorTopology(rootProcessor, sources, sinks, processors, stateStores, GlobalStateStores, storesToTopics));
}
