public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
Joined <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var thisWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var otherWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinThisGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinOtherGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinThisName = named.SuffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
var joinOtherName = named.SuffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode thisStreamsGraphNode = joinLeft.Node;
StreamGraphNode otherStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.Name; // TODO : renamed, and create a StreamJoined DTO
// TODO TO FINISH
return(null);
}
private void ApplyChildNodes(IProcessor value, IProcessor previous, StreamGraphNode root)
{
StreamGraphNode r = null;
while (r == null)
{
if ((root is ITableSourceNode && (((ITableSourceNode)root).SourceName.Equals(value.Name) ||
((ITableSourceNode)root).NodeName.Equals(value.Name))) || value.Name.Equals(root.streamGraphNode))
{
r = root;
break;
}
foreach (var i in root.ChildNodes)
{
if ((i is ITableSourceNode && (((ITableSourceNode)i).SourceName.Equals(value.Name) ||
((ITableSourceNode)i).NodeName.Equals(value.Name))) || value.Name.Equals(i.streamGraphNode))
{
r = i;
}
}
}
if (r != null)
{
value.SetPreviousProcessor(previous);
if (r is ITableSourceNode)
{
var tableSourceProcessor = processorOperators.FirstOrDefault(kp => kp.Key.Equals((r as ITableSourceNode).NodeName)).Value;
if (tableSourceProcessor != null)
{
value.SetNextProcessor(tableSourceProcessor);
value = tableSourceProcessor;
}
}
IList <StreamGraphNode> list = r.ChildNodes;
foreach (var n in list)
{
if (n is StreamSinkNode)
{
var f = sinkOperators.FirstOrDefault(kp => kp.Key.Equals(n.streamGraphNode)).Value;
if (f != null)
{
value.SetNextProcessor(f);
}
}
else if (n is ProcessorGraphNode || n is TableProcessorNode)
{
var f = processorOperators.FirstOrDefault(kp => kp.Key.Equals(n.streamGraphNode)).Value;
if (f != null)
{
value.SetNextProcessor(f);
this.ApplyChildNodes(f, value, n);
}
}
}
}
}
public GroupedStreamAggregateBuilder(InternalStreamBuilder builder, Grouped <K, V> grouped, List <string> sourceNodes, string name, StreamGraphNode node)
{
this.builder = builder;
this.grouped = grouped;
this.sourceNodes = sourceNodes;
this.name = name;
this.node = node;
}
public AbstractStream(AbstractStream <K, V> stream)
{
this.nameNode = stream.nameNode;
this.builder = stream.builder;
this.keySerdes = stream.keySerdes;
this.valueSerdes = stream.valueSerdes;
this.setSourceNodes = stream.setSourceNodes;
this.node = stream.node;
}
protected AbstractStream(String name,
ISerDes <K> keySerde,
ISerDes <V> valSerde,
List <String> sourceNodes,
StreamGraphNode streamsGraphNode,
InternalStreamBuilder builder)
: this(name, keySerde, valSerde, sourceNodes, false, streamsGraphNode, builder)
{
}
protected AbstractStream(AbstractStream <K, V> stream)
{
nameNode = stream.nameNode;
builder = stream.builder;
keySerdes = stream.keySerdes;
valueSerdes = stream.valueSerdes;
setSourceNodes = stream.setSourceNodes;
node = stream.node;
}
internal void AddGraphNode(StreamGraphNode root, StreamGraphNode node)
{
if (logger.IsDebugEnabled)
{
logger.Debug($"Adding node {node} in root node {root}");
}
root.AppendChild(node);
nodes.Add(node);
}
protected AbstractStream(AbstractStream <K, V> stream)
{
NameNode = stream.NameNode;
builder = stream.builder;
KeySerdes = stream.KeySerdes;
ValueSerdes = stream.ValueSerdes;
SetSourceNodes = stream.SetSourceNodes;
Node = stream.Node;
}
internal void AddGraphNode(List <StreamGraphNode> rootNodes, StreamGraphNode node)
{
if (rootNodes.Count == 0)
{
throw new TopologyException("Parent node collection can't be empty");
}
foreach (var p in rootNodes)
{
AddGraphNode(p, node);
}
}
private string Repartition(StoreBuilder storeBuilder)
{
if (repartitionRequired)
{
(string sourceName, RepartitionNode <K, V> repartNode) = KStream <K, V> .CreateRepartitionSource(grouped.Named ?? storeBuilder.Name, grouped.Key, grouped.Value, builder);
if (repartitionNode == null || grouped.Named == null)
{
repartitionNode = repartNode;
}
builder.AddGraphNode(node, repartitionNode);
node = repartitionNode;
return(sourceName);
}
return(name);
}
protected AbstractStream(String name,
ISerDes <K> keySerde,
ISerDes <V> valSerde,
List <String> sourceNodes,
StreamGraphNode streamsGraphNode,
InternalStreamBuilder builder)
{
if (sourceNodes == null || sourceNodes.Count == 0)
{
throw new ArgumentException("parameter <sourceNodes> must not be null or empty");
}
NameNode = name;
this.builder = builder;
KeySerdes = keySerde;
ValueSerdes = valSerde;
SetSourceNodes = sourceNodes;
Node = streamsGraphNode;
}
internal KStream(string name, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, List <string> setSourceNodes, StreamGraphNode node, InternalStreamBuilder builder)
: base(name, keySerdes, valueSerdes, setSourceNodes, node, builder)
{
}
internal KTableGrouped(string name, ISerDes <KR> keySerde, ISerDes <VR> valSerde, List <string> sourceNodes, string queryableStoreName, IProcessorSupplier <K, V> processorSupplier, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
: base(name, keySerde, valSerde, sourceNodes, queryableStoreName, (IProcessorSupplier <KR, V>)null, streamsGraphNode, builder)
{
this.processorSupplier = processorSupplier;
}
public TimeWindowedKStream(WindowOptions <W> windowOptions, GroupedStreamAggregateBuilder <K, V> aggBuilder, string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
: base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder)
{
CheckIfParamNull(windowOptions, "windowOptions");
this.windowOptions = windowOptions;
this.aggBuilder = aggBuilder;
}
public GroupedStreamAggregateBuilder(InternalStreamBuilder builder, List <string> sourceNodes, StreamGraphNode node)
{
this.builder = builder;
this.sourceNodes = sourceNodes;
this.node = node;
}
public KGroupedStream(string name, Grouped <K, V> grouped, List <string> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
: base(name, grouped.Key, grouped.Value, sourceNodes, streamsGraphNode, builder)
{
}
internal KTable(string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, String queryableStoreName, IProcessorSupplier <K, S> processorSupplier, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder)
: base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder)
{
this.processorSupplier = processorSupplier;
this.queryableStoreName = queryableStoreName;
}
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
StreamJoinProps <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode leftStreamsGraphNode = joinLeft.Node;
StreamGraphNode rightStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.StoreName;
var leftStoreSupplier = joined.LeftStoreSupplier;
var rightStoreSupplier = joined.RightStoreSupplier;
StoreBuilder <WindowStore <K, V> > leftWindowStore;
StoreBuilder <WindowStore <K, V0> > rightWindowStore;
AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);
if (leftStoreSupplier == null)
{
var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
}
else
{
AssertWindowSettings(leftStoreSupplier, windows);
leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
}
if (rightStoreSupplier == null)
{
var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
}
else
{
AssertWindowSettings(rightStoreSupplier, windows);
rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
}
var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);
builder.AddGraphNode(leftStreamsGraphNode, leftNode);
var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);
builder.AddGraphNode(rightStreamsGraphNode, rightNode);
var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
var merge = new PassThrough <K, VR>();
var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);
var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
joinMergeName,
joiner,
joinLParams,
joinRParams,
mergeParams,
leftStreamParams,
rightStreamParams,
leftWindowStore,
rightWindowStore,
joined);
builder.AddGraphNode(new List <StreamGraphNode> {
leftStreamsGraphNode, rightStreamsGraphNode
}, joinNode);
ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);
allSourceNodes.AddRange(joinRight.SetSourceNodes);
return(new KStream <K, VR>(
joinMergeName,
joined.KeySerdes,
null,
allSourceNodes.ToList(),
joinNode,
builder));
}
internal void AddGraphNode(StreamGraphNode root, StreamGraphNode node)
{
logger.LogDebug("Adding node {Node} in root node {Root}", node, root);
root.AppendChild(node);
nodes.Add(node);
}
