public KStream <K, VR> Join <K, V, V0, VR>( KStream <K, V> joinLeft, KStream <K, V0> joinRight, IValueJoiner <V, V0, VR> joiner, JoinWindowOptions windows, Joined <K, V, V0> joined) { var named = new Named(joined.Name); var joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join"; var joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join"; var thisWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME); var otherWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME); var joinThisGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME); var joinOtherGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME); var joinThisName = named.SuffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName); var joinOtherName = named.SuffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName); var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME); StreamGraphNode thisStreamsGraphNode = joinLeft.Node; StreamGraphNode otherStreamsGraphNode = joinRight.Node; var userProvidedBaseStoreName = joined.Name; // TODO : renamed, and create a StreamJoined DTO // TODO TO FINISH return(null); }
private void ApplyChildNodes(IProcessor value, IProcessor previous, StreamGraphNode root) { StreamGraphNode r = null; while (r == null) { if ((root is ITableSourceNode && (((ITableSourceNode)root).SourceName.Equals(value.Name) || ((ITableSourceNode)root).NodeName.Equals(value.Name))) || value.Name.Equals(root.streamGraphNode)) { r = root; break; } foreach (var i in root.ChildNodes) { if ((i is ITableSourceNode && (((ITableSourceNode)i).SourceName.Equals(value.Name) || ((ITableSourceNode)i).NodeName.Equals(value.Name))) || value.Name.Equals(i.streamGraphNode)) { r = i; } } } if (r != null) { value.SetPreviousProcessor(previous); if (r is ITableSourceNode) { var tableSourceProcessor = processorOperators.FirstOrDefault(kp => kp.Key.Equals((r as ITableSourceNode).NodeName)).Value; if (tableSourceProcessor != null) { value.SetNextProcessor(tableSourceProcessor); value = tableSourceProcessor; } } IList <StreamGraphNode> list = r.ChildNodes; foreach (var n in list) { if (n is StreamSinkNode) { var f = sinkOperators.FirstOrDefault(kp => kp.Key.Equals(n.streamGraphNode)).Value; if (f != null) { value.SetNextProcessor(f); } } else if (n is ProcessorGraphNode || n is TableProcessorNode) { var f = processorOperators.FirstOrDefault(kp => kp.Key.Equals(n.streamGraphNode)).Value; if (f != null) { value.SetNextProcessor(f); this.ApplyChildNodes(f, value, n); } } } } }
public GroupedStreamAggregateBuilder(InternalStreamBuilder builder, Grouped <K, V> grouped, List <string> sourceNodes, string name, StreamGraphNode node) { this.builder = builder; this.grouped = grouped; this.sourceNodes = sourceNodes; this.name = name; this.node = node; }
public AbstractStream(AbstractStream <K, V> stream) { this.nameNode = stream.nameNode; this.builder = stream.builder; this.keySerdes = stream.keySerdes; this.valueSerdes = stream.valueSerdes; this.setSourceNodes = stream.setSourceNodes; this.node = stream.node; }
protected AbstractStream(String name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <String> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) : this(name, keySerde, valSerde, sourceNodes, false, streamsGraphNode, builder) { }
protected AbstractStream(AbstractStream <K, V> stream) { nameNode = stream.nameNode; builder = stream.builder; keySerdes = stream.keySerdes; valueSerdes = stream.valueSerdes; setSourceNodes = stream.setSourceNodes; node = stream.node; }
internal void AddGraphNode(StreamGraphNode root, StreamGraphNode node) { if (logger.IsDebugEnabled) { logger.Debug($"Adding node {node} in root node {root}"); } root.AppendChild(node); nodes.Add(node); }
protected AbstractStream(AbstractStream <K, V> stream) { NameNode = stream.NameNode; builder = stream.builder; KeySerdes = stream.KeySerdes; ValueSerdes = stream.ValueSerdes; SetSourceNodes = stream.SetSourceNodes; Node = stream.Node; }
internal void AddGraphNode(List <StreamGraphNode> rootNodes, StreamGraphNode node) { if (rootNodes.Count == 0) { throw new TopologyException("Parent node collection can't be empty"); } foreach (var p in rootNodes) { AddGraphNode(p, node); } }
private string Repartition(StoreBuilder storeBuilder) { if (repartitionRequired) { (string sourceName, RepartitionNode <K, V> repartNode) = KStream <K, V> .CreateRepartitionSource(grouped.Named ?? storeBuilder.Name, grouped.Key, grouped.Value, builder); if (repartitionNode == null || grouped.Named == null) { repartitionNode = repartNode; } builder.AddGraphNode(node, repartitionNode); node = repartitionNode; return(sourceName); } return(name); }
protected AbstractStream(String name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <String> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) { if (sourceNodes == null || sourceNodes.Count == 0) { throw new ArgumentException("parameter <sourceNodes> must not be null or empty"); } NameNode = name; this.builder = builder; KeySerdes = keySerde; ValueSerdes = valSerde; SetSourceNodes = sourceNodes; Node = streamsGraphNode; }
internal KStream(string name, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, List <string> setSourceNodes, StreamGraphNode node, InternalStreamBuilder builder) : base(name, keySerdes, valueSerdes, setSourceNodes, node, builder) { }
internal KTableGrouped(string name, ISerDes <KR> keySerde, ISerDes <VR> valSerde, List <string> sourceNodes, string queryableStoreName, IProcessorSupplier <K, V> processorSupplier, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) : base(name, keySerde, valSerde, sourceNodes, queryableStoreName, (IProcessorSupplier <KR, V>)null, streamsGraphNode, builder) { this.processorSupplier = processorSupplier; }
public TimeWindowedKStream(WindowOptions <W> windowOptions, GroupedStreamAggregateBuilder <K, V> aggBuilder, string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) : base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder) { CheckIfParamNull(windowOptions, "windowOptions"); this.windowOptions = windowOptions; this.aggBuilder = aggBuilder; }
public GroupedStreamAggregateBuilder(InternalStreamBuilder builder, List <string> sourceNodes, StreamGraphNode node) { this.builder = builder; this.sourceNodes = sourceNodes; this.node = node; }
public KGroupedStream(string name, Grouped <K, V> grouped, List <string> sourceNodes, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) : base(name, grouped.Key, grouped.Value, sourceNodes, streamsGraphNode, builder) { }
internal KTable(string name, ISerDes <K> keySerde, ISerDes <V> valSerde, List <string> sourceNodes, String queryableStoreName, IProcessorSupplier <K, S> processorSupplier, StreamGraphNode streamsGraphNode, InternalStreamBuilder builder) : base(name, keySerde, valSerde, sourceNodes, streamsGraphNode, builder) { this.processorSupplier = processorSupplier; this.queryableStoreName = queryableStoreName; }
public KStream <K, VR> Join <K, V, V0, VR>( KStream <K, V> joinLeft, KStream <K, V0> joinRight, IValueJoiner <V, V0, VR> joiner, JoinWindowOptions windows, StreamJoinProps <K, V, V0> joined) { var named = new Named(joined.Name); var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join"; var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join"; var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME); var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME); var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME); var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME); var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName); var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName); var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME); StreamGraphNode leftStreamsGraphNode = joinLeft.Node; StreamGraphNode rightStreamsGraphNode = joinRight.Node; var userProvidedBaseStoreName = joined.StoreName; var leftStoreSupplier = joined.LeftStoreSupplier; var rightStoreSupplier = joined.RightStoreSupplier; StoreBuilder <WindowStore <K, V> > leftWindowStore; StoreBuilder <WindowStore <K, V0> > rightWindowStore; AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier); if (leftStoreSupplier == null) { var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix; leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes); } else { AssertWindowSettings(leftStoreSupplier, windows); leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes); } if (rightStoreSupplier == null) { var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix; rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes); } else { AssertWindowSettings(rightStoreSupplier, windows); rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes); } var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name); var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName); var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams); builder.AddGraphNode(leftStreamsGraphNode, leftNode); var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name); var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName); var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams); builder.AddGraphNode(rightStreamsGraphNode, rightNode); var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter); var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName); var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter); var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName); var merge = new PassThrough <K, VR>(); var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName); var joinNode = new StreamStreamJoinNode <K, V, V0, VR>( joinMergeName, joiner, joinLParams, joinRParams, mergeParams, leftStreamParams, rightStreamParams, leftWindowStore, rightWindowStore, joined); builder.AddGraphNode(new List <StreamGraphNode> { leftStreamsGraphNode, rightStreamsGraphNode }, joinNode); ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes); allSourceNodes.AddRange(joinRight.SetSourceNodes); return(new KStream <K, VR>( joinMergeName, joined.KeySerdes, null, allSourceNodes.ToList(), joinNode, builder)); }
internal void AddGraphNode(StreamGraphNode root, StreamGraphNode node) { logger.LogDebug("Adding node {Node} in root node {Root}", node, root); root.AppendChild(node); nodes.Add(node); }