private IKStream <K, V>[] DoBranch(string named = null, params Func <K, V, bool>[] predicates) { var namedInternal = new Named(named); if (predicates.Length == 0) { throw new ArgumentException("branch() requires at least one predicate"); } String branchName = namedInternal.OrElseGenerateWithPrefix(this.builder, BRANCH_NAME); String[] childNames = new String[predicates.Length]; for (int i = 0; i < predicates.Length; i++) { childNames[i] = namedInternal.SuffixWithOrElseGet($"predicate-{i}", this.builder, BRANCHCHILD_NAME); } ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamBranch <K, V>(predicates, childNames), branchName); ProcessorGraphNode <K, V> branchNode = new ProcessorGraphNode <K, V>(branchName, processorParameters); this.builder.AddGraphNode(node, branchNode); IKStream <K, V>[] branchChildren = new IKStream <K, V> [predicates.Length]; for (int i = 0; i < predicates.Length; i++) { ProcessorParameters <K, V> innerProcessorParameters = new ProcessorParameters <K, V>(new PassThrough <K, V>(), childNames[i]); ProcessorGraphNode <K, V> branchChildNode = new ProcessorGraphNode <K, V>(childNames[i], innerProcessorParameters); builder.AddGraphNode(branchNode, branchChildNode); branchChildren[i] = new KStream <K, V>(childNames[i], this.keySerdes, this.valueSerdes, setSourceNodes, branchChildNode, builder); } return(branchChildren); }
public KStream <K, VR> Join <K, V, V0, VR>( KStream <K, V> joinLeft, KStream <K, V0> joinRight, IValueJoiner <V, V0, VR> joiner, JoinWindowOptions windows, Joined <K, V, V0> joined) { var named = new Named(joined.Name); var joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join"; var joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join"; var thisWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME); var otherWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME); var joinThisGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME); var joinOtherGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME); var joinThisName = named.SuffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName); var joinOtherName = named.SuffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName); var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME); StreamGraphNode thisStreamsGraphNode = joinLeft.Node; StreamGraphNode otherStreamsGraphNode = joinRight.Node; var userProvidedBaseStoreName = joined.Name; // TODO : renamed, and create a StreamJoined DTO // TODO TO FINISH return(null); }
internal KStream <K, V> Stream <K, V>(string topic, ConsumedInternal <K, V> consumed) { var name = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KStream.SOURCE_NAME); var node = new StreamSourceNode <K, V>(topic, name, consumed); this.AddGraphNode(root, node); KStream <K, V> stream = new KStream <K, V>(name, consumed.KeySerdes, consumed.ValueSerdes, new List <string> { name }, node, this); return(stream); }
private string Repartition(StoreBuilder storeBuilder) { if (repartitionRequired) { (string sourceName, RepartitionNode <K, V> repartNode) = KStream <K, V> .CreateRepartitionSource(grouped.Named ?? storeBuilder.Name, grouped.Key, grouped.Value, builder); if (repartitionNode == null || grouped.Named == null) { repartitionNode = repartNode; } builder.AddGraphNode(node, repartitionNode); node = repartitionNode; return(sourceName); } return(name); }
public KStream <K, VR> Join <K, V, V0, VR>( KStream <K, V> joinLeft, KStream <K, V0> joinRight, IValueJoiner <V, V0, VR> joiner, JoinWindowOptions windows, StreamJoinProps <K, V, V0> joined) { var named = new Named(joined.Name); var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join"; var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join"; var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME); var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME); var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME); var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME); var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName); var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName); var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME); StreamGraphNode leftStreamsGraphNode = joinLeft.Node; StreamGraphNode rightStreamsGraphNode = joinRight.Node; var userProvidedBaseStoreName = joined.StoreName; var leftStoreSupplier = joined.LeftStoreSupplier; var rightStoreSupplier = joined.RightStoreSupplier; StoreBuilder <WindowStore <K, V> > leftWindowStore; StoreBuilder <WindowStore <K, V0> > rightWindowStore; AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier); if (leftStoreSupplier == null) { var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix; leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes); } else { AssertWindowSettings(leftStoreSupplier, windows); leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes); } if (rightStoreSupplier == null) { var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix; rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes); } else { AssertWindowSettings(rightStoreSupplier, windows); rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes); } var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name); var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName); var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams); builder.AddGraphNode(leftStreamsGraphNode, leftNode); var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name); var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName); var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams); builder.AddGraphNode(rightStreamsGraphNode, rightNode); var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter); var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName); var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter); var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName); var merge = new PassThrough <K, VR>(); var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName); var joinNode = new StreamStreamJoinNode <K, V, V0, VR>( joinMergeName, joiner, joinLParams, joinRParams, mergeParams, leftStreamParams, rightStreamParams, leftWindowStore, rightWindowStore, joined); builder.AddGraphNode(new List <StreamGraphNode> { leftStreamsGraphNode, rightStreamsGraphNode }, joinNode); ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes); allSourceNodes.AddRange(joinRight.SetSourceNodes); return(new KStream <K, VR>( joinMergeName, joined.KeySerdes, null, allSourceNodes.ToList(), joinNode, builder)); }