Exemple #1
0
 public KStreamKTableJoinProcessor(IKTableValueGetter <K2, V2> valueGetter, IKeyValueMapper <K1, V1, K2> keyValueMapper, IValueJoiner <V1, V2, R> valueJoiner, bool leftJoin)
 {
     this.valueGetter = valueGetter;
     mapper           = keyValueMapper;
     joiner           = valueJoiner;
     this.leftJoin    = leftJoin;
 }
Exemple #2
0
 public KStreamGlobalKTableJoin(IKTableValueGetterSupplier <K2, V2> supplier, IValueJoiner <V1, V2, VR> valueJoiner, IKeyValueMapper <K1, V1, K2> keyMapper, bool leftJoin)
 {
     this.supplier    = supplier;
     this.valueJoiner = valueJoiner;
     this.leftJoin    = leftJoin;
     this.keyMapper   = keyMapper;
 }
Exemple #3
0
        public KStream <K, VR> Join <K, V, V0, VR>(
            KStream <K, V> joinLeft,
            KStream <K, V0> joinRight,
            IValueJoiner <V, V0, VR> joiner,
            JoinWindowOptions windows,
            Joined <K, V, V0> joined)
        {
            var named           = new Named(joined.Name);
            var joinThisSuffix  = rightOuter ? "-outer-this-join" : "-this-join";
            var joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";

            var thisWindowStreamProcessorName  = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
            var otherWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);

            var joinThisGeneratedName  = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
            var joinOtherGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);

            var joinThisName  = named.SuffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
            var joinOtherName = named.SuffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);

            var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);

            StreamGraphNode thisStreamsGraphNode      = joinLeft.Node;
            StreamGraphNode otherStreamsGraphNode     = joinRight.Node;
            var             userProvidedBaseStoreName = joined.Name; // TODO : renamed, and create a StreamJoined DTO

            // TODO TO FINISH

            return(null);
        }
Exemple #4
0
 public KStreamKTableJoin(
     IKTableValueGetterSupplier <K, V2> valueGetter,
     IValueJoiner <V1, V2, R> valueJoiner,
     bool leftJoin)
 {
     this.valueGetter = valueGetter;
     this.valueJoiner = valueJoiner;
     this.leftJoin    = leftJoin;
 }
 public KStreamKStreamJoin(string name, string otherWindowName, long beforeMs, long afterMs, IValueJoiner <V, V1, VR> joiner, bool outer)
 {
     this.name            = name;
     this.otherWindowName = otherWindowName;
     this.beforeMs        = beforeMs;
     this.afterMs         = afterMs;
     this.joiner          = joiner;
     this.outer           = outer;
 }
Exemple #6
0
 public AbstractKTableKTableJoin(
     IKTableGetter <K, V1> table1,
     IKTableGetter <K, V2> table2,
     IValueJoiner <V1, V2, VR> valueJoiner)
 {
     this.table1      = table1;
     this.table2      = table2;
     this.valueJoiner = valueJoiner;
 }
 public KStreamKStreamJoinProcessor(
     string name,
     string windowStoreName,
     long beforeMs,
     long afterMs,
     IValueJoiner <V, V1, VR> joiner,
     bool outer)
     : base(name)
 {
     this.windowStoreName = windowStoreName;
     this.beforeMs        = beforeMs;
     this.afterMs         = afterMs;
     this.joiner          = joiner;
     this.outer           = outer;
 }
 public StreamStreamJoinNode(
     string name,
     IValueJoiner <V1, V2, VR> valueJoiner,
     ProcessorParameters <K, V1> joinLeftParams,
     ProcessorParameters <K, V2> joinRightParams,
     ProcessorParameters <K, VR> joinMergeParams,
     ProcessorParameters <K, V1> windowedLeftParams,
     ProcessorParameters <K, V2> windowedRightParams,
     IStoreBuilder <IWindowStore <K, V1> > windowedLeftStoreBuilder,
     IStoreBuilder <IWindowStore <K, V2> > windowedRightStoreBuilder)
     : base(name, valueJoiner, joinLeftParams, joinRightParams, joinMergeParams, null, null)
 {
     this.windowedLeftParams        = windowedLeftParams;
     this.windowedRightParams       = windowedRightParams;
     this.windowedLeftStoreBuilder  = windowedLeftStoreBuilder;
     this.windowedRightStoreBuilder = windowedRightStoreBuilder;
 }
 public BaseJoinProcessorNode(
     string name,
     IValueJoiner <V1, V2, VR> valueJoiner,
     ProcessorParameters <K, V1> joinLeftParams,
     ProcessorParameters <K, V2> joinRightParams,
     ProcessorParameters <K, VR> joinMergeParams,
     string leftJoinSideName,
     string rightJoinSideName)
     : base(name)
 {
     ValueJoiner       = valueJoiner;
     JoinLeftParams    = joinLeftParams ?? throw new ArgumentNullException(nameof(joinLeftParams));
     JoinRightParams   = joinRightParams ?? throw new ArgumentNullException(nameof(joinRightParams));
     JoinMergeParams   = joinMergeParams;
     LeftJoinSideName  = leftJoinSideName;
     RightJoinSideName = rightJoinSideName;
 }
 public KTableKTableOuterJoin(IKTableGetter <K, V1> table1, IKTableGetter <K, V2> table2, IValueJoiner <V1, V2, VR> valueJoiner)
     : base(table1, table2, valueJoiner)
 {
 }
 public KTableKTableLeftJoinProcessor(IKTableValueGetter <K, V2> valueGetter, IValueJoiner <V1, V2, VR> valueJoiner, bool sendOldValues)
     : base(valueGetter, valueJoiner, sendOldValues)
 {
 }
Exemple #12
0
        public IKTable <K, VR> Join <K, V, V0, VR>(
            IKTable <K, V> tableLeft,
            IKTable <K, V0> tableRight,
            IValueJoiner <V, V0, VR> joiner,
            string named,
            Materialized <K, VR, IKeyValueStore <Bytes, byte[]> > materializedInternal)
        {
            var           renamed        = new Named(named);
            var           joinMergeName  = renamed.OrElseGenerateWithPrefix(builder, KTable.MERGE_NAME);
            ISet <string> allSourceNodes = new HashSet <string>((tableLeft as AbstractStream <K, V>).SetSourceNodes);

            allSourceNodes.AddRange((tableRight as AbstractStream <K, V0>).SetSourceNodes);
            materializedInternal.UseProvider(builder, $"{joinMergeName}-");

            if (leftOuter)
            {
                (tableLeft as IKTableGetter <K, V>)?.EnableSendingOldValues();
            }

            if (rightOuter)
            {
                (tableRight as IKTableGetter <K, V0>)?.EnableSendingOldValues();
            }

            AbstractKTableKTableJoin <K, VR, V, V0> joinLeft  = null;
            AbstractKTableKTableJoin <K, VR, V0, V> joinRight = null;

            if (!leftOuter) // INNER JOIN
            {
                joinLeft  = new KTableKTableInnerJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
                joinRight = new KTableKTableInnerJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
            }
            else if (!rightOuter) // LEFT JOIN
            {
                joinLeft  = new KTableKTableLeftJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
                joinRight = new KTableKTableRightJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
            }
            else // OUTER JOIN
            {
                joinLeft  = new KTableKTableOuterJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
                joinRight = new KTableKTableOuterJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
            }

            var joinLeftName  = renamed.SuffixWithOrElseGet("-join-this", builder, KTable.JOINTHIS_NAME);
            var joinRigthName = renamed.SuffixWithOrElseGet("-join-other", builder, KTable.JOINOTHER_NAME);

            var joinLeftProcessorParameters  = new TableProcessorParameters <K, V>(joinLeft, joinLeftName);
            var joinRightProcessorParameters = new TableProcessorParameters <K, V0>(joinRight, joinRigthName);

            if (materializedInternal.KeySerdes == null && tableLeft is AbstractStream <K, V> )
            {
                materializedInternal.WithKeySerdes((tableLeft as AbstractStream <K, V>).KeySerdes);
            }

            ISerDes <K>  keySerdes          = materializedInternal.KeySerdes;
            ISerDes <VR> ValueSerdes        = materializedInternal.ValueSerdes;
            string       queryableStoreName = materializedInternal.QueryableStoreName;
            StoreBuilder <TimestampedKeyValueStore <K, VR> > storeBuilder = queryableStoreName != null ? new TimestampedKeyValueStoreMaterializer <K, VR>(materializedInternal).Materialize() : null;

            var tableNode = new KTableKTableJoinNode <K, V, V0, VR>(
                joinMergeName,
                joinLeftProcessorParameters,
                joinRightProcessorParameters,
                (tableLeft as AbstractStream <K, V>).NameNode,
                (tableRight as AbstractStream <K, V0>).NameNode,
                (tableLeft as IKTableGetter <K, V>).ValueGetterSupplier.StoreNames,
                (tableRight as IKTableGetter <K, V0>).ValueGetterSupplier.StoreNames,
                queryableStoreName,
                storeBuilder);

            builder.AddGraphNode((tableLeft as AbstractStream <K, V>).Node, tableNode);

            return(new KTable <K, VR, VR>(
                       tableNode.streamGraphNode,
                       keySerdes,
                       ValueSerdes,
                       allSourceNodes.ToList(),
                       queryableStoreName,
                       tableNode.JoinMergeProcessorSupplier,
                       tableNode,
                       builder));
        }
 public KTableKTableLeftJoinValueGetterSupplier(IKTableValueGetterSupplier <K, V1> getter1, IKTableValueGetterSupplier <K, V2> getter2, IValueJoiner <V1, V2, VR> valueJoiner)
     : base(getter1, getter2)
 {
     joiner = valueJoiner;
 }
 public AbstractKTableKTableJoinProcessor(IKTableValueGetter <K, V2> valueGetter, IValueJoiner <V1, V2, VR> joiner, bool sendOldValues)
 {
     this.valueGetter   = valueGetter;
     this.joiner        = joiner;
     this.sendOldValues = sendOldValues;
 }
Exemple #15
0
        public KStream <K, VR> Join <K, V, V0, VR>(
            KStream <K, V> joinLeft,
            KStream <K, V0> joinRight,
            IValueJoiner <V, V0, VR> joiner,
            JoinWindowOptions windows,
            StreamJoinProps <K, V, V0> joined)
        {
            var named           = new Named(joined.Name);
            var joinLeftSuffix  = rightOuter ? "-outer-this-join" : "-this-join";
            var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";

            var leftWindowStreamProcessorName  = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
            var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);

            var joinLeftGeneratedName  = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
            var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);

            var joinLeftName  = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
            var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);

            var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);

            StreamGraphNode leftStreamsGraphNode      = joinLeft.Node;
            StreamGraphNode rightStreamsGraphNode     = joinRight.Node;
            var             userProvidedBaseStoreName = joined.StoreName;

            var leftStoreSupplier  = joined.LeftStoreSupplier;
            var rightStoreSupplier = joined.RightStoreSupplier;

            StoreBuilder <WindowStore <K, V> >  leftWindowStore;
            StoreBuilder <WindowStore <K, V0> > rightWindowStore;

            AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);

            if (leftStoreSupplier == null)
            {
                var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
                leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
            }
            else
            {
                AssertWindowSettings(leftStoreSupplier, windows);
                leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
            }

            if (rightStoreSupplier == null)
            {
                var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
                rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
            }
            else
            {
                AssertWindowSettings(rightStoreSupplier, windows);
                rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
            }


            var leftStream       = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
            var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
            var leftNode         = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);

            builder.AddGraphNode(leftStreamsGraphNode, leftNode);

            var rightStream       = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
            var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
            var rightNode         = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);

            builder.AddGraphNode(rightStreamsGraphNode, rightNode);

            var joinL       = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
            var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
            var joinR       = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
            var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
            var merge       = new PassThrough <K, VR>();
            var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);

            var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
                joinMergeName,
                joiner,
                joinLParams,
                joinRParams,
                mergeParams,
                leftStreamParams,
                rightStreamParams,
                leftWindowStore,
                rightWindowStore,
                joined);

            builder.AddGraphNode(new List <StreamGraphNode> {
                leftStreamsGraphNode, rightStreamsGraphNode
            }, joinNode);

            ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);

            allSourceNodes.AddRange(joinRight.SetSourceNodes);

            return(new KStream <K, VR>(
                       joinMergeName,
                       joined.KeySerdes,
                       null,
                       allSourceNodes.ToList(),
                       joinNode,
                       builder));
        }
Exemple #16
0
 public KTableKTableInnerJoinValueGetter(IKTableValueGetter <K, V1> iKTableValueGetter1, IKTableValueGetter <K, V2> iKTableValueGetter2, IValueJoiner <V1, V2, VR> joiner)
 {
     this.iKTableValueGetter1 = iKTableValueGetter1;
     this.iKTableValueGetter2 = iKTableValueGetter2;
     this.joiner = joiner;
 }
Exemple #17
0
 public static IValueJoiner <V2, V1, VR> Reverse <V1, V2, VR>(this IValueJoiner <V1, V2, VR> joiner)
 {
     return(new WrappedValueJoiner <V2, V1, VR>((v2, v1) => joiner.Apply(v1, v2)));
 }
Exemple #18
0
 public KTableKTableInnerJoinValueGetterSupplier(IKTableValueGetterSupplier <K, V1> valueGetterSupplier1, IKTableValueGetterSupplier <K, V2> valueGetterSupplier2, IValueJoiner <V1, V2, VR> valueJoiner)
     : base(valueGetterSupplier1, valueGetterSupplier2)
 {
     joiner = valueJoiner;
 }