public void StreamStreamJoinSpecificSerdes()
{
var stringSerdes = new StringSerDes();
var config = new StreamConfig
{
ApplicationId = "test-stream-stream-join"
};
StreamBuilder builder = new StreamBuilder();
var stream = builder.Stream("topic1", stringSerdes, stringSerdes);
builder
.Stream("topic2", stringSerdes, stringSerdes)
.Join(
stream,
(s, v) => $"{s}-{v}",
JoinWindowOptions.Of(TimeSpan.FromSeconds(10)),
StreamJoinProps.With(
keySerde: stringSerdes,
valueSerde: stringSerdes,
otherValueSerde: stringSerdes))
.To("output-join");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string, StringSerDes, StringSerDes>("topic1");
var outputTopic = driver.CreateOuputTopic <string, string, StringSerDes, StringSerDes>("output-join");
inputTopic.PipeInput("test", "test");
var record = outputTopic.ReadKeyValue();
Assert.IsNull(record);
}
}
public void TestJoinPropsNames()
{
var props = StreamJoinProps.As("name1", "store1");
Assert.IsNotNull(props);
Assert.AreEqual("store1", props.StoreName);
Assert.AreEqual("name1", props.Name);
}
public void TestJoinPropsSupplier()
{
var props = StreamJoinProps.With(new TestSupplier(), new TestSupplier());
Assert.IsNotNull(props);
Assert.IsAssignableFrom <TestSupplier>(props.LeftStoreSupplier);
Assert.IsAssignableFrom <TestSupplier>(props.RightStoreSupplier);
Assert.IsNull(props.Name);
Assert.IsNull(props.StoreName);
}
public void TestJoinPropsNames2()
{
var props = StreamJoinProps.As <string, string, string>("name1", "store1");
Assert.IsNotNull(props);
Assert.AreEqual("store1", props.StoreName);
Assert.AreEqual("name1", props.Name);
Assert.IsNull(props.KeySerdes);
Assert.IsNull(props.LeftValueSerdes);
Assert.IsNull(props.RightValueSerdes);
}
public void TestJoinPropsSupplier2()
{
var props = StreamJoinProps.With <string, string, string>(new TestSupplier(), new TestSupplier());
Assert.IsNotNull(props);
Assert.IsAssignableFrom <TestSupplier>(props.LeftStoreSupplier);
Assert.IsAssignableFrom <TestSupplier>(props.RightStoreSupplier);
Assert.IsNull(props.Name);
Assert.IsNull(props.StoreName);
Assert.IsNull(props.KeySerdes);
Assert.IsNull(props.LeftValueSerdes);
Assert.IsNull(props.RightValueSerdes);
}
public void TestJoinPropsWithSerdes()
{
var props = StreamJoinProps.With <string, int, long>(new StringSerDes(), new Int32SerDes(), new Int64SerDes());
Assert.IsNotNull(props);
Assert.IsNull(props.LeftStoreSupplier);
Assert.IsNull(props.RightStoreSupplier);
Assert.IsNull(props.Name);
Assert.IsNull(props.StoreName);
Assert.IsAssignableFrom <StringSerDes>(props.KeySerdes);
Assert.IsAssignableFrom <Int32SerDes>(props.LeftValueSerdes);
Assert.IsAssignableFrom <Int64SerDes>(props.RightValueSerdes);
}
public void StreamSameStoreName()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-stream-left-join"
};
StreamBuilder builder = new StreamBuilder();
var stream = builder.Stream <string, string>("topic1");
var joinProps = StreamJoinProps.From <string, string, string>(StreamJoinProps.With(
State.Stores.InMemoryWindowStore("test", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)),
State.Stores.InMemoryWindowStore("test", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10))));
Assert.Throws <StreamsException>(() => builder
.Stream <string, string>("topic2")
.LeftJoin(stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)), joinProps));
}
public StreamStreamJoinNode(
string name,
IValueJoiner <V1, V2, VR> valueJoiner,
ProcessorParameters <K, V1> joinLeftParams,
ProcessorParameters <K, V2> joinRightParams,
ProcessorParameters <K, VR> joinMergeParams,
ProcessorParameters <K, V1> windowedLeftParams,
ProcessorParameters <K, V2> windowedRightParams,
StoreBuilder <WindowStore <K, V1> > windowedLeftStoreBuilder,
StoreBuilder <WindowStore <K, V2> > windowedRightStoreBuilder,
StreamJoinProps <K, V1, V2> joinedProps)
: base(name, valueJoiner, joinLeftParams, joinRightParams, joinMergeParams, null, null)
{
this.windowedLeftParams = windowedLeftParams;
this.windowedRightParams = windowedRightParams;
this.windowedLeftStoreBuilder = windowedLeftStoreBuilder;
this.windowedRightStoreBuilder = windowedRightStoreBuilder;
this.joinedProps = joinedProps;
}
public void RepartitionTestJoinTopology()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-repartition-processor"
};
var supplier = new MockKafkaSupplier(10);
StreamBuilder builder = new StreamBuilder();
IKStream <string, string> stream1 = builder
.Stream <string, string>("topic")
.Map((k, v) => KeyValuePair.Create(k.ToUpper(), v));
IKStream <string, string> stream2 = builder.Stream <string, string>("topic2");
stream1.Join(stream2,
(v1, v2) => $"{v1}-{v2}",
JoinWindowOptions.Of(TimeSpan.FromMinutes(1)),
StreamJoinProps.As <string, string, string>("join-store"))
.To("output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
var inputTopic2 = driver.CreateInputTopic <string, string>("topic2");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTopic.PipeInput("test", "coucou");
inputTopic2.PipeInput("TEST", "sylvain");
inputTopic2.PipeInput("TEST2", "antoine");
inputTopic.PipeInput("test2", "test");
var records = IntegrationTestUtils
.WaitUntilMinKeyValueRecordsReceived(outputTopic, 2)
.ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value);
Assert.IsNotNull(records);
Assert.AreEqual(2, records.Count);
Assert.AreEqual("coucou-sylvain", records["TEST"]);
Assert.AreEqual("test-antoine", records["TEST2"]);
}
}
public void StreamInvalidTimeSettings()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-stream-left-join"
};
StreamBuilder builder = new StreamBuilder();
var stream = builder.Stream <string, string>("topic1");
var joinProps = StreamJoinProps.From <string, string, string>(StreamJoinProps.With(
State.Stores.InMemoryWindowStore("test1", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)),
State.Stores.InMemoryWindowStore("test2", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10))));
// JoinWindowOptions.Of => use default retention => One Day
// joinProps use supplier with retention 10 secondes => BAD THING !!
Assert.Throws <StreamsException>(() => builder
.Stream <string, string>("topic2")
.LeftJoin(stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)), joinProps));
}
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
StreamJoinProps <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode leftStreamsGraphNode = joinLeft.Node;
StreamGraphNode rightStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.StoreName;
var leftStoreSupplier = joined.LeftStoreSupplier;
var rightStoreSupplier = joined.RightStoreSupplier;
StoreBuilder <WindowStore <K, V> > leftWindowStore;
StoreBuilder <WindowStore <K, V0> > rightWindowStore;
AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);
if (leftStoreSupplier == null)
{
var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
}
else
{
AssertWindowSettings(leftStoreSupplier, windows);
leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
}
if (rightStoreSupplier == null)
{
var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
}
else
{
AssertWindowSettings(rightStoreSupplier, windows);
rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
}
var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);
builder.AddGraphNode(leftStreamsGraphNode, leftNode);
var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);
builder.AddGraphNode(rightStreamsGraphNode, rightNode);
var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
var merge = new PassThrough <K, VR>();
var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);
var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
joinMergeName,
joiner,
joinLParams,
joinRParams,
mergeParams,
leftStreamParams,
rightStreamParams,
leftWindowStore,
rightWindowStore,
joined);
builder.AddGraphNode(new List <StreamGraphNode> {
leftStreamsGraphNode, rightStreamsGraphNode
}, joinNode);
ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);
allSourceNodes.AddRange(joinRight.SetSourceNodes);
return(new KStream <K, VR>(
joinMergeName,
joined.KeySerdes,
null,
allSourceNodes.ToList(),
joinNode,
builder));
}
