public void StreamStreamLeftJoinWithNoRecordInLeftJoin() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); builder .Stream <string, string>("topic2") .LeftJoin( stream, (s, v) => $"{s}-{v}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output-join"); Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic = driver.CreateInputTopic <string, string>("topic1"); var outputTopic = driver.CreateOuputTopic <string, string>("output-join"); inputTopic.PipeInput("test", "test"); var record = outputTopic.ReadKeyValue(); Assert.IsNull(record); }
public void StreamStreamLeftJoin3() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); builder .Stream <string, string>("topic2") .LeftJoin <string, string, StringSerDes>( stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output-join"); Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic = driver.CreateInputTopic <string, string>("topic1"); var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output-join"); inputTopic.PipeInput("test", "test"); inputTopic2.PipeInput("test", "coucou"); var record = outputTopic.ReadKeyValue(); Assert.IsNotNull(record); Assert.AreEqual("test", record.Message.Key); Assert.AreEqual("coucou-test", record.Message.Value); }
public void StreamStreamJoinSpecificSerdes() { var stringSerdes = new StringSerDes(); var config = new StreamConfig { ApplicationId = "test-stream-stream-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream("topic1", stringSerdes, stringSerdes); builder .Stream("topic2", stringSerdes, stringSerdes) .Join( stream, (s, v) => $"{s}-{v}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10)), StreamJoinProps.With( keySerde: stringSerdes, valueSerde: stringSerdes, otherValueSerde: stringSerdes)) .To("output-join"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string, StringSerDes, StringSerDes>("topic1"); var outputTopic = driver.CreateOuputTopic <string, string, StringSerDes, StringSerDes>("output-join"); inputTopic.PipeInput("test", "test"); var record = outputTopic.ReadKeyValue(); Assert.IsNull(record); } }
public void StreamStreamOuterJoinWithNoRecordInRigthJoin() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-outer-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); builder .Stream <string, string>("topic2") .OuterJoin( stream, (s, v) => $"{s}-{v}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output-join"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output-join"); inputTopic2.PipeInput("test", "coucou"); var record = outputTopic.ReadKeyValue(); Assert.IsNotNull(record); Assert.AreEqual("test", record.Message.Key); Assert.AreEqual("coucou-", record.Message.Value); } }
public void WorkflowCompleteMaxTaskIdleTest() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-group"; config.MaxTaskIdleMs = (long)TimeSpan.FromSeconds(100).TotalMilliseconds; var builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("topic1"); var stream2 = builder.Stream <string, string>("topic2"); stream1 .LeftJoin(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output"); Topology t = builder.Build(); using var driver = new TopologyTestDriver(t, config); var inputTopic1 = driver.CreateInputTopic <string, string>("topic1"); var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output"); inputTopic1.PipeInput("key", "i1"); inputTopic1.PipeInput("key", "i2"); inputTopic1.PipeInput("key", "i3"); var items = outputTopic.ReadKeyValueList().ToList(); Assert.AreEqual(0, items.Count); }
public void TopologyDescriptionRepartitionJoinStreamStreamTopology() { var builder = new StreamBuilder(); var streamJoin = builder.Stream <String, String>("topic-to-join"); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Join(streamJoin, (s, s1) => $"{s}-{s1}", JoinWindowOptions.Of(TimeSpan.FromSeconds(1))) .To("return"); var topology = builder.Build(); var description = topology.Describe(); Assert.AreEqual(2, description.SubTopologies.Count()); Assert.AreEqual(0, description.SubTopologies.ToArray()[0].Id); Assert.AreEqual(8, description.SubTopologies.ToArray()[0].Nodes.Count()); Assert.AreEqual(1, description.SubTopologies.ToArray()[1].Id); Assert.AreEqual(4, description.SubTopologies.ToArray()[1].Nodes.Count()); var sourceNode1 = description.SubTopologies.ToArray()[0].Nodes.ToArray()[0] as ISourceNodeDescription; var sourceNode1bis = description.SubTopologies.ToArray()[0].Nodes.ToArray()[1] as ISourceNodeDescription; var sourceNode2 = description.SubTopologies.ToArray()[1].Nodes.ToArray()[0] as ISourceNodeDescription; Assert.IsNotNull(sourceNode1); Assert.IsTrue(sourceNode1.Name.StartsWith(KStream.SOURCE_NAME)); Assert.AreEqual(null, sourceNode1.TimestampExtractorType); Assert.AreEqual(new List <string> { "topic-to-join" }, sourceNode1.Topics); Assert.IsNotNull(sourceNode1bis); Assert.IsTrue(sourceNode1bis.Name.StartsWith(KStream.SOURCE_NAME)); Assert.AreEqual(typeof(FailOnInvalidTimestamp), sourceNode1bis.TimestampExtractorType); Assert.AreEqual(new List <string> { "KSTREAM-MAP-0000000002-repartition" }, sourceNode1bis.Topics); Assert.IsTrue(sourceNode2.Name.StartsWith(KStream.SOURCE_NAME)); Assert.AreEqual(null, sourceNode2.TimestampExtractorType); Assert.AreEqual(new List <string> { "topic" }, sourceNode2.Topics); }
public void StreamWithNullLeftStream() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); Assert.Throws <ArgumentNullException>(() => builder .Stream <string, string>("topic2") .LeftJoin( null, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)))); }
public void StreamSameStoreName() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); var joinProps = StreamJoinProps.From <string, string, string>(StreamJoinProps.With( State.Stores.InMemoryWindowStore("test", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)), State.Stores.InMemoryWindowStore("test", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)))); Assert.Throws <StreamsException>(() => builder .Stream <string, string>("topic2") .LeftJoin(stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)), joinProps)); }
public void TopologyDescriptionJoinCompareWithJavaToString() { #region Expected StringBuilder stringBuilder = new StringBuilder(); stringBuilder.AppendLine("Topologies:"); stringBuilder.AppendLine(" Sub-topology: 0"); stringBuilder.AppendLine(" Source: KSTREAM-SOURCE-0000000000 (topics: [test])"); stringBuilder.AppendLine(" --> KSTREAM-WINDOWED-0000000002"); stringBuilder.AppendLine(" Source: KSTREAM-SOURCE-0000000001 (topics: [test2])"); stringBuilder.AppendLine(" --> KSTREAM-WINDOWED-0000000003"); stringBuilder.AppendLine(" Processor: KSTREAM-WINDOWED-0000000002 (stores: [KSTREAM-JOINTHIS-0000000004-store])"); stringBuilder.AppendLine(" --> KSTREAM-JOINTHIS-0000000004"); stringBuilder.AppendLine(" <-- KSTREAM-SOURCE-0000000000"); stringBuilder.AppendLine(" Processor: KSTREAM-WINDOWED-0000000003 (stores: [KSTREAM-JOINOTHER-0000000005-store])"); stringBuilder.AppendLine(" --> KSTREAM-JOINOTHER-0000000005"); stringBuilder.AppendLine(" <-- KSTREAM-SOURCE-0000000001"); stringBuilder.AppendLine(" Processor: KSTREAM-JOINTHIS-0000000004 (stores: [KSTREAM-JOINOTHER-0000000005-store])"); stringBuilder.AppendLine(" --> KSTREAM-MERGE-0000000006"); stringBuilder.AppendLine(" <-- KSTREAM-WINDOWED-0000000002"); stringBuilder.AppendLine(" Processor: KSTREAM-JOINOTHER-0000000005 (stores: [KSTREAM-JOINTHIS-0000000004-store])"); stringBuilder.AppendLine(" --> KSTREAM-MERGE-0000000006"); stringBuilder.AppendLine(" <-- KSTREAM-WINDOWED-0000000003"); stringBuilder.AppendLine(" Processor: KSTREAM-MERGE-0000000006 (stores: [])"); stringBuilder.AppendLine(" --> KSTREAM-SINK-0000000007"); stringBuilder.AppendLine(" <-- KSTREAM-JOINTHIS-0000000004, KSTREAM-JOINOTHER-0000000005"); stringBuilder.AppendLine(" Sink: KSTREAM-SINK-0000000007 (topic: output-join)"); stringBuilder.AppendLine(" <-- KSTREAM-MERGE-0000000006"); #endregion // COMPARE TOSTRING() WITH JAVA TOSTRING() DESCRIBE TOPO var builder = new StreamBuilder(); var stream1 = builder.Stream("test", new StringSerDes(), new StringSerDes()); var stream2 = builder.Stream("test2", new StringSerDes(), new StringSerDes()); stream1.Join <string, string, StringSerDes>(stream2, (v, v2) => $"{v}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1))) .To("output-join"); var topology = builder.Build(); var description = topology.Describe(); Assert.AreEqual(stringBuilder.ToString(), description.ToString()); }
public void RepartitionTestJoinTopology() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-repartition-processor" }; var supplier = new MockKafkaSupplier(10); StreamBuilder builder = new StreamBuilder(); IKStream <string, string> stream1 = builder .Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v)); IKStream <string, string> stream2 = builder.Stream <string, string>("topic2"); stream1.Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1)), StreamJoinProps.As <string, string, string>("join-store")) .To("output"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t.Builder, config, TopologyTestDriver.Mode.ASYNC_CLUSTER_IN_MEMORY, supplier)) { var inputTopic = driver.CreateInputTopic <string, string>("topic"); var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output"); inputTopic.PipeInput("test", "coucou"); inputTopic2.PipeInput("TEST", "sylvain"); inputTopic2.PipeInput("TEST2", "antoine"); inputTopic.PipeInput("test2", "test"); var records = IntegrationTestUtils .WaitUntilMinKeyValueRecordsReceived(outputTopic, 2) .ToUpdateDictionary(r => r.Message.Key, r => r.Message.Value); Assert.IsNotNull(records); Assert.AreEqual(2, records.Count); Assert.AreEqual("coucou-sylvain", records["TEST"]); Assert.AreEqual("test-antoine", records["TEST2"]); } }
public void StreamInvalidTimeSettings() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-left-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); var joinProps = StreamJoinProps.From <string, string, string>(StreamJoinProps.With( State.Stores.InMemoryWindowStore("test1", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)), State.Stores.InMemoryWindowStore("test2", TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10)))); // JoinWindowOptions.Of => use default retention => One Day // joinProps use supplier with retention 10 secondes => BAD THING !! Assert.Throws <StreamsException>(() => builder .Stream <string, string>("topic2") .LeftJoin(stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10)), joinProps)); }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("test"); var stream2 = builder.Stream <string, string>("test2"); stream1.Join <string, string>(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1))) .To("output-join"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public void StreamStreamOuterJoin4() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-outer-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); builder .Stream <string, string>("topic2") .OuterJoin( stream, new MyJoinerMapper(), JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output-join"); Topology t = builder.Build(); using (var driver = new TopologyTestDriver(t, config)) { var inputTopic = driver.CreateInputTopic <string, string>("topic1"); var inputTopic2 = driver.CreateInputTopic <string, string>("topic2"); var outputTopic = driver.CreateOuputTopic <string, string>("output-join"); inputTopic.PipeInput("test", "test"); inputTopic2.PipeInput("test", "coucou"); var records = outputTopic.ReadKeyValueList().ToList(); Assert.AreEqual(2, records.Count); Assert.AreEqual("test", records[0].Message.Key); Assert.AreEqual("-test", records[0].Message.Value); Assert.AreEqual("test", records[1].Message.Key); Assert.AreEqual("coucou-test", records[1].Message.Value); } }
public void StreamWithNullOuterStream2() { var config = new StreamConfig <StringSerDes, StringSerDes> { ApplicationId = "test-stream-stream-outer-join" }; StreamBuilder builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic1"); Assert.Throws <ArgumentNullException>(() => builder .Stream <string, string>("topic2") .OuterJoin(stream, (IValueJoiner <string, string, string>)null, JoinWindowOptions.Of(TimeSpan.FromSeconds(10)))); }
//[Test] // TODO : fix that public void WorkflowCompleteBufferedRecordsTest() { int maxBuffered = 10; var token = new System.Threading.CancellationTokenSource(); var serdes = new StringSerDes(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-group"; config.MaxTaskIdleMs = (long)TimeSpan.FromSeconds(100).TotalMilliseconds; config.BufferedRecordsPerPartition = maxBuffered; config.PollMs = 10; var builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("topic1"); var stream2 = builder.Stream <string, string>("topic2"); stream1 .Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("output"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(token.Token); for (int i = 0; i < maxBuffered + 1; ++i) { producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{i}", new SerializationContext()) }); } // CONSUME PAUSE AFTER maxBuffered + 1 messages System.Threading.Thread.Sleep(50); // Add one message more with consumer in stream thread in pause producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{maxBuffered+1}", new SerializationContext()) }); Assert.AreEqual(1, thread.ActiveTasks.Count()); var task = thread.ActiveTasks.ToArray()[0]; Assert.IsNotNull(task.Grouper); Assert.IsFalse(task.Grouper.AllPartitionsBuffered); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered()); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered(new TopicPartition("topic1", 0))); Assert.AreEqual(0, task.Grouper.NumBuffered(new TopicPartition("topic2", 0))); producer.Produce("topic2", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"test", new SerializationContext()) }); List <ConsumeResult <byte[], byte[]> > records = new List <ConsumeResult <byte[], byte[]> >(); do { records.AddRange(consumer.ConsumeRecords(TimeSpan.FromMilliseconds(100)).ToList()); } while (records.Count() <= 12); Assert.AreEqual(maxBuffered + 2, records.Count()); for (int i = 0; i < maxBuffered + 2; ++i) { var message = records.ToArray()[i]; Assert.AreEqual("key", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.IsTrue(serdes.Deserialize(message.Message.Value, new SerializationContext()).Contains($"coucou{i}-")); } token.Cancel(); thread.Dispose(); }