private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-perf-app"; config.BootstrapServers = "localhost:29092"; config.PollMs = 100; config.MaxPollRecords = 500; StreamBuilder builder = new StreamBuilder(); builder .Stream <string, string>("test") .To("test-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public void GetElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1"), Value = serdes.Serialize("coucou") }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); Assert.AreEqual(1, store.ApproximateNumEntries()); var item = store.Get("key1"); Assert.IsNotNull(item); Assert.AreEqual("coucou", item); } source.Cancel(); stream.Close(); }
static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .FilterNot((k, v) => v.Contains("test")) .To("test-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")) .ToStream() .Map((k, v) => KeyValuePair.Create(k.Key, v.ToString())) .To("output"); Topology t = builder.Build(); //bool taskStart = false; KafkaStream stream = new KafkaStream(t, config); // Subscribe state changed //stream.StateChanged += (old, @new) => //{ // if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store. // { // Task.Factory.StartNew(() => // { // while (!source.Token.IsCancellationRequested) // { // var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>())); // var items = store.All().ToList(); // Thread.Sleep(500); // } // }, source.Token); // taskStart = true; // } //}; Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .FilterNot((k, v) => v.Contains("test")) .To("test-output"); builder.Table("test-ktable", InMemory <string, string> .As("test-store")); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); bool taskGetStateStoreRunning = false; stream.StateChanged += (old, @new) => { if (@new == KafkaStream.State.RUNNING && !taskGetStateStoreRunning) { Task.Factory.StartNew(() => { taskGetStateStoreRunning = true; while (!source.Token.IsCancellationRequested) { var store = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.KeyValueStore <string, string>())); var items = store.All().ToList(); Thread.Sleep(500); } }, source.Token); } }; Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public void GetWindowStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); } source.Cancel(); stream.Close(); }
public void GetKVStateStoreInvalidStateStoreException() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool state = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .Count(InMemory <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if ([email protected](KafkaStream.State.RUNNING)) { if (!state) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>()))); state = true; } } }; stream.Start(source.Token); while (!state) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(state); source.Cancel(); stream.Close(); }
public void GetStateStoreDoesntExists() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var supplier = new SyncKafkaSupplier(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>()))); } source.Cancel(); stream.Close(); }
public void GetStateStoreBeforeRunningState() { var source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()))); source.Cancel(); stream.Close(); }
public void StartKafkaStream() { var source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.Start(source.Token); Thread.Sleep(1500); source.Cancel(); stream.Close(); }
public void StartKafkaStreamWaitRunningState() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } source.Cancel(); stream.Close(); Assert.IsTrue(isRunningState); }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("test"); var stream2 = builder.Stream <string, string>("test2"); stream1.Join <string, string>(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1))) .To("output-join"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; // NEED FOR SchemaAvroSerDes config.SchemaRegistryUrl = "http://192.168.56.1:8081"; config.AutoRegisterSchemas = false; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, Person, StringSerDes, SchemaAvroSerDes <Person> >("person") .Filter((k, v) => v.age >= 18) .MapValues((v) => $"{v.firstName}-{v.lastName}-{v.age}") .To <StringSerDes, StringSerDes>("output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public void Close() { stream?.Close(); }
public void GetWindowElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); dt = DateTime.Now; producer.Produce("test", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()), Timestamp = new Confluent.Kafka.Timestamp(dt) }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); var @enum = store.All(); Assert.AreEqual(1, store.All().ToList().Count); var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1)); Assert.IsNotNull(item); Assert.IsTrue(item.MoveNext()); Assert.IsTrue(item.Current.HasValue); Assert.AreEqual(1, item.Current.Value.Value); item.Dispose(); } source.Cancel(); stream.Close(); }