public static async Task Main(string[] args) { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app2"; config.BootstrapServers = "localhost:9092"; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.StateDir = Path.Combine("."); config.CommitIntervalMs = 5000; config.Logger = LoggerFactory.Create(builder => { builder.SetMinimumLevel(LogLevel.Information); builder.AddLog4Net(); }); StreamBuilder builder = new StreamBuilder(); var table = builder.GlobalTable <string, string>("topic"); builder.Stream <string, string>("input") .GroupBy((k, v) => v) .Count() .ToStream() .To("output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => stream.Dispose(); await stream.StartAsync(); }
public async Task StartKafkaStreamWithToken() { CancellationTokenSource source = new CancellationTokenSource(); KafkaStream.State lastState = KafkaStream.State.CREATED; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (o, n) => lastState = n; await stream.StartAsync(source.Token); Thread.Sleep(1500); source.Cancel(); Thread.Sleep(1500); Assert.AreEqual(KafkaStream.State.NOT_RUNNING, lastState); }
public void GetElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1"), Value = serdes.Serialize("coucou") }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); Assert.AreEqual(1, store.ApproximateNumEntries()); var item = store.Get("key1"); Assert.IsNotNull(item); Assert.AreEqual("coucou", item); } source.Cancel(); stream.Close(); }
private static async Task MainToken(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-token-app"; config.BootstrapServers = "localhost:29092"; config.PollMs = 100; config.MaxPollRecords = 500; StreamBuilder builder = new StreamBuilder(); builder .Stream <string, string>("test") .To("test-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); }; await stream.StartAsync(source.Token); }
static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .FilterNot((k, v) => v.Contains("test")) .To("test-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
static async Task Main(string[] args) { string inputTopic = "input", outputTopic = "output"; int numberPartitions = 4; await CreateTopics(inputTopic, outputTopic, numberPartitions); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "sample-streamiz-demo"; config.BootstrapServers = "localhost:9092"; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.StateDir = Path.Combine("."); config.CommitIntervalMs = 5000; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.MetricsRecording = MetricsRecordingLevel.DEBUG; config.UsePrometheusReporter(9090, true); StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>(inputTopic) .FlatMapValues((v) => v.Split(" ").AsEnumerable()) .GroupBy((k, v) => v) .Count() .ToStream() .MapValues((v) => v.ToString()) .To(outputTopic); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => stream.Dispose(); await stream.StartAsync(); }
static async Task Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; // NEED FOR SchemaAvroSerDes config.SchemaRegistryUrl = "http://192.168.56.1:8081"; config.AutoRegisterSchemas = false; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, Person, StringSerDes, SchemaAvroSerDes <Person> >("person") .Filter((k, v) => v.age >= 18) .MapValues((v) => $"{v.firstName}-{v.lastName}-{v.age}") .To <StringSerDes, StringSerDes>("output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { stream.Dispose(); }; await stream.StartAsync(); }
public async Task BuildGlobalStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 1; var builder = new StreamBuilder(); builder.GlobalTable <string, string>("test", InMemory <string, string> .As("store")); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(new ProducerConfig()); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var stringSerdes = new StringSerDes(); producer.Produce("test", new Message <byte[], byte[]> { Key = stringSerdes.Serialize("key", new SerializationContext()), Value = stringSerdes.Serialize("value", new SerializationContext()) }); Thread.Sleep(250); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); Assert.AreEqual(1, store.ApproximateNumEntries()); } stream.Dispose(); }
public ConsumerPerfThread(int threadId, string name, KafkaStream <byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead) { this.threadId = threadId; this.name = name; this.stream = stream; this.config = config; this.totalMessagesRead = totalMessagesRead; this.totalBytesRead = totalBytesRead; }
static async Task Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig(); config.ApplicationId = "test-app"; config.BootstrapServers = "localhost:9092"; // NEED FOR SchemaAvroSerDes config.SchemaRegistryUrl = "http://localhost:8081"; config.AutoRegisterSchemas = true; StreamBuilder builder = new StreamBuilder(); var table = builder.Table("product", new Int32SerDes(), new SchemaAvroSerDes <Product>(), InMemory <int, Product> .As("product-store")); var orders = builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders"); orders.Join(table, (order, product) => new OrderProduct { order_id = order.order_id, price = order.price, product_id = product.product_id, product_name = product.name, product_price = product.price }) .To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output"); orders .GroupByKey() .Aggregate <OrderAgg, SchemaAvroSerDes <OrderAgg> >( () => new OrderAgg(), (key, order, agg) => { agg.order_id = order.order_id; agg.price = order.price; agg.product_id = order.product_id; agg.totalPrice += order.price; return(agg); }) .ToStream() .Print(Printed <int, OrderAgg> .ToOut()); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { stream.Dispose(); }; await stream.StartAsync(); }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")) .ToStream() .Map((k, v) => KeyValuePair.Create(k.Key, v.ToString())) .To("output"); Topology t = builder.Build(); //bool taskStart = false; KafkaStream stream = new KafkaStream(t, config); // Subscribe state changed //stream.StateChanged += (old, @new) => //{ // if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store. // { // Task.Factory.StartNew(() => // { // while (!source.Token.IsCancellationRequested) // { // var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>())); // var items = store.All().ToList(); // Thread.Sleep(500); // } // }, source.Token); // taskStart = true; // } //}; Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public void Start(CancellationToken token = default) { foreach (var query in querys) { query.Analyze(builder); } var topology = builder.Build(); stream = new KafkaStream(topology, config); stream.Start(token); }
static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .FilterNot((k, v) => v.Contains("test")) .To("test-output"); builder.Table("test-ktable", InMemory <string, string> .As("test-store")); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); bool taskGetStateStoreRunning = false; stream.StateChanged += (old, @new) => { if (@new == KafkaStream.State.RUNNING && !taskGetStateStoreRunning) { Task.Factory.StartNew(() => { taskGetStateStoreRunning = true; while (!source.Token.IsCancellationRequested) { var store = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.KeyValueStore <string, string>())); var items = store.All().ToList(); Thread.Sleep(500); } }, source.Token); } }; Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public async Task GetWindowStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); } stream.Dispose(); }
public void GetWStateStoreInvalidStateStoreException() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool state = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if ([email protected](KafkaStream.State.RUNNING)) { if (!state) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()))); state = true; } } }; stream.Start(source.Token); while (!state) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(state); source.Cancel(); stream.Close(); }
public async Task GetKVStateStoreInvalidStateStoreException() { var timeout = TimeSpan.FromSeconds(10); bool state = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .Count(InMemory <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if ([email protected](KafkaStream.State.RUNNING)) { if (!state) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>()))); state = true; } } }; await stream.StartAsync(); while (!state) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(state); stream.Dispose(); }
static async Task Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; // NEED FOR SchemaAvroSerDes config.SchemaRegistryUrl = "http://192.168.56.1:8081"; config.AutoRegisterSchemas = true; StreamBuilder builder = new StreamBuilder(); var table = builder.Table("product", new Int32SerDes(), new SchemaAvroSerDes <Product>(), InMemory <int, Product> .As("product-store")); builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders") .Join(table, (order, product) => new OrderProduct { order_id = order.order_id, price = order.price, product_id = product.product_id, product_name = product.name, product_price = product.price }) .To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { stream.Dispose(); }; await stream.StartAsync(); }
public void GetStateStoreDoesntExists() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var supplier = new SyncKafkaSupplier(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>()))); } source.Cancel(); stream.Close(); }
public void GetStateStoreBeforeRunningState() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()))); stream.Dispose(); }
public async Task StartKafkaStream() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); await stream.StartAsync(); Thread.Sleep(1500); stream.Dispose(); }
public async Task StartKafkaStreamWaitRunningState() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } var sensors = stream.Metrics(); Assert.IsTrue(sensors.Any()); stream.Dispose(); Assert.IsTrue(isRunningState); }
static async System.Threading.Tasks.Task Main(string[] args) { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "localhost:9092"; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("test") .To("test2"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => stream.Dispose(); await stream.StartAsync(); }
public void StartKafkaStream() { var source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.Start(source.Token); Thread.Sleep(1500); source.Cancel(); stream.Close(); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "localhost:9092"; config.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Earliest; config.StateDir = Path.Combine(".", Guid.NewGuid().ToString()); StreamBuilder builder = new StreamBuilder(); builder .Table("table", RocksDb <string, string> .As("table-store").WithLoggingEnabled()) .ToStream() .Print(Printed <string, string> .ToOut()); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); await stream.StartAsync(stoppingToken); }
public void StartKafkaStreamWaitRunningState() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var t = builder.Build(); var stream = new KafkaStream(t, config, new SyncKafkaSupplier()); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } source.Cancel(); stream.Close(); Assert.IsTrue(isRunningState); }
private static void Main(string[] args) { CancellationTokenSource source = new CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "192.168.56.1:9092"; config.SaslMechanism = SaslMechanism.Plain; config.SaslUsername = "******"; config.SaslPassword = "******"; config.SecurityProtocol = SecurityProtocol.SaslPlaintext; config.AutoOffsetReset = AutoOffsetReset.Earliest; config.NumStreamThreads = 1; StreamBuilder builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("test"); var stream2 = builder.Stream <string, string>("test2"); stream1.Join <string, string>(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromMinutes(1))) .To("output-join"); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => { source.Cancel(); stream.Close(); }; stream.Start(source.Token); }
public async void process(IConfiguration config) { var sConfig = new StreamConfig <StringSerDes, StringSerDes>(); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; StreamBuilder builder = new StreamBuilder(); var table = builder.Table("product", new StringSerDes(), new StringSerDes(), InMemory <String, String> .As("product-store")); builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.input.destination"]) .Join(table, (order, product) => order + product) .To(config["spring.cloud.stream.bindings.output.destination"]); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, sConfig); await stream.StartAsync(); }
public async Task process(IConfiguration config) { Console.WriteLine("Process"); var sConfig = new StreamConfig <StringSerDes, StringSerDes>(); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; StreamBuilder builder = new StreamBuilder(); var table = builder.Table(config["simpleNetcoreProcessor.externaltopic"], new StringSerDes(), new StringSerDes(), InMemory <String, String> .As(config["simpleNetcoreProcessor.table"])); builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.input.destination"]) .Join(table, (order, product) => order + product) .To(config["spring.cloud.stream.bindings.output.destination"]); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, sConfig); await stream.StartAsync(); }
public async Task exec(IConfiguration config, IServiceProvider services) { Console.WriteLine("Process"); // Inyectamos los datos obtenidos al Stream var sConfig = new StreamConfig <StringSerDes, StringSerDes>(); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; StreamBuilder builder = new StreamBuilder(); builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.output.destination"]) .To(config["spring.cloud.stream.bindings.output.destination"]); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, sConfig); await stream.StartAsync(); }
static async System.Threading.Tasks.Task Main(string[] args) { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.BootstrapServers = "localhost:9093"; StreamBuilder builder = new StreamBuilder(); builder.Stream <string, string>("evenements") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Aggregate(() => "", (k, v, va) => va += v) .ToStream() .Print(Printed <Windowed <String>, String> .ToOut()); Topology t = builder.Build(); KafkaStream stream = new KafkaStream(t, config); Console.CancelKeyPress += (o, e) => stream.Dispose(); await stream.StartAsync(); }
public ConsumerPerfThread(int threadId, string name, KafkaStream<byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead) { this.threadId = threadId; this.name = name; this.stream = stream; this.config = config; this.totalMessagesRead = totalMessagesRead; this.totalBytesRead = totalBytesRead; }