public static async Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app2";
config.BootstrapServers = "localhost:9092";
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.StateDir = Path.Combine(".");
config.CommitIntervalMs = 5000;
config.Logger = LoggerFactory.Create(builder =>
{
builder.SetMinimumLevel(LogLevel.Information);
builder.AddLog4Net();
});
StreamBuilder builder = new StreamBuilder();
var table = builder.GlobalTable <string, string>("topic");
builder.Stream <string, string>("input")
.GroupBy((k, v) => v)
.Count()
.ToStream()
.To("output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
public async Task StartKafkaStreamWithToken()
{
CancellationTokenSource source = new CancellationTokenSource();
KafkaStream.State lastState = KafkaStream.State.CREATED;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (o, n) => lastState = n;
await stream.StartAsync(source.Token);
Thread.Sleep(1500);
source.Cancel();
Thread.Sleep(1500);
Assert.AreEqual(KafkaStream.State.NOT_RUNNING, lastState);
}
public void GetElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
producer.Produce("topic",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1"),
Value = serdes.Serialize("coucou")
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
var item = store.Get("key1");
Assert.IsNotNull(item);
Assert.AreEqual("coucou", item);
}
source.Cancel();
stream.Close();
}
private static async Task MainToken(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-token-app";
config.BootstrapServers = "localhost:29092";
config.PollMs = 100;
config.MaxPollRecords = 500;
StreamBuilder builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.To("test-output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
};
await stream.StartAsync(source.Token);
}
static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.FilterNot((k, v) => v.Contains("test"))
.To("test-output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => {
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
static async Task Main(string[] args)
{
string inputTopic = "input", outputTopic = "output";
int numberPartitions = 4;
await CreateTopics(inputTopic, outputTopic, numberPartitions);
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "sample-streamiz-demo";
config.BootstrapServers = "localhost:9092";
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.StateDir = Path.Combine(".");
config.CommitIntervalMs = 5000;
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.MetricsRecording = MetricsRecordingLevel.DEBUG;
config.UsePrometheusReporter(9090, true);
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>(inputTopic)
.FlatMapValues((v) => v.Split(" ").AsEnumerable())
.GroupBy((k, v) => v)
.Count()
.ToStream()
.MapValues((v) => v.ToString())
.To(outputTopic);
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://192.168.56.1:8081";
config.AutoRegisterSchemas = false;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, Person, StringSerDes, SchemaAvroSerDes <Person> >("person")
.Filter((k, v) => v.age >= 18)
.MapValues((v) => $"{v.firstName}-{v.lastName}-{v.age}")
.To <StringSerDes, StringSerDes>("output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
public async Task BuildGlobalStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 1;
var builder = new StreamBuilder();
builder.GlobalTable <string, string>("test", InMemory <string, string> .As("store"));
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(new ProducerConfig());
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var stringSerdes = new StringSerDes();
producer.Produce("test",
new Message <byte[], byte[]>
{
Key = stringSerdes.Serialize("key", new SerializationContext()),
Value = stringSerdes.Serialize("value", new SerializationContext())
});
Thread.Sleep(250);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
}
stream.Dispose();
}
public ConsumerPerfThread(int threadId, string name, KafkaStream <byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead)
{
this.threadId = threadId;
this.name = name;
this.stream = stream;
this.config = config;
this.totalMessagesRead = totalMessagesRead;
this.totalBytesRead = totalBytesRead;
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://localhost:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
var orders = builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders");
orders.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
orders
.GroupByKey()
.Aggregate <OrderAgg, SchemaAvroSerDes <OrderAgg> >(
() => new OrderAgg(),
(key, order, agg) =>
{
agg.order_id = order.order_id;
agg.price = order.price;
agg.product_id = order.product_id;
agg.totalPrice += order.price;
return(agg);
})
.ToStream()
.Print(Printed <int, OrderAgg> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
private static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"))
.ToStream()
.Map((k, v) => KeyValuePair.Create(k.Key, v.ToString()))
.To("output");
Topology t = builder.Build();
//bool taskStart = false;
KafkaStream stream = new KafkaStream(t, config);
// Subscribe state changed
//stream.StateChanged += (old, @new) =>
//{
// if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store.
// {
// Task.Factory.StartNew(() =>
// {
// while (!source.Token.IsCancellationRequested)
// {
// var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>()));
// var items = store.All().ToList();
// Thread.Sleep(500);
// }
// }, source.Token);
// taskStart = true;
// }
//};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
public void Start(CancellationToken token = default)
{
foreach (var query in querys)
{
query.Analyze(builder);
}
var topology = builder.Build();
stream = new KafkaStream(topology, config);
stream.Start(token);
}
static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.FilterNot((k, v) => v.Contains("test"))
.To("test-output");
builder.Table("test-ktable", InMemory <string, string> .As("test-store"));
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
bool taskGetStateStoreRunning = false;
stream.StateChanged += (old, @new) =>
{
if (@new == KafkaStream.State.RUNNING && !taskGetStateStoreRunning)
{
Task.Factory.StartNew(() =>
{
taskGetStateStoreRunning = true;
while (!source.Token.IsCancellationRequested)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.KeyValueStore <string, string>()));
var items = store.All().ToList();
Thread.Sleep(500);
}
}, source.Token);
}
};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
public async Task GetWindowStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
}
stream.Dispose();
}
public void GetWStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())));
state = true;
}
}
};
stream.Start(source.Token);
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
source.Cancel();
stream.Close();
}
public async Task GetKVStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.Count(InMemory <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>())));
state = true;
}
}
};
await stream.StartAsync();
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
stream.Dispose();
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://192.168.56.1:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders")
.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
public void GetStateStoreDoesntExists()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var supplier = new SyncKafkaSupplier();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>())));
}
source.Cancel();
stream.Close();
}
public void GetStateStoreBeforeRunningState()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())));
stream.Dispose();
}
public async Task StartKafkaStream()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
await stream.StartAsync();
Thread.Sleep(1500);
stream.Dispose();
}
public async Task StartKafkaStreamWaitRunningState()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
var sensors = stream.Metrics();
Assert.IsTrue(sensors.Any());
stream.Dispose();
Assert.IsTrue(isRunningState);
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.To("test2");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
public void StartKafkaStream()
{
var source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.Start(source.Token);
Thread.Sleep(1500);
source.Cancel();
stream.Close();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
config.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Earliest;
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
StreamBuilder builder = new StreamBuilder();
builder
.Table("table", RocksDb <string, string> .As("table-store").WithLoggingEnabled())
.ToStream()
.Print(Printed <string, string> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
await stream.StartAsync(stoppingToken);
}
public void StartKafkaStreamWaitRunningState()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Stream <string, string>("topic").To("topic2");
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
source.Cancel();
stream.Close();
Assert.IsTrue(isRunningState);
}
private static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
var stream1 = builder.Stream <string, string>("test");
var stream2 = builder.Stream <string, string>("test2");
stream1.Join <string, string>(stream2,
(v1, v2) => $"{v1}-{v2}",
JoinWindowOptions.Of(TimeSpan.FromMinutes(1)))
.To("output-join");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
public async void process(IConfiguration config)
{
var sConfig = new StreamConfig <StringSerDes, StringSerDes>();
sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"];
sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"];
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new StringSerDes(),
new StringSerDes(),
InMemory <String, String> .As("product-store"));
builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.input.destination"])
.Join(table, (order, product) => order + product)
.To(config["spring.cloud.stream.bindings.output.destination"]);
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, sConfig);
await stream.StartAsync();
}
public async Task process(IConfiguration config)
{
Console.WriteLine("Process");
var sConfig = new StreamConfig <StringSerDes, StringSerDes>();
sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"];
sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"];
StreamBuilder builder = new StreamBuilder();
var table = builder.Table(config["simpleNetcoreProcessor.externaltopic"],
new StringSerDes(),
new StringSerDes(),
InMemory <String, String> .As(config["simpleNetcoreProcessor.table"]));
builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.input.destination"])
.Join(table, (order, product) => order + product)
.To(config["spring.cloud.stream.bindings.output.destination"]);
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, sConfig);
await stream.StartAsync();
}
public async Task exec(IConfiguration config, IServiceProvider services)
{
Console.WriteLine("Process");
// Inyectamos los datos obtenidos al Stream
var sConfig = new StreamConfig <StringSerDes, StringSerDes>();
sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"];
sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"];
StreamBuilder builder = new StreamBuilder();
builder.Stream <String, String, StringSerDes, StringSerDes>(config["spring.cloud.stream.bindings.output.destination"])
.To(config["spring.cloud.stream.bindings.output.destination"]);
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, sConfig);
await stream.StartAsync();
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9093";
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("evenements")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Aggregate(() => "", (k, v, va) => va += v)
.ToStream()
.Print(Printed <Windowed <String>, String> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
public ConsumerPerfThread(int threadId, string name, KafkaStream<byte[], byte[]> stream, ConsumerPerfConfig config, AtomicLong totalMessagesRead, AtomicLong totalBytesRead)
{
this.threadId = threadId;
this.name = name;
this.stream = stream;
this.config = config;
this.totalMessagesRead = totalMessagesRead;
this.totalBytesRead = totalBytesRead;
}
