static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://localhost:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
var orders = builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders");
orders.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
orders
.GroupByKey()
.Aggregate <OrderAgg, SchemaAvroSerDes <OrderAgg> >(
() => new OrderAgg(),
(key, order, agg) =>
{
agg.order_id = order.order_id;
agg.price = order.price;
agg.product_id = order.product_id;
agg.totalPrice += order.price;
return(agg);
})
.ToStream()
.Print(Printed <int, OrderAgg> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
StreamBuilder builder = new StreamBuilder();
IKStream <string, string> kStream = builder.Stream <string, string>("test-topic");
kStream.Print(Printed <string, string> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
config.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Earliest;
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
StreamBuilder builder = new StreamBuilder();
builder
.Table("table", RocksDb <string, string> .As("table-store").WithLoggingEnabled())
.ToStream()
.Print(Printed <string, string> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
await stream.StartAsync(stoppingToken);
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9093";
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("evenements")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Aggregate(() => "", (k, v, va) => va += v)
.ToStream()
.Print(Printed <Windowed <String>, String> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
public async Task process(IConfiguration config)
{
// creando AvroSchemas from clases
AvroSerializerSettings settings = new AvroSerializerSettings();
settings.Resolver = new AvroPublicMemberContractResolver();
var endpontSchema = AvroSerializer.Create <Endpoint> (settings).WriterSchema.ToString();
var messageDSchema = AvroSerializer.Create <MessageDestination> (settings).WriterSchema.ToString();
Console.WriteLine("Endpoint Schema: " + endpontSchema);
Console.WriteLine("Message Destination Schema: " + messageDSchema);
Console.WriteLine("RouterProcess");
var sConfig = new StreamConfig <StringSerDes, StringSerDes> ();
sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GROUP"];
sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"];
sConfig.AutoOffsetReset = AutoOffsetReset.Earliest;
sConfig.SchemaRegistryUrl = config["SchemaRegistryUrl"];
sConfig.AutoRegisterSchemas = true;
sConfig.NumStreamThreads = 1;
sConfig.Acks = Acks.All;
//sConfig.Debug = "consumer,cgrp,topic,fetch";
sConfig.AddConsumerConfig("allow.auto.create.topics", "true");
sConfig.MaxTaskIdleMs = 50;
sConfig.InnerExceptionHandler = (e) => ExceptionHandlerResponse.CONTINUE;
var timeout = TimeSpan.FromSeconds(10);
DateTime dt = DateTime.Now;
MessageDestination op = new MessageDestination();
var serializer = new SchemaAvroSerDes <OrderProduct>();
StreamBuilder builder = new StreamBuilder();
var table = builder.Table(config["endpoints"],
new Int32SerDes(),
new SchemaAvroSerDes <Endpoint> (),
InMemory <int, Endpoint> .As(config["endpoints-table"]));
builder.Stream <int, OrderProduct, Int32SerDes, SchemaAvroSerDes <OrderProduct> >(config["spring.cloud.stream.bindings.input.destination"])
.Map <int, OrderProduct>((k, v) =>
{
return(KeyValuePair.Create(v.product_id, v));
})
.Peek((k, v) =>
{
Console.WriteLine($"Sending message {k} to endpoint {v.product_id}");
//calcular metrica
})
.Join(table, (orderProduct, endpoint) => {
Console.WriteLine("OrderProduct: " + orderProduct?.order_id);
Console.WriteLine("Endpoint: " + endpoint?.endpoint_id);
op = new MessageDestination {
messageId = orderProduct.order_id,
endpoint = endpoint,
payload = orderProduct
};
return(op);
})
.Peek((k, v) =>
{
Console.WriteLine($"Sending message {k} to endpoint {v.endpoint.endpoint_url}");
// crear metricas
if (_messageCounterList != null)
{
var counterMessage = Metrics
.CreateCounter($"router_{v.endpoint.endpoint_id}_processed_total", $"Number of messages sent to {v.endpoint.endpoint_url}");
counterMessage.Inc();
_messageCounterList.Add(counterMessage);
}
})
.Print(Printed <int, MessageDestination> .ToOut());
Topology t = builder.Build();
Console.WriteLine(t.Describe());
KafkaStream stream = new KafkaStream(t, sConfig);
bool isRunningState = false;
stream.StateChanged += (old, @new) => {
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
if (isRunningState)
{
Console.WriteLine("Stream running state is " + isRunningState.ToString());
}
}