public void StateStoreUpdateKey()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Table <string, string>("test", InMemory <string, string> .As("store"));
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNotNull(store);
Assert.IsInstanceOf <ReadOnlyKeyValueStore <string, ValueAndTimestamp <string> > >(store);
input.PipeInput("coucou", "1");
Assert.AreEqual(1, ((ReadOnlyKeyValueStore <string, ValueAndTimestamp <string> >)store).All().Count());
Assert.AreEqual("1", ((ReadOnlyKeyValueStore <string, ValueAndTimestamp <string> >)store).Get("coucou").Value);
input.PipeInput("coucou", "2");
Assert.AreEqual(1, ((ReadOnlyKeyValueStore <string, ValueAndTimestamp <string> >)store).All().Count());
Assert.AreEqual("2", ((ReadOnlyKeyValueStore <string, ValueAndTimestamp <string> >)store).Get("coucou").Value);
driver.Dispose();
}
public void StreamTableJoin3()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-table-join"
};
StreamBuilder builder = new StreamBuilder();
var table = builder
.Table("test", InMemory <string, string> .As("store"));
builder
.Stream <string, string>("stream")
.Join(table, new MyJoinValueMapper())
.To("output");
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic = driver.CreateInputTopic <string, string>("test");
var inputTopic2 = driver.CreateInputTopic <string, string>("stream");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTopic.PipeInput("test", "test");
inputTopic2.PipeInput("test", "coucou");
var record = outputTopic.ReadKeyValue();
Assert.IsNotNull(record);
Assert.AreEqual("test", record.Message.Key);
Assert.AreEqual("coucou-test", record.Message.Value);
}
public void GetStateStoreExist()
{
var source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var topicConfiguration = config.Clone();
topicConfiguration.ApplicationId = $"test-driver-{config.ApplicationId}";
var builder = new StreamBuilder();
builder.Table <string, string>("test", InMemory <string, string> .As("store"));
var driver = new ClusterInMemoryTopologyDriver("client", builder.Build().Builder, config, topicConfiguration, TimeSpan.FromSeconds(1), source.Token);
driver.StartDriver();
var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNotNull(store);
Assert.IsInstanceOf <MockReadOnlyKeyValueStore <string, string> >(store);
input.PipeInput("coucou", "1");
Thread.Sleep(100);
Assert.AreEqual(1, ((MockReadOnlyKeyValueStore <string, string>)store).All().Count());
Assert.AreEqual("1", ((MockReadOnlyKeyValueStore <string, string>)store).Get("coucou"));
source.Cancel();
driver.Dispose();
}
private static void Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-test-driver-app"
};
StreamBuilder builder = new StreamBuilder();
var table = builder
.Table <string, string>("test")
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Count(InMemory <string, long> .As("count-store"));
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("test");
inputTopic.PipeInput("renault", "clio");
inputTopic.PipeInput("renault", "megane");
inputTopic.PipeInput("ferrari", "red");
var store = driver.GetKeyValueStore <string, long>("count-store");
var elements = store.All();
}
}
public void KStreamGlobalLeftJoinNull()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-table-join"
};
StreamBuilder builder = new StreamBuilder();
var global = builder.GlobalTable("global", InMemory <string, string> .As("global-store"));
builder
.Stream <string, string>("stream")
.LeftJoin(global, (k, v) => k, (s, v) => $"{s}-{v}")
.To("output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("global");
var inputTopic2 = driver.CreateInputTopic <string, string>("stream");
var outputTopic = driver.CreateOuputTopic <string, string>("output");
inputTopic2.PipeInput("test", "coucou");
inputTopic.PipeInput("test", "test");
var record = outputTopic.ReadKeyValue();
Assert.IsNotNull(record);
Assert.AreEqual("test", record.Message.Key);
Assert.AreEqual("coucou-", record.Message.Value);
}
}
public void TableTableJoinValueJoiner()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-table-table-join"
};
StreamBuilder builder = new StreamBuilder();
var table1 = builder.Table("table1", InMemory <string, string> .As("store1"));
var table2 = builder.Table("table2", InMemory <string, string> .As("store2"));
var tableJoin = table1.Join(table2, new ValueJoiner());
tableJoin.ToStream().To("topic-output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic1 = driver.CreateInputTopic <string, string>("table1");
var inputTopic2 = driver.CreateInputTopic <string, string>("table2");
var outputTopic = driver.CreateOuputTopic <string, string>("topic-output");
inputTopic1.PipeInput("test", "test");
inputTopic2.PipeInput("test", "coucou");
var record = outputTopic.ReadKeyValue();
Assert.IsNotNull(record);
Assert.AreEqual("test", record.Message.Key);
Assert.AreEqual("test-coucou", record.Message.Value);
}
}
public void TestToTable()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-stream-to-table"
};
StreamBuilder builder = new StreamBuilder();
var table = builder
.Stream <string, string>("test")
.Filter((k, v) => v.Length % 2 == 0)
.ToTable(InMemory <string, string> .As("table-store"));
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic = driver.CreateInputTopic <string, string>("test");
inputTopic.PipeInput("test", "test");
inputTopic.PipeInput("test", "tes");
var store = driver.GetKeyValueStore <string, string>("table-store");
Assert.IsNotNull(store);
Assert.AreEqual("test", store.Get("test"));
inputTopic.PipeInput("test", "test12");
Assert.AreEqual("test12", store.Get("test"));
}
public void TableTableOuterJoinNullKey()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-table-table-outer-join"
};
StreamBuilder builder = new StreamBuilder();
var table1 = builder.Table("table1", InMemory <string, string> .As("store1"));
var table2 = builder.Table("table2", InMemory <string, string> .As("store2"));
var tableJoin = table1.LeftJoin(table2, (v1, v2) => $"{v1}-{v2}");
tableJoin.ToStream().To("topic-output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic1 = driver.CreateInputTopic <string, string>("table1");
var inputTopic2 = driver.CreateInputTopic <string, string>("table2");
var outputTopic = driver.CreateOuputTopic <string, string>("topic-output");
inputTopic1.PipeInput("test", "test");
inputTopic1.PipeInput(null, "test");
var records = outputTopic.ReadKeyValuesToMap();
Assert.IsNotNull(records);
Assert.AreEqual(1, records.Count);
Assert.AreEqual("test-", records["test"]);
}
}
public void Start()
{
// TODO: I guess we should dispose of this stream at some point
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = bootstrapServers;
config.SaslMechanism = SaslMechanism.ScramSha256;
config.SaslUsername = saslUsername;
config.SaslPassword = saslPassword;
config.SecurityProtocol = SecurityProtocol.SaslSsl;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
// TODO: this should be a global store I guess, but it is not supported in Streamiz.Kafka.Net yet
// for now we use topic with one partition to get all the data in one StreamTask
builder
// TODO: we need to provide serdes for key anc value as Streamiz.Kafka.Net tries to deserialize
// value with deserializer for ValueAndTimestamp<string> which is incorrect
.Table(topic, new StringSerDes(), new StringSerDes(), InMemory <string, string> .As("topics-store")
.WithCachingDisabled());
Topology t = builder.Build();
_stream = new KafkaStream(t, config);
_stream.Start(source.Token);
}
public void GetElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
producer.Produce("topic",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1"),
Value = serdes.Serialize("coucou")
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
var item = store.Get("key1");
Assert.IsNotNull(item);
Assert.AreEqual("coucou", item);
}
source.Cancel();
stream.Close();
}
public void Agg4()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-agg";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupBy((k, v) => k.ToUpper())
.Aggregate(
() => 0L,
(k, v, agg) => agg + 1,
InMemory <string, long> .As("agg-store").WithValueSerdes <Int64SerDes>());
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", null);
input.PipeInput("test", "12");
var store = driver.GetKeyValueStore <string, long>("agg-store");
Assert.IsNotNull(store);
// null doesn't matter
Assert.AreEqual(1, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual(2, el);
}
}
public void Reduce2()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
builder
.Table <string, string>("topic")
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Reduce(
(v1, v2) => v2.Length > v1.Length ? v2 : v1,
(v1, v2) => v2,
InMemory <string, string> .As("reduce-store"));
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "12");
var store = driver.GetKeyValueStore <string, string>("reduce-store");
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual("12", el);
}
}
public void Reduce4()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-count";
var builder = new StreamBuilder();
builder
.Table <string, string>("topic")
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Reduce(
new MyAddReducer(), new MySubReducer(),
InMemory <string, string> .As("reduce-store"),
"reduce-processor");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", null);
input.PipeInput("test", "12");
var store = driver.GetKeyValueStore <string, string>("reduce-store");
Assert.IsNotNull(store);
// null doesn't matter
Assert.AreEqual(1, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual("12", el);
}
}
public void MapValuesSameValueType()
{
var builder = new StreamBuilder();
var data = new List <KeyValuePair <string, string> >();
data.Add(KeyValuePair.Create("key1", "test1234"));
data.Add(KeyValuePair.Create("key2", "test"));
data.Add(KeyValuePair.Create("key3", "paper"));
builder.Table <string, string>("table-topic")
.MapValues((v) => v.ToCharArray()[0].ToString(), InMemory <string, string> .As("test-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "table-test-mapvalues";
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInputs(data);
var store = driver.GetKeyValueStore <string, string>("test-store");
Assert.IsNotNull(store);
Assert.AreEqual(store.All().Count(), 3);
var r1 = store.Get("key1");
var r2 = store.Get("key2");
var r3 = store.Get("key3");
Assert.AreEqual("t", r1);
Assert.AreEqual("t", r2);
Assert.AreEqual("p", r3);
}
}
public void SimpleKTableSource3()
{
var builder = new StreamBuilder();
builder.Table <string, string, StringSerDes, StringSerDes>
("table-topic", InMemory <string, string> .As("table-topic-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
Topology t = builder.Build();
using var driver = new TopologyTestDriver(t, config);
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var resultK1 = store.Get("key1");
var resultK2 = store.Get("key2");
Assert.AreEqual("1", resultK1);
Assert.AreEqual("2", resultK2);
}
public void Reduce3()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupBy((k, v) => k.ToUpper())
.Reduce(new MyReducer(), InMemory <string, string> .As("reduce-store"));
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "15151500");
input.PipeInput("test", "1200");
var store = driver.GetKeyValueStore <string, string>("reduce-store");
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual("15151500", el);
}
public void KTableToStreamWithLastUpdate()
{
var builder = new StreamBuilder();
builder.Table("table-topic", InMemory <string, string> .As("table-topic-store"))
.ToStream().To("table-stream");
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
var outputTopic = driver.CreateOuputTopic <string, string>("table-stream");
var expected = new List <KeyValuePair <string, string> >();
expected.Add(KeyValuePair.Create("key1", "c"));
inputTopic.PipeInput("key1", "a");
inputTopic.PipeInput("key1", "b");
inputTopic.PipeInput("key1", "c");
var results = outputTopic.ReadKeyValuesToMap();
Assert.AreEqual(1, results.Count);
Assert.AreEqual("c", results["key1"]);
}
}
public void CountWithNullValue()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-count";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupBy((k, v) => k.ToCharArray()[0])
.Count(InMemory <char, long> .As("count-store").WithKeySerdes(new CharSerDes()));
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", null);
input.PipeInput("coucou", "120");
var store = driver.GetKeyValueStore <char, long>("count-store");
Assert.IsNotNull(store);
Assert.AreEqual(2, store.ApproximateNumEntries());
Assert.AreEqual(1, store.Get('t'));
Assert.AreEqual(1, store.Get('c'));
}
}
public void KTableSourceReverseAllStateStore()
{
var builder = new StreamBuilder();
builder.Table("table-topic", InMemory <string, string> .As("table-topic-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key3", "2");
inputTopic.PipeInput("key2", "2");
inputTopic.PipeInput("key4", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var results = store.ReverseAll().ToList();
Assert.AreEqual(4, results.Count);
Assert.AreEqual("key4", results[0].Key);
Assert.AreEqual("key2", results[1].Key);
Assert.AreEqual("key3", results[2].Key);
Assert.AreEqual("key1", results[3].Key);
}
}
public async Task BuildGlobalStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 1;
var builder = new StreamBuilder();
builder.GlobalTable <string, string>("test", InMemory <string, string> .As("store"));
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(new ProducerConfig());
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var stringSerdes = new StringSerDes();
producer.Produce("test",
new Message <byte[], byte[]>
{
Key = stringSerdes.Serialize("key", new SerializationContext()),
Value = stringSerdes.Serialize("value", new SerializationContext())
});
Thread.Sleep(250);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
}
stream.Dispose();
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://localhost:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
var orders = builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders");
orders.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
orders
.GroupByKey()
.Aggregate <OrderAgg, SchemaAvroSerDes <OrderAgg> >(
() => new OrderAgg(),
(key, order, agg) =>
{
agg.order_id = order.order_id;
agg.price = order.price;
agg.product_id = order.product_id;
agg.totalPrice += order.price;
return(agg);
})
.ToStream()
.Print(Printed <int, OrderAgg> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.FilterNot((k, v) => v.Contains("test"))
.To("test-output");
builder.Table("test-ktable", InMemory <string, string> .As("test-store"));
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
bool taskGetStateStoreRunning = false;
stream.StateChanged += (old, @new) =>
{
if (@new == KafkaStream.State.RUNNING && !taskGetStateStoreRunning)
{
Task.Factory.StartNew(() =>
{
taskGetStateStoreRunning = true;
while (!source.Token.IsCancellationRequested)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.KeyValueStore <string, string>()));
var items = store.All().ToList();
Thread.Sleep(500);
}
}, source.Token);
}
};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
private static void Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-test-driver-app";
StreamBuilder builder = new StreamBuilder();
var stream = builder
.Stream <string, string>("test")
.GroupBy((k, v) => k.ToUpper());
stream.Count(InMemory <string, long> .As("count-store"));
stream.Aggregate(
() => new Dictionary <char, int>(),
(k, v, old) =>
{
var caracs = v.ToCharArray();
foreach (var c in caracs)
{
if (old.ContainsKey(c))
{
++old[c];
}
else
{
old.Add(c, 1);
}
}
return(old);
},
InMemory <string, Dictionary <char, int> > .As("agg-store").WithValueSerdes(new DictionarySerDes())
);
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("test");
var outputTopic = driver.CreateOuputTopic <string, string>("test-output", TimeSpan.FromSeconds(5));
inputTopic.PipeInput("test", "test");
inputTopic.PipeInput("test", "test2");
var store = driver.GetKeyValueStore <string, Dictionary <char, int> >("agg-store");
var el = store.Get("TEST");
var storeCount = driver.GetKeyValueStore <string, long>("count-store");
var e = storeCount.Get("TEST");
}
}
public async Task GetKVStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.Count(InMemory <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>())));
state = true;
}
}
};
await stream.StartAsync();
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
stream.Dispose();
}
public void ReduceAndQueryInStateStore()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
var table = builder
.Table <string, string>("topic")
.MapValues(v => v.Length)
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v));
table.Count(InMemory <string, long> .As("count-store"));
table.Reduce(
(v1, v2) => Math.Max(v1, v2),
(v1, v2) => Math.Max(v1, v2),
InMemory <string, int> .As("reduce-store").WithValueSerdes <Int32SerDes>());
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "120");
input.PipeInput("test", "30");
input.PipeInput("coucou", "120");
var store = driver.GetKeyValueStore <string, int>("reduce-store");
Assert.IsNotNull(store);
Assert.AreEqual(2, store.ApproximateNumEntries());
var el = store.Get("TEST");
Assert.IsNotNull(el);
Assert.AreEqual(3, el);
var storeCount = driver.GetKeyValueStore <string, long>("count-store");
Assert.IsNotNull(storeCount);
Assert.AreEqual(2, store.ApproximateNumEntries());
var e = storeCount.Get("TEST");
Assert.IsNotNull(e);
Assert.AreEqual(1, e);
}
public void GetStateStoreBeforeRunningState()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())));
stream.Dispose();
}
public async Task GetStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));;
Assert.IsNotNull(store);
}
stream.Dispose();
}
public void GetStateStoreDoesntExists()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var supplier = new SyncKafkaSupplier();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>())));
}
source.Cancel();
stream.Close();
}
static async Task Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
// NEED FOR SchemaAvroSerDes
config.SchemaRegistryUrl = "http://192.168.56.1:8081";
config.AutoRegisterSchemas = true;
StreamBuilder builder = new StreamBuilder();
var table = builder.Table("product",
new Int32SerDes(),
new SchemaAvroSerDes <Product>(),
InMemory <int, Product> .As("product-store"));
builder.Stream <int, Order, Int32SerDes, SchemaAvroSerDes <Order> >("orders")
.Join(table, (order, product) => new OrderProduct
{
order_id = order.order_id,
price = order.price,
product_id = product.product_id,
product_name = product.name,
product_price = product.price
})
.To <Int32SerDes, SchemaAvroSerDes <OrderProduct> >("orders-output");
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) =>
{
stream.Dispose();
};
await stream.StartAsync();
}
public void TableTableOuterJoinGetterSupplier()
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-table-table-outer-join"
};
StreamBuilder builder = new StreamBuilder();
var table1 = builder.Table("users", InMemory <string, string> .As("store-users"));
var table2 = builder.Table("regions", InMemory <string, string> .As("store-regions"));
var stream = builder.Stream <string, string>("orders");
var tableJoin = table1.OuterJoin(table2, (v1, v2) => $"{v1 ?? "?"}-{v2 ?? "?"}");
stream
.Join(tableJoin,
(order, ur) => $"Order:{order}|UserRegion:{ur}")
.To("topic-output");
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic1 = driver.CreateInputTopic <string, string>("users");
var inputTopic2 = driver.CreateInputTopic <string, string>("regions");
var inputTopic3 = driver.CreateInputTopic <string, string>("orders");
var outputTopic = driver.CreateOuputTopic <string, string>("topic-output");
inputTopic1.PipeInput("sylvain", "sylvain");
inputTopic1.PipeInput("lise", "lise");
inputTopic2.PipeInput("sylvain", "France");
inputTopic2.PipeInput("remi", "USA");
inputTopic3.PipeInput("sylvain", "iPhone12Pro");
inputTopic3.PipeInput("lise", "PixelA4");
inputTopic3.PipeInput("remi", "Tab");
var records = outputTopic.ReadKeyValuesToMap();
Assert.IsNotNull(records);
Assert.AreEqual(3, records.Count);
Assert.AreEqual("Order:iPhone12Pro|UserRegion:sylvain-France", records["sylvain"]);
Assert.AreEqual("Order:PixelA4|UserRegion:lise-?", records["lise"]);
Assert.AreEqual("Order:Tab|UserRegion:?-USA", records["remi"]);
}
}