public void TimeWindowingAggKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingReduceKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingQueryStoreAll()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>());
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "2567");
input.PipeInput("test", "32");
var store = driver.GetWindowStore <string, int>("store");
var elements = store.All().ToList();
Assert.AreEqual(1, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual(4, elements[0].Value);
}
}
public void TimeWindowingCountKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingQueryStore2Window()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(InMemoryWindows <string, long> .As("count-store"));
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
DateTime dt = DateTime.Now;
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1", dt);
input.PipeInput("test", "2", dt);
input.PipeInput("test", "3", dt.AddMinutes(1));
var store = driver.GetWindowStore <string, long>("count-store");
var elements = store.All().ToList();
Assert.AreEqual(2, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual(2, elements[0].Value);
Assert.AreEqual("test", elements[1].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs);
Assert.AreEqual(1, elements[1].Value);
}
}
public void TimeWindowingQueryStoreAll()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To <StringTimeWindowedSerDes, StringSerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "2567");
input.PipeInput("test", "32");
var store = driver.GetWindowStore <string, string>("store");
var elements = store.All().ToList();
Assert.AreEqual(1, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual("2567", elements[0].Value);
}
}
public void TimeWindowingReduceNothing()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To <StringTimeWindowedSerDes, StringSerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(0, elements.Count);
}
}
public void TestGetWindowStoreIncorrectType()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(InMemoryWindows <string, long> .As("count-store"));
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
DateTime dt = DateTime.Now;
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1", dt);
var store = driver.GetWindowStore <int, long>("count-store");
Assert.IsNull(store);
}
public void WithNullValueSerDes()
{
// WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, string, IWindowStore <Bytes, byte[]> > m =
Materialized <string, string, IWindowStore <Bytes, byte[]> >
.Create("store")
.With(null, null);
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2, m)
.ToStream()
.To("output-topic");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingCountNothing()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(1)))
.Count()
.ToStream()
.To <StringTimeWindowedSerDes, Int64SerDes>("output");
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(0, elements.Count);
}
public void WindowStateStoreExist()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Count(InMemoryWindows <string, long> .As("store"));
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNotNull(store);
Assert.IsInstanceOf <TimestampedWindowStore <string, long> >(store);
DateTime dt = DateTime.Now;
input.PipeInput("coucou", "1");
Assert.AreEqual(1, ((TimestampedWindowStore <string, long>)store).All().ToList().Count);
var it = ((TimestampedWindowStore <string, long>)store).Fetch("coucou", dt.AddSeconds(-5), dt.AddSeconds(5));
Assert.IsTrue(it.MoveNext());
Assert.AreEqual(1, it.Current.Value.Value.Value);
driver.Dispose();
}
private static void Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-test-driver-app"
};
StreamBuilder builder = new StreamBuilder();
var table = builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(InMemoryWindows <string, long> .As("count-store"));
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("test");
inputTopic.PipeInput("renault", "clio", dt);
inputTopic.PipeInput("renault", "megane", dt.AddMilliseconds(10));
Thread.Sleep((int)TimeSpan.FromSeconds(10).TotalMilliseconds);
inputTopic.PipeInput("renault", "scenic", dt.AddSeconds(1));
var store = driver.GetWindowStore <string, long>("count-store");
var elements = store.All().ToList();
}
}
public void TimeWindowingAggNothing()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To <StringTimeWindowedSerDes, Int32SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(0, elements.Count);
}
}
public void Windowing()
{
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(1000))
.Count(RocksDbWindows <string, long> .As <StringSerDes, Int64SerDes>("rocksdb-w-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-rocksdb-window-store";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("key1", "1", dt);
inputTopic.PipeInput("key2", "2", dt);
var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
Assert.IsNotNull(store);
var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();
Assert.AreEqual(2, k1.Count);
Assert.AreEqual(1L, k1[0].Value);
Assert.AreEqual(1L, k1[1].Value);
}
config.RemoveRocksDbFolderForTest();
}
public void WithNullMaterialize()
{
// CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
var config = new StreamConfig <StringSerDes, StringSerDes>();
var serdes = new StringSerDes();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, long, IWindowStore <Bytes, byte[]> > m = null;
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Count(m);
var topology = builder.Build();
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
null,
new MockChangelogRegister(),
new StreamMetricsRegistry());
task.GroupMetadata = consumer as SyncConsumer;
task.InitializeStateStores();
task.InitializeTopology();
task.RestorationIfNeeded();
task.CompleteRestoration();
Assert.AreEqual(1, task.Context.States.StateStoreNames.Count());
var nameStore = task.Context.States.StateStoreNames.ElementAt(0);
Assert.IsNotNull(nameStore);
Assert.AreNotEqual(string.Empty, nameStore);
var store = task.GetStore(nameStore);
Assert.IsInstanceOf <ITimestampedWindowStore <string, long> >(store);
Assert.AreEqual(0, (store as ITimestampedWindowStore <string, long>).All().ToList().Count);
}
private static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"))
.ToStream()
.Map((k, v) => KeyValuePair.Create(k.Key, v.ToString()))
.To("output");
Topology t = builder.Build();
//bool taskStart = false;
KafkaStream stream = new KafkaStream(t, config);
// Subscribe state changed
//stream.StateChanged += (old, @new) =>
//{
// if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store.
// {
// Task.Factory.StartNew(() =>
// {
// while (!source.Token.IsCancellationRequested)
// {
// var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>()));
// var items = store.All().ToList();
// Thread.Sleep(500);
// }
// }, source.Token);
// taskStart = true;
// }
//};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
private void BuildTopology(StreamBuilder builder)
{
builder.Stream <string, ObjectA, StringSerDes, JSONSerDes <ObjectA> >("source")
.Map((key, value) => new KeyValuePair <string, ObjectA>(value.Symbol, value))
.GroupByKey <StringSerDes, JSONSerDes <ObjectA> >()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(5)))
.Aggregate <ObjectB, JSONSerDes <ObjectB> >(
() => new ObjectB(),
(key, ObjectA, ObjectB) => ObjectBHelper.CreateObjectB(key, ObjectA, ObjectB))
.ToStream()
.Map((key, ObjectB) => new KeyValuePair <string, ObjectB>(key.Key, ObjectB))
.To <StringSerDes, JSONSerDes <ObjectB> >("sink");
}
public async Task GetWindowStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
}
stream.Dispose();
}
public void GetWStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())));
state = true;
}
}
};
stream.Start(source.Token);
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
source.Cancel();
stream.Close();
}
public void WithNullMaterialize()
{
// CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
var config = new StreamConfig <StringSerDes, StringSerDes>();
var serdes = new StringSerDes();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, int, IWindowStore <Bytes, byte[]> > m = null;
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
m);
var topology = builder.Build();
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
null);
task.GroupMetadata = consumer as SyncConsumer;
Assert.Throws <StreamsException>(() => task.InitializeStateStores());
}
public void WithNullMaterialize()
{
// CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore
var config = new StreamConfig();
var serdes = new StringSerDes();
config.ApplicationId = "test-window-reduce";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2);
var topology = builder.Build();
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
null,
new MockChangelogRegister(),
new StreamMetricsRegistry());
task.GroupMetadata = consumer as SyncConsumer;
Assert.Throws <StreamsException>(() => task.InitializeStateStores());
}
public void TimeWindowingAggWithMaterialize()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To <StringTimeWindowedSerDes, Int32SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int32SerDes());
input.PipeInput("test", "1");
input.PipeInput("test", "230");
input.PipeInput("test", "32");
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(3, elements.Count);
Assert.AreEqual("test", elements[0].Message.Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
Assert.AreEqual(1, elements[0].Message.Value);
Assert.AreEqual("test", elements[1].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
Assert.AreEqual(3, elements[1].Message.Value);
Assert.AreEqual("test", elements[2].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
Assert.AreEqual(3, elements[2].Message.Value);
}
}
public void WindowStateStoreNotExist()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Count(InMemoryWindows <string, long> .As("store"));
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store2");
Assert.IsNull(store);
driver.Dispose();
}
public void WithNullValueSerDes()
{
// WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, long, WindowStore <Bytes, byte[]> > m =
Materialized <string, long, WindowStore <Bytes, byte[]> >
.Create("count-store")
.With(null, null);
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(m)
.ToStream()
.To <StringTimeWindowedSerDes, Int64SerDes>("output-topic");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
input.PipeInput("test-test", "30");
var records = output.ReadKeyValueList().ToList();
Assert.AreEqual(2, records.Count);
Assert.AreEqual("test", records[0].Message.Key.Key);
Assert.AreEqual(1, records[0].Message.Value);
Assert.AreEqual("test-test", records[1].Message.Key.Key);
Assert.AreEqual(1, records[1].Message.Value);
Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window);
}
}
public void TimeWindowingCountWithMaterialize()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
.ToStream()
.To <StringTimeWindowedSerDes, Int64SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
input.PipeInput("test", "2");
input.PipeInput("test", "3");
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(3, elements.Count);
Assert.AreEqual("test", elements[0].Message.Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
Assert.AreEqual(1, elements[0].Message.Value);
Assert.AreEqual("test", elements[1].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
Assert.AreEqual(2, elements[1].Message.Value);
Assert.AreEqual("test", elements[2].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
Assert.AreEqual(3, elements[2].Message.Value);
}
}
public void TimeWindowingReduce()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2)
.ToStream()
.To <StringTimeWindowedSerDes, StringSerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new StringSerDes());
input.PipeInput("test", "1");
input.PipeInput("test", "230");
input.PipeInput("test", "32");
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(3, elements.Count);
Assert.AreEqual("test", elements[0].Message.Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
Assert.AreEqual("1", elements[0].Message.Value);
Assert.AreEqual("test", elements[1].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
Assert.AreEqual("230", elements[1].Message.Value);
Assert.AreEqual("test", elements[2].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
Assert.AreEqual("230", elements[2].Message.Value);
}
}
public void StoresPersistentKeyValueStoreTest()
{
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(1000))
.Count(
Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create(
Streamiz.Kafka.Net.State.Stores.PersistentWindowStore(
"rocksdb-w-store",
TimeSpan.FromDays(1),
TimeSpan.FromSeconds(1))));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-rocksdb-window-store";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("abc", "1", dt);
inputTopic.PipeInput("key1", "1", dt);
inputTopic.PipeInput("test", "1", dt);
var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
Assert.IsNotNull(store);
var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();
Assert.AreEqual(3, k1.Count);
}
config.RemoveRocksDbFolderForTest();
}
static async System.Threading.Tasks.Task Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9093";
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("evenements")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Aggregate(() => "", (k, v, va) => va += v)
.ToStream()
.Print(Printed <Windowed <String>, String> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
Console.CancelKeyPress += (o, e) => stream.Dispose();
await stream.StartAsync();
}
public async Task GetWindowElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
dt = DateTime.Now;
producer.Produce("test",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext()),
Timestamp = new Confluent.Kafka.Timestamp(dt)
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
var @enum = store.All();
Assert.AreEqual(1, store.All().ToList().Count);
var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1));
Assert.IsNotNull(item);
Assert.IsTrue(item.MoveNext());
Assert.IsTrue(item.Current.HasValue);
Assert.AreEqual(1, item.Current.Value.Value);
item.Dispose();
}
stream.Dispose();
}