public void WindowStateStoreExist()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Count(InMemoryWindows <string, long> .As("store"));
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
var input = driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store");
Assert.IsNotNull(store);
Assert.IsInstanceOf <TimestampedWindowStore <string, long> >(store);
DateTime dt = DateTime.Now;
input.PipeInput("coucou", "1");
Assert.AreEqual(1, ((TimestampedWindowStore <string, long>)store).All().ToList().Count);
var it = ((TimestampedWindowStore <string, long>)store).Fetch("coucou", dt.AddSeconds(-5), dt.AddSeconds(5));
Assert.IsTrue(it.MoveNext());
Assert.AreEqual(1, it.Current.Value.Value.Value);
driver.Dispose();
}
public void TimeWindowingAggNothing()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To <StringTimeWindowedSerDes, Int32SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(0, elements.Count);
}
}
public void TestGetWindowStoreIncorrectType()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(InMemoryWindows <string, long> .As("count-store"));
var topology = builder.Build();
using var driver = new TopologyTestDriver(topology, config);
DateTime dt = DateTime.Now;
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1", dt);
var store = driver.GetWindowStore <int, long>("count-store");
Assert.IsNull(store);
}
public void TimeWindowingQueryStoreAll()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To <StringTimeWindowedSerDes, StringSerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "2567");
input.PipeInput("test", "32");
var store = driver.GetWindowStore <string, string>("store");
var elements = store.All().ToList();
Assert.AreEqual(1, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual("2567", elements[0].Value);
}
}
public void TimeWindowingReduceKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingReduceNothing()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce(
(v1, v2) => v1.Length > v2.Length ? v1 : v2,
InMemoryWindows <string, string> .As("store"))
.ToStream()
.To <StringTimeWindowedSerDes, StringSerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(0, elements.Count);
}
}
private static void Main(string[] args)
{
var config = new StreamConfig <StringSerDes, StringSerDes>
{
ApplicationId = "test-test-driver-app"
};
StreamBuilder builder = new StreamBuilder();
var table = builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(InMemoryWindows <string, long> .As("count-store"));
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("test");
inputTopic.PipeInput("renault", "clio", dt);
inputTopic.PipeInput("renault", "megane", dt.AddMilliseconds(10));
Thread.Sleep((int)TimeSpan.FromSeconds(10).TotalMilliseconds);
inputTopic.PipeInput("renault", "scenic", dt.AddSeconds(1));
var store = driver.GetWindowStore <string, long>("count-store");
var elements = store.All().ToList();
}
}
public void TimeWindowingQueryStoreAll()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>());
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
input.PipeInput("test", "2567");
input.PipeInput("test", "32");
var store = driver.GetWindowStore <string, int>("store");
var elements = store.All().ToList();
Assert.AreEqual(1, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(2).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual(4, elements[0].Value);
}
}
public void TimeWindowingAggKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TimeWindowingQueryStore2Window()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(InMemoryWindows <string, long> .As("count-store"));
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
DateTime dt = DateTime.Now;
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1", dt);
input.PipeInput("test", "2", dt);
input.PipeInput("test", "3", dt.AddMinutes(1));
var store = driver.GetWindowStore <string, long>("count-store");
var elements = store.All().ToList();
Assert.AreEqual(2, elements.Count);
Assert.AreEqual("test", elements[0].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[0].Key.Window.EndMs - elements[0].Key.Window.StartMs);
Assert.AreEqual(2, elements[0].Value);
Assert.AreEqual("test", elements[1].Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(5).TotalMilliseconds, elements[1].Key.Window.EndMs - elements[1].Key.Window.StartMs);
Assert.AreEqual(1, elements[1].Value);
}
}
private static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"))
.ToStream()
.Map((k, v) => KeyValuePair.Create(k.Key, v.ToString()))
.To("output");
Topology t = builder.Build();
//bool taskStart = false;
KafkaStream stream = new KafkaStream(t, config);
// Subscribe state changed
//stream.StateChanged += (old, @new) =>
//{
// if (!taskStart && @new == KafkaStream.State.RUNNING) // If new state is running, we can quering state store.
// {
// Task.Factory.StartNew(() =>
// {
// while (!source.Token.IsCancellationRequested)
// {
// var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore<string, long>()));
// var items = store.All().ToList();
// Thread.Sleep(500);
// }
// }, source.Token);
// taskStart = true;
// }
//};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
/// <summary>
/// Materialize a <see cref="InMemoryWindowStore"/> with the given name.
/// </summary>
/// <typeparam name="KS">New serializer for <typeparamref name="K"/> type</typeparam>
/// <typeparam name="VS">New serializer for <typeparamref name="V"/> type</typeparam>
/// <param name="storeName">the name of the underlying <see cref="IKTable{K, V}"/> state store; valid characters are ASCII alphanumerics, '.', '_' and '-'.</param>
/// <param name="windowSize">the windows size aggregation</param>
/// <returns>a new <see cref="InMemoryWindows{K, V}"/> instance with the given storeName</returns>
public static InMemoryWindows <K, V> @As <KS, VS>(string storeName, TimeSpan?windowSize)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
{
var m = new InMemoryWindows <K, V>(storeName, new InMemoryWindowStoreSupplier(storeName, TimeSpan.FromDays(1), windowSize.HasValue ? (long)windowSize.Value.TotalMilliseconds : (long?)null))
{
KeySerdes = new KS(),
ValueSerdes = new VS()
};
return(m);
}
public async Task GetWindowStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
}
stream.Dispose();
}
public void GetWStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())));
state = true;
}
}
};
stream.Start(source.Token);
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
source.Cancel();
stream.Close();
}
public void TimeWindowingAggWithMaterialize()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Aggregate(
() => 0,
(k, v, agg) => Math.Max(v.Length, agg),
InMemoryWindows <string, int> .As("store").WithValueSerdes <Int32SerDes>())
.ToStream()
.To <StringTimeWindowedSerDes, Int32SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int32SerDes());
input.PipeInput("test", "1");
input.PipeInput("test", "230");
input.PipeInput("test", "32");
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(3, elements.Count);
Assert.AreEqual("test", elements[0].Message.Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
Assert.AreEqual(1, elements[0].Message.Value);
Assert.AreEqual("test", elements[1].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
Assert.AreEqual(3, elements[1].Message.Value);
Assert.AreEqual("test", elements[2].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
Assert.AreEqual(3, elements[2].Message.Value);
}
}
public void WindowStateStoreNotExist()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-config";
var builder = new StreamBuilder();
builder.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Count(InMemoryWindows <string, long> .As("store"));
var driver = new TaskSynchronousTopologyDriver("client", builder.Build().Builder, config, config, default);
driver.CreateInputTopic("test", new StringSerDes(), new StringSerDes());
var store = driver.GetStateStore <string, string>("store2");
Assert.IsNull(store);
driver.Dispose();
}
public async Task GetWindowElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
dt = DateTime.Now;
producer.Produce("test",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext()),
Timestamp = new Confluent.Kafka.Timestamp(dt)
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
var @enum = store.All();
Assert.AreEqual(1, store.All().ToList().Count);
var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1));
Assert.IsNotNull(item);
Assert.IsTrue(item.MoveNext());
Assert.IsTrue(item.Current.HasValue);
Assert.AreEqual(1, item.Current.Value.Value);
item.Dispose();
}
stream.Dispose();
}
