public void WithNullSerDes()
{
// WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-agg";
var builder = new StreamBuilder();
Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m =
Materialized <string, long, IKeyValueStore <Bytes, byte[]> >
.Create("agg-store")
.With(null, null);
builder
.Stream <string, string>("topic")
.GroupByKey()
.Aggregate(() => 0, (k, v, agg) => agg + 1, m)
.ToStream()
.To("output-topic");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TestWithKeySerdes()
{
var m = Materialized <string, string, IStateStore> .Create("test").WithKeySerdes(new StringSerDes());
Assert.IsNotNull(m.KeySerdes);
Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes);
}
public void TestWithValueSerdes2()
{
var m = Materialized <string, string, IStateStore> .Create("test").WithValueSerdes <StringSerDes>();
Assert.IsNotNull(m.ValueSerdes);
Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes);
}
public IKTable <Windowed <K>, VR> Aggregate <VR>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator, Materialized <K, VR, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(initializer, "initializer");
CheckIfParamNull(aggregator, "aggregator");
materialized = materialized ?? Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.AGGREGATE_NAME);
materialized.UseProvider(builder, KGroupedStream.AGGREGATE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, VR, W>(
windowOptions,
materialized.StoreName,
initializer,
aggregator);
ISerDes <Windowed <K> > windowSerdes = materialized.KeySerdes != null ? new TimeWindowedSerDes <K>(materialized.KeySerdes, windowOptions.Size) : null;
return(aggBuilder.BuildWindow(name,
new TimestampedWindowStoreMaterializer <K, VR, W>(windowOptions, materialized).Materialize(),
aggSupplier,
materialized.QueryableStoreName,
windowSerdes,
materialized.ValueSerdes));
}
public void StoresPersistentKeyValueStoreTest()
{
var builder = new StreamBuilder();
// Same like that :
// builder.Table("table-topic", RocksDb<string, string>.As("table-topic-store"));
builder.Table("table-topic",
Materialized <string, string, IKeyValueStore <Bytes, byte[]> > .Create(
Streamiz.Kafka.Net.State.Stores.PersistentKeyValueStore("table-topic-store")));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var resultK1 = store.Get("key1");
var resultK2 = store.Get("key2");
Assert.AreEqual("1", resultK1);
Assert.AreEqual("2", resultK2);
}
config.RemoveRocksDbFolderForTest();
}
public void TimeWindowingCountKeySerdesUnknow()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
.ToStream()
.To("output");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void WithNullValueSerDes()
{
// WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, string, IWindowStore <Bytes, byte[]> > m =
Materialized <string, string, IWindowStore <Bytes, byte[]> >
.Create("store")
.With(null, null);
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(2000))
.Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2, m)
.ToStream()
.To("output-topic");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
}
});
}
public void WithNullSerDes()
{
// WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
Materialized <string, int, IKeyValueStore <Bytes, byte[]> > m =
Materialized <string, int, IKeyValueStore <Bytes, byte[]> >
.Create("reduce-store")
.With(null, null);
builder
.Table <string, string>("topic")
.MapValues((v) => v.Length)
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Reduce((v1, v2) => Math.Max(v1, v2), (v1, v2) => v2, m)
.ToStream()
.To("output-topic");
var topology = builder.Build();
Assert.Throws <StreamsException>(() =>
{
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
input.PipeInput("test", "1");
}
});
}
public void TestCreateKeyValueBytesStoreSupplier()
{
var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> >
.Create(new InMemoryKeyValueBytesStoreSupplier("name"));
Assert.IsNotNull(m);
Assert.IsNotNull(m.StoreSupplier);
}
public void TestStoreName1()
{
var m = Materialized <string, string, IStateStore>
.Create("test");
Assert.IsNotNull(m);
Assert.AreEqual("test", m.StoreName);
Assert.IsNull(m.QueryableStoreName);
}
public void TestWithCachingDisabled()
{
var m = Materialized <string, string, IStateStore>
.Create("test")
.WithCachingDisabled();
Assert.IsNotNull(m);
Assert.IsFalse(m.CachingEnabled);
}
public void TestWithRetention()
{
var m = Materialized <string, string, IStateStore>
.Create("test")
.WithRetention(TimeSpan.FromSeconds(10));
Assert.IsNotNull(m);
Assert.AreEqual(TimeSpan.FromSeconds(10), m.Retention);
}
public void TestStoreName3()
{
var m = Materialized <string, string, IStateStore>
.Create()
.UseProvider(null, "");
Assert.IsNotNull(m);
Assert.AreEqual(string.Empty, m.StoreName);
Assert.IsNull(m.QueryableStoreName);
}
/// <summary>
/// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic.
/// Input keyvalue records with <code>null</code> key will be dropped.
/// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// However, no internal changelog topic is created since the original input topic can be used for recovery.
/// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code>
/// regardless of the specified value in <see cref="IStreamConfig"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-")?.InitConsumed(consumedInternal);
return(internalStreamBuilder.GlobalTable(topic, consumedInternal, materialized));
}
public void TestStoreName5()
{
var m = Materialized <string, string, IStateStore>
.Create("store")
.UseProvider(new MyNameProvider(), "pref");
Assert.IsNotNull(m);
Assert.AreEqual("store", m.StoreName);
Assert.AreEqual("store", m.QueryableStoreName);
}
public void TestCreateWithSerdes()
{
var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> >
.Create <StringSerDes, StringSerDes>();
Assert.IsNotNull(m);
Assert.IsNotNull(m.KeySerdes);
Assert.IsNotNull(m.ValueSerdes);
Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes);
Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes);
}
public void TestCreateKeyValueBytesStoreSupplierWithSerdes()
{
var m = Materialized <string, string, IKeyValueStore <Bytes, byte[]> >
.Create <StringSerDes, StringSerDes>(new InMemoryKeyValueBytesStoreSupplier("name"));
Assert.IsNotNull(m);
Assert.IsNotNull(m.StoreSupplier);
Assert.IsNotNull(m.KeySerdes);
Assert.IsNotNull(m.ValueSerdes);
Assert.IsAssignableFrom <StringSerDes>(m.KeySerdes);
Assert.IsAssignableFrom <StringSerDes>(m.ValueSerdes);
}
public void TestWithLoggingEnabled()
{
var topicConfig = new Dictionary <string, string>
{
{ "test", "test" }
};
var m = Materialized <string, string, IStateStore>
.Create("test")
.WithLoggingEnabled(topicConfig);
Assert.IsNotNull(m);
Assert.IsTrue(m.LoggingEnabled);
Assert.AreEqual(topicConfig, m.TopicConfig);
}
/// <summary>
/// Create a <see cref="IKTable{K, V}"/> for the specified topic.
/// Input keyvalue records with null key will be dropped.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted.
///
/// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKTable <K, V> Table <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KTable must not be null or empty");
}
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal);
return(internalStreamBuilder.Table(topic, consumedInternal, materialized));
}
public void WithNullSerDes()
{
// WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-count";
var builder = new StreamBuilder();
Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m =
Materialized <string, long, IKeyValueStore <Bytes, byte[]> >
.Create("count-store")
.With(null, null);
builder
.Table <string, string>("topic")
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Count(m)
.ToStream()
.To("output-topic");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic <string, long, StringSerDes, Int64SerDes>("output-topic");
input.PipeInput("test", "1");
input.PipeInput("test", "30");
IEnumerable <KeyValuePair <string, long> > expected = new List <KeyValuePair <string, long> > {
KeyValuePair.Create("TEST", 1L),
KeyValuePair.Create("TEST", 0L),
KeyValuePair.Create("TEST", 1L)
};
var records = output.ReadKeyValueList().Select(r => KeyValuePair.Create(r.Message.Key, r.Message.Value)).ToList();
Assert.AreEqual(expected, records);
}
}
public void WithNullReducer()
{
// WITH NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-reduce";
var builder = new StreamBuilder();
Materialized <string, int, IKeyValueStore <Bytes, byte[]> > m =
Materialized <string, int, IKeyValueStore <Bytes, byte[]> >
.Create("reduce-store")
.With(null, null);
Assert.Throws <ArgumentNullException>(() =>
{
builder
.Table <string, string>("topic")
.MapValues((v) => v.Length)
.GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v))
.Reduce((Reducer <int>)null, (Reducer <int>)null, m);
});
}
public void WithNullValueSerDes()
{
// WITH VALUE NULL SERDES, in running KeySerdes must be StringSerdes, and ValueSerdes Int64SerDes
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-count";
var builder = new StreamBuilder();
Materialized <string, long, WindowStore <Bytes, byte[]> > m =
Materialized <string, long, WindowStore <Bytes, byte[]> >
.Create("count-store")
.With(null, null);
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(5)))
.Count(m)
.ToStream()
.To <StringTimeWindowedSerDes, Int64SerDes>("output-topic");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output-topic", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
input.PipeInput("test-test", "30");
var records = output.ReadKeyValueList().ToList();
Assert.AreEqual(2, records.Count);
Assert.AreEqual("test", records[0].Message.Key.Key);
Assert.AreEqual(1, records[0].Message.Value);
Assert.AreEqual("test-test", records[1].Message.Key.Key);
Assert.AreEqual(1, records[1].Message.Value);
Assert.AreEqual(records[0].Message.Key.Window, records[1].Message.Key.Window);
}
}
public void TimeWindowingCountWithMaterialize()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-window-stream";
var builder = new StreamBuilder();
builder
.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromSeconds(10)))
.Count(Materialized <string, long, WindowStore <Bytes, byte[]> > .Create("count-store"))
.ToStream()
.To <StringTimeWindowedSerDes, Int64SerDes>("output");
var topology = builder.Build();
using (var driver = new TopologyTestDriver(topology, config))
{
var input = driver.CreateInputTopic <string, string>("topic");
var output = driver.CreateOuputTopic("output", TimeSpan.FromSeconds(1), new StringTimeWindowedSerDes(), new Int64SerDes());
input.PipeInput("test", "1");
input.PipeInput("test", "2");
input.PipeInput("test", "3");
var elements = output.ReadKeyValueList().ToList();
Assert.AreEqual(3, elements.Count);
Assert.AreEqual("test", elements[0].Message.Key.Key);
Assert.AreEqual((long)TimeSpan.FromSeconds(10).TotalMilliseconds, elements[0].Message.Key.Window.EndMs - elements[0].Message.Key.Window.StartMs);
Assert.AreEqual(1, elements[0].Message.Value);
Assert.AreEqual("test", elements[1].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[1].Message.Key.Window);
Assert.AreEqual(2, elements[1].Message.Value);
Assert.AreEqual("test", elements[2].Message.Key.Key);
Assert.AreEqual(elements[0].Message.Key.Window, elements[2].Message.Key.Window);
Assert.AreEqual(3, elements[2].Message.Value);
}
}
public void StoresPersistentKeyValueStoreTest()
{
var builder = new StreamBuilder();
builder.Stream <string, string>("topic")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(1000))
.Count(
Materialized <string, long, IWindowStore <Bytes, byte[]> > .Create(
Streamiz.Kafka.Net.State.Stores.PersistentWindowStore(
"rocksdb-w-store",
TimeSpan.FromDays(1),
TimeSpan.FromSeconds(1))));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-rocksdb-window-store";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
DateTime dt = DateTime.Now;
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("abc", "1", dt);
inputTopic.PipeInput("key1", "1", dt);
inputTopic.PipeInput("test", "1", dt);
var store = driver.GetWindowStore <string, long>("rocksdb-w-store");
Assert.IsNotNull(store);
var k1 = store.FetchAll(dt.AddMinutes(-10), dt.AddMinutes(10)).ToList();
Assert.AreEqual(3, k1.Count);
}
config.RemoveRocksDbFolderForTest();
}
public IKTable <Windowed <K>, V> Reduce(Reducer <V> reducer, Materialized <K, V, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(reducer, "reducer");
materialized = materialized ?? Materialized <K, V, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
if (materialized.ValueSerdes == null)
{
materialized.WithValueSerdes(ValueSerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.REDUCE_NAME);
materialized.UseProvider(builder, KGroupedStream.REDUCE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, V, W>(
windowOptions,
materialized.StoreName,
() => default,
public IKTable <Windowed <K>, long> Count(string named) => Count(Materialized <K, long, WindowStore <Bytes, byte[]> > .Create(), named);
public IKTable <Windowed <K>, long> Count(Materialized <K, long, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
materialized = materialized ?? Materialized <K, long, WindowStore <Bytes, byte[]> > .Create();
return(DoCount(materialized, named));
}
public IKTable <Windowed <K>, VR> Aggregate <VR, VRS>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator) where VRS : ISerDes <VR>, new()
{
var materialized = Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create().WithValueSerdes(new VRS()).WithKeySerdes(KeySerdes);
return(Aggregate(initializer, aggregator, materialized));
}
