public void ShouldReturnTimestampedKeyValueStore()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores);
var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-kv-store",
QueryableStoreTypes.TimestampedKeyValueStore <object, object>()));
Assert.AreEqual(timestampedKVStore, result.Single());
}
public void ShouldReturnWindowStoreAsTimestampedWindowStore()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores);
var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-w-store",
QueryableStoreTypes.WindowStore <object, object>()));
Assert.AreEqual(1, result.Count());
}
public void ShouldNotReturnKeyValueStoreAsTimestampedStore()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores);
var result = provider.Stores(StoreQueryParameters.FromNameAndType("kv-store",
QueryableStoreTypes.TimestampedKeyValueStore <object, object>()));
Assert.AreEqual(0, result.Count());
}
public void ShouldReturnTimestampedKeyValueStoreAsKeyValueStore()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores);
var result = provider.Stores(StoreQueryParameters.FromNameAndType("ts-kv-store",
QueryableStoreTypes.KeyValueStore <object, object>()));
Assert.IsInstanceOf(typeof(IReadOnlyKeyValueStore <object, object>), result.Single());
Assert.IsNotInstanceOf(typeof(ITimestampedKeyValueStore <object, object>), result.Single());
}
public void ShouldReturnWindowStore()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(stores);
var result =
provider.Stores(StoreQueryParameters.FromNameAndType("ws-store",
QueryableStoreTypes.WindowStore <object, object>()));
Assert.AreEqual(wStore, result.Single());
}
public void ShouldReturnEmptyItemListIfStoreDoesntExist()
{
GlobalStateStoreProvider provider = new GlobalStateStoreProvider(new Dictionary <string, IStateStore>());
var result =
provider.Stores(StoreQueryParameters.FromNameAndType("test",
QueryableStoreTypes.KeyValueStore <object, object>()));
Assert.AreEqual(0, result.Count());
}
public async Task GetWindowElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
dt = DateTime.Now;
producer.Produce("test",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext()),
Timestamp = new Confluent.Kafka.Timestamp(dt)
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
var @enum = store.All();
Assert.AreEqual(1, store.All().ToList().Count);
var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1));
Assert.IsNotNull(item);
Assert.IsTrue(item.MoveNext());
Assert.IsTrue(item.Current.HasValue);
Assert.AreEqual(1, item.Current.Value.Value);
item.Dispose();
}
stream.Dispose();
}
public async Task GetWindowStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>()));
Assert.IsNotNull(store);
}
stream.Dispose();
}
public void GetStateStoreBeforeRunningState()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
Assert.Throws <IllegalStateException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())));
stream.Dispose();
}
public async Task GetRangeKVStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
producer.Produce("topic",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
var list = store.Range("key1", "key2").ToList();
Assert.AreEqual(1, list.Count);
var item = list[0];
Assert.IsNotNull(item);
Assert.AreEqual("coucou", item.Value);
Assert.AreEqual("key1", item.Key);
}
stream.Dispose();
}
public void GetWStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1)))
.Count(InMemoryWindows <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())));
state = true;
}
}
};
stream.Start(source.Token);
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
source.Cancel();
stream.Close();
}
public void GetElementInStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 10;
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var serdes = new StringSerDes();
producer.Produce("topic",
new Confluent.Kafka.Message <byte[], byte[]>
{
Key = serdes.Serialize("key1", new SerializationContext()),
Value = serdes.Serialize("coucou", new SerializationContext())
});
Thread.Sleep(50);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
var item = store.Get("key1");
Assert.IsNotNull(item);
Assert.AreEqual("coucou", item);
}
source.Cancel();
stream.Close();
}
static void Main(string[] args)
{
CancellationTokenSource source = new CancellationTokenSource();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "192.168.56.1:9092";
config.SaslMechanism = SaslMechanism.Plain;
config.SaslUsername = "******";
config.SaslPassword = "******";
config.SecurityProtocol = SecurityProtocol.SaslPlaintext;
config.AutoOffsetReset = AutoOffsetReset.Earliest;
config.NumStreamThreads = 1;
StreamBuilder builder = new StreamBuilder();
builder.Stream <string, string>("test")
.FilterNot((k, v) => v.Contains("test"))
.To("test-output");
builder.Table("test-ktable", InMemory <string, string> .As("test-store"));
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
stream.StateChanged += (old, @new) =>
{
if (@new == KafkaStream.State.RUNNING)
{
Task.Factory.StartNew(() =>
{
while (!source.Token.IsCancellationRequested)
{
var store = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.TimestampedKeyValueStore <string, string>()));
var items = store.All().ToList();
var store2 = stream.Store(StoreQueryParameters.FromNameAndType("test-store", QueryableStoreTypes.KeyValueStore <string, string>()));
var items2 = store2.All().ToList();
Thread.Sleep(500);
}
}, source.Token);
}
};
Console.CancelKeyPress += (o, e) =>
{
source.Cancel();
stream.Close();
};
stream.Start(source.Token);
}
public void ShouldThrowExceptionIfStoreIsNotOpen()
{
var mockStore = CreateMockStore <IKeyValueStore <object, object> >(isOpen: false);
GlobalStateStoreProvider provider =
new GlobalStateStoreProvider(new Dictionary <string, IStateStore> {
{ "test", mockStore }
});
Assert.Throws <InvalidStateStoreException>(() =>
provider.Stores(StoreQueryParameters.FromNameAndType("test",
QueryableStoreTypes.KeyValueStore <object, object>())));
}
public async Task GetKVStateStoreInvalidStateStoreException()
{
var timeout = TimeSpan.FromSeconds(10);
bool state = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder
.Stream <string, string>("test")
.GroupByKey()
.Count(InMemory <string, long> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, new SyncKafkaSupplier());
stream.StateChanged += (old, @new) =>
{
if ([email protected](KafkaStream.State.RUNNING))
{
if (!state)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, long>())));
state = true;
}
}
};
await stream.StartAsync();
while (!state)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(state);
stream.Dispose();
}
public async Task BuildGlobalStateStore()
{
var timeout = TimeSpan.FromSeconds(10);
bool isRunningState = false;
DateTime dt = DateTime.Now;
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
config.PollMs = 1;
var builder = new StreamBuilder();
builder.GlobalTable <string, string>("test", InMemory <string, string> .As("store"));
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(new ProducerConfig());
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
await stream.StartAsync();
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
var stringSerdes = new StringSerDes();
producer.Produce("test",
new Message <byte[], byte[]>
{
Key = stringSerdes.Serialize("key", new SerializationContext()),
Value = stringSerdes.Serialize("value", new SerializationContext())
});
Thread.Sleep(250);
var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>()));
Assert.IsNotNull(store);
Assert.AreEqual(1, store.ApproximateNumEntries());
}
stream.Dispose();
}
public void GetStateStoreDoesntExists()
{
var timeout = TimeSpan.FromSeconds(10);
var source = new CancellationTokenSource();
bool isRunningState = false;
DateTime dt = DateTime.Now;
var supplier = new SyncKafkaSupplier();
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test";
config.BootstrapServers = "127.0.0.1";
var builder = new StreamBuilder();
builder.Table("topic", InMemory <string, string> .As("store"));
var t = builder.Build();
var stream = new KafkaStream(t, config, supplier);
stream.StateChanged += (old, @new) =>
{
if (@new.Equals(KafkaStream.State.RUNNING))
{
isRunningState = true;
}
};
stream.Start(source.Token);
while (!isRunningState)
{
Thread.Sleep(250);
if (DateTime.Now > dt + timeout)
{
break;
}
}
Assert.IsTrue(isRunningState);
if (isRunningState)
{
Assert.Throws <InvalidStateStoreException>(() => stream.Store(StoreQueryParameters.FromNameAndType("stodfdsfdsfre", QueryableStoreTypes.KeyValueStore <string, string>())));
}
source.Cancel();
stream.Close();
}
