public void FetchAllTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now;
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2));
Assert.IsNotNull(result);
var items = result.ToList();
Assert.AreEqual(2, items.Count);
var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test"));
var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2"));
Assert.IsNotNull(c1);
Assert.IsNotNull(c2);
Assert.AreEqual("coucou1", c1.Value);
Assert.AreEqual("coucou2", c2.Value);
}
public void PutTwoElementDifferentKeyDifferentTime()
{
var date = DateTime.Now;
var dt2 = date.AddSeconds(1);
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
store.Put(key2, BitConverter.GetBytes(300), dt2.GetMilliseconds());
var r = store.Fetch(key, date.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(100), r);
r = store.Fetch(key, dt2.GetMilliseconds());
Assert.IsNull(r);
r = store.Fetch(key2, dt2.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(300), r);
r = store.Fetch(key2, date.GetMilliseconds());
Assert.IsNull(r);
}
public void PutElementsAndUpdateNullValueSameWindow()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var value = Encoding.UTF8.GetBytes("test");
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, value, date.GetMilliseconds());
store.Put(key, null, date.GetMilliseconds());
var r = store.All().ToList();
Assert.AreEqual(0, r.Count);
}
public void WindowStoreEnumeratorTestReset()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
int i = 0;
while (enumerator.MoveNext())
{
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
++i;
}
Assert.AreEqual(1, i);
enumerator.Reset();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
}
public void TestRetentionWithOpenIt()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var value = Encoding.UTF8.GetBytes("test");
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(1), (long)defaultSize.TotalMilliseconds);
store.Put(key, value, date.GetMilliseconds());
var it = store.All();
it.MoveNext();
Thread.Sleep(2000);
store.Put(key, value, date.AddSeconds(4).GetMilliseconds());
var r = it.ToList().Count;
Assert.AreEqual(0, r);
}
public void PutElementsAndFetch()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
store.Put(key2, BitConverter.GetBytes(300), date.AddSeconds(1).GetMilliseconds());
var r = store.FetchAll(date.AddSeconds(-10), date.AddSeconds(20)).ToList();
Assert.AreEqual(2, r.Count);
Assert.AreEqual(key, r[0].Key.Key);
Assert.AreEqual(BitConverter.GetBytes(100), r[0].Value);
Assert.AreEqual(defaultSize, r[0].Key.Window.TotalTime);
Assert.AreEqual(key2, r[1].Key.Key);
Assert.AreEqual(BitConverter.GetBytes(300), r[1].Value);
Assert.AreEqual(defaultSize, r[1].Key.Window.TotalTime);
}
public void TestRetention()
{
var date = DateTime.Now.AddDays(-1);
var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds);
store.Put(new Bytes(new byte[1] {
13
}), new byte[0], date.GetMilliseconds());
Assert.AreEqual(0, store.All().ToList().Count);
}
public void PutOneElement()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
var r = store.Fetch(key, date.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(100), r);
}
public void TestWithUnknwonSerdes2()
{
wrapped = new WrappedWindowStore <string, int>(inmemorystore, 1000 * 2, null, null);
var id = new TaskId {
Id = 0, Topic = "test", Partition = 0
};
var stateManager = new ProcessorStateManager(id, new Confluent.Kafka.TopicPartition("test", 0));
var context = new ProcessorContext(new StreamConfig(), stateManager);
wrapped.Init(context, inmemorystore);
inmemorystore.Put(new Bytes(Encoding.UTF8.GetBytes("test")), BitConverter.GetBytes(100), 300);
Assert.Throws <StreamsException>(() => wrapped.Fetch("test", 300));
}
public void WindowStoreEnumeratorTestDispose()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
enumerator.Dispose();
Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext());
}
public void WindowStoreEnumeratorWithSerdes()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
var items = enumerator.ToList();
Assert.AreEqual(1, items.Count);
Assert.AreEqual("value", items[0].Value);
Assert.AreEqual(date.GetMilliseconds(), items[0].Key);
}
public void FetchTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt);
var result = composite.Fetch("test", dt);
Assert.IsNotNull(result);
Assert.AreEqual("coucou1", result);
}
public void WindowStoreEnumeratorTestDispose()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
enumerator.Dispose();
Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext());
}
public void TestWithUnknwonSerdes2()
{
wrapped = new MeteredWindowStore <string, int>(inmemorystore, 1000 * 2, null, null, "in-memory-window");
var id = new TaskId {
Id = 0, Partition = 0
};
var stateManager = new ProcessorStateManager(id,
new List <Confluent.Kafka.TopicPartition> {
new Confluent.Kafka.TopicPartition("test", 0)
}, null, null,
null);
var context = new ProcessorContext(UnassignedStreamTask.Create(), config, stateManager,
new StreamMetricsRegistry());
wrapped.Init(context, inmemorystore);
inmemorystore.Put(new Bytes(Encoding.UTF8.GetBytes("test")), BitConverter.GetBytes(100), 300);
Assert.Throws <StreamsException>(() => wrapped.Fetch("test", 300));
}
public void WindowStoreEnumeratorWithSerdes()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
var items = enumerator.ToList();
Assert.AreEqual(1, items.Count);
Assert.AreEqual("value", items[0].Value);
Assert.AreEqual(date.GetMilliseconds(), items[0].Key);
}
public void WindowStoreEnumeratorTestNext()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
int i = 0;
while (enumerator.MoveNext())
{
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
++i;
}
Assert.AreEqual(1, i);
}
public void TestRetention()
{
var metricsRegistry = new StreamMetricsRegistry();
var mockContext = new Moq.Mock <ProcessorContext>();
mockContext.Setup(c => c.Id).Returns(new TaskId {
Id = 0, Partition = 0
});
mockContext.Setup(c => c.Metrics).Returns(metricsRegistry);
mockContext.Setup(c => c.Timestamp).Returns(DateTime.Now.GetMilliseconds());
var date = DateTime.Now.AddDays(-1);
var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds);
store.Init(mockContext.Object, null);
store.Put(new Bytes(new byte[1] {
13
}), new byte[0], date.GetMilliseconds());
Assert.AreEqual(0, store.All().ToList().Count);
}