public void WindowStoreEnumeratorTestReset()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
int i = 0;
while (enumerator.MoveNext())
{
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
++i;
}
Assert.AreEqual(1, i);
enumerator.Reset();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
}
public void Setup()
{
inmemorystore = new InMemoryWindowStore("store", TimeSpan.FromMinutes(20), 1000 * 2);
store = new MeteredTimestampedWindowStore <string, int>(inmemorystore, 1000 * 2, new StringSerDes(),
new ValueAndTimestampSerDes <int>(new Int32SerDes()), "in-memory-window");
facade = new ReadOnlyWindowStoreFacade <string, int>(store);
}
public void FetchKeyDoesNotExist()
{
var date = DateTime.Now;
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
Assert.IsNull(store.Fetch(new Bytes(new byte[0]), 100));
}
public void FetchAllTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now;
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2));
Assert.IsNotNull(result);
var items = result.ToList();
Assert.AreEqual(2, items.Count);
var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test"));
var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2"));
Assert.IsNotNull(c1);
Assert.IsNotNull(c2);
Assert.AreEqual("coucou1", c1.Value);
Assert.AreEqual("coucou2", c2.Value);
}
public void PutTwoElementDifferentKeyDifferentTime()
{
var date = DateTime.Now;
var dt2 = date.AddSeconds(1);
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
store.Put(key2, BitConverter.GetBytes(300), dt2.GetMilliseconds());
var r = store.Fetch(key, date.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(100), r);
r = store.Fetch(key, dt2.GetMilliseconds());
Assert.IsNull(r);
r = store.Fetch(key2, dt2.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(300), r);
r = store.Fetch(key2, date.GetMilliseconds());
Assert.IsNull(r);
}
public void CreateInMemoryWindowStore()
{
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
Assert.IsFalse(store.Persistent);
Assert.AreEqual("store", store.Name);
Assert.AreEqual(0, store.All().ToList().Count);
}
public void Setup()
{
inmemorystore = new InMemoryWindowStore("store", TimeSpan.FromMinutes(20), 1000 * 2);
wrapped = new MeteredWindowStore <string, int>(
inmemorystore, 1000 * 2,
new StringSerDes(),
new Int32SerDes(),
"in-memory-window");
}
public void TestRetention()
{
var date = DateTime.Now.AddDays(-1);
var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds);
store.Put(new Bytes(new byte[1] {
13
}), new byte[0], date.GetMilliseconds());
Assert.AreEqual(0, store.All().ToList().Count);
}
public void FetchRangeDoesNotExist()
{
var date = DateTime.Now;
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
var it = store.FetchAll(date.AddDays(-1), date.AddDays(1));
Assert.AreEqual(null, it.Current);
Assert.IsFalse(it.MoveNext());
Assert.AreEqual(null, it.Current);
}
public void EmptyWindowStoreIteratorTest()
{
var dt = DateTime.Now;
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(1), (long)defaultSize.TotalMilliseconds);
var enumerator = store.Fetch(new Bytes(null), dt.AddDays(1), dt);
Assert.IsAssignableFrom <EmptyWindowStoreEnumerator <byte[]> >(enumerator);
Assert.IsFalse(enumerator.MoveNext());
enumerator.Reset();
Assert.AreEqual(0, enumerator.ToList().Count);
}
public void PutElementsWithNullValue()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, null, date.GetMilliseconds());
var r = store.All().ToList();
Assert.AreEqual(0, r.Count);
}
public void PutOneElement()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
var r = store.Fetch(key, date.GetMilliseconds());
Assert.IsNotNull(r);
Assert.AreEqual(BitConverter.GetBytes(100), r);
}
public void WindowStoreEnumeratorTestDispose()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
enumerator.Dispose();
Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext());
}
public void WindowStoreEnumeratorWithSerdes()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
var items = enumerator.ToList();
Assert.AreEqual(1, items.Count);
Assert.AreEqual("value", items[0].Value);
Assert.AreEqual(date.GetMilliseconds(), items[0].Key);
}
public void PutElementsAndUpdateNullValueDifferentWindow()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var value = Encoding.UTF8.GetBytes("test");
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, value, date.GetMilliseconds());
store.Put(key, null, date.AddSeconds(1).GetMilliseconds());
var r = store.All().ToList();
Assert.AreEqual(1, r.Count);
Assert.AreEqual(value, store.Fetch(key, date.GetMilliseconds()));
Assert.IsNull(store.Fetch(key, date.AddSeconds(1).GetMilliseconds()));
}
public void FetchTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt);
var result = composite.Fetch("test", dt);
Assert.IsNotNull(result);
Assert.AreEqual("coucou1", result);
}
public void TestRetentionWithOpenIt()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var value = Encoding.UTF8.GetBytes("test");
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(1), (long)defaultSize.TotalMilliseconds);
store.Put(key, value, date.GetMilliseconds());
var it = store.All();
it.MoveNext();
Thread.Sleep(2000);
store.Put(key, value, date.AddSeconds(4).GetMilliseconds());
var r = it.ToList().Count;
Assert.AreEqual(0, r);
}
public void WindowStoreEnumeratorTestDispose()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
enumerator.Dispose();
Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext());
}
public void Begin()
{
config = new StreamConfig();
config.ApplicationId = $"unit-test-changelogging-w";
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
kafkaSupplier = new SyncKafkaSupplier();
var producerConfig = new ProducerConfig();
producerConfig.ClientId = "producer-1";
var producerClient = kafkaSupplier.GetProducer(producerConfig);
recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG));
recordCollector.Init(ref producerClient);
var changelogsTopics = new Dictionary <string, string> {
{ "test-store", "test-store-changelog" }
};
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
changelogsTopics,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
context.UseRecordCollector(recordCollector);
var inmemorystore = new InMemoryWindowStore("test-store", TimeSpan.FromDays(1), TimeSpan.FromSeconds(1).Milliseconds);
store = new ChangeLoggingWindowBytesStore(inmemorystore);
store.Init(context, store);
}
public void PutElementsAndFetch()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("test-key"));
var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key"));
var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds);
store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds());
store.Put(key2, BitConverter.GetBytes(300), date.AddSeconds(1).GetMilliseconds());
var r = store.FetchAll(date.AddSeconds(-10), date.AddSeconds(20)).ToList();
Assert.AreEqual(2, r.Count);
Assert.AreEqual(key, r[0].Key.Key);
Assert.AreEqual(BitConverter.GetBytes(100), r[0].Value);
Assert.AreEqual(defaultSize, r[0].Key.Window.TotalTime);
Assert.AreEqual(key2, r[1].Key.Key);
Assert.AreEqual(BitConverter.GetBytes(300), r[1].Value);
Assert.AreEqual(defaultSize, r[1].Key.Window.TotalTime);
}
public void WindowStoreEnumeratorTestNext()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes());
int i = 0;
while (enumerator.MoveNext())
{
Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key);
Assert.AreEqual("value", enumerator.Current.Value.Value);
++i;
}
Assert.AreEqual(1, i);
}
public void WindowStoreEnumeratorWithSerdes()
{
var date = DateTime.Now;
var key = new Bytes(Encoding.UTF8.GetBytes("key"));
var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10),
(long)TimeSpan.FromSeconds(1).TotalMilliseconds);
store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds());
var enumerator = new MeteredWindowEnumerator <string>(
store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)),
(b) => (new StringSerDes()).Deserialize(b, new SerializationContext()),
new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO));
var items = enumerator.ToList();
Assert.AreEqual(1, items.Count);
Assert.AreEqual("value", items[0].Value);
Assert.AreEqual(date.GetMilliseconds(), items[0].Key);
}
public void TestRetention()
{
var metricsRegistry = new StreamMetricsRegistry();
var mockContext = new Moq.Mock <ProcessorContext>();
mockContext.Setup(c => c.Id).Returns(new TaskId {
Id = 0, Partition = 0
});
mockContext.Setup(c => c.Metrics).Returns(metricsRegistry);
mockContext.Setup(c => c.Timestamp).Returns(DateTime.Now.GetMilliseconds());
var date = DateTime.Now.AddDays(-1);
var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds);
store.Init(mockContext.Object, null);
store.Put(new Bytes(new byte[1] {
13
}), new byte[0], date.GetMilliseconds());
Assert.AreEqual(0, store.All().ToList().Count);
}
