public void WindowStoreEnumeratorTestReset() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new MeteredWindowEnumerator <string>( store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), (b) => (new StringSerDes()).Deserialize(b, new SerializationContext()), new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO)); int i = 0; while (enumerator.MoveNext()) { Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key); Assert.AreEqual("value", enumerator.Current.Value.Value); ++i; } Assert.AreEqual(1, i); enumerator.Reset(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key); Assert.AreEqual("value", enumerator.Current.Value.Value); }
public void Setup() { inmemorystore = new InMemoryWindowStore("store", TimeSpan.FromMinutes(20), 1000 * 2); store = new MeteredTimestampedWindowStore <string, int>(inmemorystore, 1000 * 2, new StringSerDes(), new ValueAndTimestampSerDes <int>(new Int32SerDes()), "in-memory-window"); facade = new ReadOnlyWindowStoreFacade <string, int>(store); }
public void FetchKeyDoesNotExist() { var date = DateTime.Now; var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); Assert.IsNull(store.Fetch(new Bytes(new byte[0]), 100)); }
public void FetchAllTest() { InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); var dt = DateTime.Now; var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds()); store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds()); var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2)); Assert.IsNotNull(result); var items = result.ToList(); Assert.AreEqual(2, items.Count); var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test")); var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2")); Assert.IsNotNull(c1); Assert.IsNotNull(c2); Assert.AreEqual("coucou1", c1.Value); Assert.AreEqual("coucou2", c2.Value); }
public void PutTwoElementDifferentKeyDifferentTime() { var date = DateTime.Now; var dt2 = date.AddSeconds(1); var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key")); var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds()); store.Put(key2, BitConverter.GetBytes(300), dt2.GetMilliseconds()); var r = store.Fetch(key, date.GetMilliseconds()); Assert.IsNotNull(r); Assert.AreEqual(BitConverter.GetBytes(100), r); r = store.Fetch(key, dt2.GetMilliseconds()); Assert.IsNull(r); r = store.Fetch(key2, dt2.GetMilliseconds()); Assert.IsNotNull(r); Assert.AreEqual(BitConverter.GetBytes(300), r); r = store.Fetch(key2, date.GetMilliseconds()); Assert.IsNull(r); }
public void CreateInMemoryWindowStore() { var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); Assert.IsFalse(store.Persistent); Assert.AreEqual("store", store.Name); Assert.AreEqual(0, store.All().ToList().Count); }
public void Setup() { inmemorystore = new InMemoryWindowStore("store", TimeSpan.FromMinutes(20), 1000 * 2); wrapped = new MeteredWindowStore <string, int>( inmemorystore, 1000 * 2, new StringSerDes(), new Int32SerDes(), "in-memory-window"); }
public void TestRetention() { var date = DateTime.Now.AddDays(-1); var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds); store.Put(new Bytes(new byte[1] { 13 }), new byte[0], date.GetMilliseconds()); Assert.AreEqual(0, store.All().ToList().Count); }
public void FetchRangeDoesNotExist() { var date = DateTime.Now; var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); var it = store.FetchAll(date.AddDays(-1), date.AddDays(1)); Assert.AreEqual(null, it.Current); Assert.IsFalse(it.MoveNext()); Assert.AreEqual(null, it.Current); }
public void EmptyWindowStoreIteratorTest() { var dt = DateTime.Now; var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(1), (long)defaultSize.TotalMilliseconds); var enumerator = store.Fetch(new Bytes(null), dt.AddDays(1), dt); Assert.IsAssignableFrom <EmptyWindowStoreEnumerator <byte[]> >(enumerator); Assert.IsFalse(enumerator.MoveNext()); enumerator.Reset(); Assert.AreEqual(0, enumerator.ToList().Count); }
public void PutElementsWithNullValue() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); store.Put(key, null, date.GetMilliseconds()); var r = store.All().ToList(); Assert.AreEqual(0, r.Count); }
public void PutOneElement() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds()); var r = store.Fetch(key, date.GetMilliseconds()); Assert.IsNotNull(r); Assert.AreEqual(BitConverter.GetBytes(100), r); }
public void WindowStoreEnumeratorTestDispose() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes()); enumerator.Dispose(); Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext()); }
public void WindowStoreEnumeratorWithSerdes() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes()); var items = enumerator.ToList(); Assert.AreEqual(1, items.Count); Assert.AreEqual("value", items[0].Value); Assert.AreEqual(date.GetMilliseconds(), items[0].Key); }
public void PutElementsAndUpdateNullValueDifferentWindow() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var value = Encoding.UTF8.GetBytes("test"); var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); store.Put(key, value, date.GetMilliseconds()); store.Put(key, null, date.AddSeconds(1).GetMilliseconds()); var r = store.All().ToList(); Assert.AreEqual(1, r.Count); Assert.AreEqual(value, store.Fetch(key, date.GetMilliseconds())); Assert.IsNull(store.Fetch(key, date.AddSeconds(1).GetMilliseconds())); }
public void FetchTest() { InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt); var result = composite.Fetch("test", dt); Assert.IsNotNull(result); Assert.AreEqual("coucou1", result); }
public void TestRetentionWithOpenIt() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var value = Encoding.UTF8.GetBytes("test"); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(1), (long)defaultSize.TotalMilliseconds); store.Put(key, value, date.GetMilliseconds()); var it = store.All(); it.MoveNext(); Thread.Sleep(2000); store.Put(key, value, date.AddSeconds(4).GetMilliseconds()); var r = it.ToList().Count; Assert.AreEqual(0, r); }
public void WindowStoreEnumeratorTestDispose() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new MeteredWindowEnumerator <string>( store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), (b) => (new StringSerDes()).Deserialize(b, new SerializationContext()), new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO)); enumerator.Dispose(); Assert.Throws <ObjectDisposedException>(() => enumerator.MoveNext()); }
public void Begin() { config = new StreamConfig(); config.ApplicationId = $"unit-test-changelogging-w"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); kafkaSupplier = new SyncKafkaSupplier(); var producerConfig = new ProducerConfig(); producerConfig.ClientId = "producer-1"; var producerClient = kafkaSupplier.GetProducer(producerConfig); recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG)); recordCollector.Init(ref producerClient); var changelogsTopics = new Dictionary <string, string> { { "test-store", "test-store-changelog" } }; stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, changelogsTopics, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); context.UseRecordCollector(recordCollector); var inmemorystore = new InMemoryWindowStore("test-store", TimeSpan.FromDays(1), TimeSpan.FromSeconds(1).Milliseconds); store = new ChangeLoggingWindowBytesStore(inmemorystore); store.Init(context, store); }
public void PutElementsAndFetch() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("test-key")); var key2 = new Bytes(Encoding.UTF8.GetBytes("coucou-key")); var store = new InMemoryWindowStore("store", defaultRetention, (long)defaultSize.TotalMilliseconds); store.Put(key, BitConverter.GetBytes(100), date.GetMilliseconds()); store.Put(key2, BitConverter.GetBytes(300), date.AddSeconds(1).GetMilliseconds()); var r = store.FetchAll(date.AddSeconds(-10), date.AddSeconds(20)).ToList(); Assert.AreEqual(2, r.Count); Assert.AreEqual(key, r[0].Key.Key); Assert.AreEqual(BitConverter.GetBytes(100), r[0].Value); Assert.AreEqual(defaultSize, r[0].Key.Window.TotalTime); Assert.AreEqual(key2, r[1].Key.Key); Assert.AreEqual(BitConverter.GetBytes(300), r[1].Value); Assert.AreEqual(defaultSize, r[1].Key.Window.TotalTime); }
public void WindowStoreEnumeratorTestNext() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new WindowStoreEnumerator <string>(store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), new StringSerDes()); int i = 0; while (enumerator.MoveNext()) { Assert.AreEqual(date.GetMilliseconds(), enumerator.Current.Value.Key); Assert.AreEqual("value", enumerator.Current.Value.Value); ++i; } Assert.AreEqual(1, i); }
public void WindowStoreEnumeratorWithSerdes() { var date = DateTime.Now; var key = new Bytes(Encoding.UTF8.GetBytes("key")); var store = new InMemoryWindowStore("store", TimeSpan.FromSeconds(10), (long)TimeSpan.FromSeconds(1).TotalMilliseconds); store.Put(key, Encoding.UTF8.GetBytes("value"), date.GetMilliseconds()); var enumerator = new MeteredWindowEnumerator <string>( store.Fetch(key, date.AddSeconds(-1), date.AddSeconds(1)), (b) => (new StringSerDes()).Deserialize(b, new SerializationContext()), new NoRunnableSensor("s", "s", MetricsRecordingLevel.INFO)); var items = enumerator.ToList(); Assert.AreEqual(1, items.Count); Assert.AreEqual("value", items[0].Value); Assert.AreEqual(date.GetMilliseconds(), items[0].Key); }
public void TestRetention() { var metricsRegistry = new StreamMetricsRegistry(); var mockContext = new Moq.Mock <ProcessorContext>(); mockContext.Setup(c => c.Id).Returns(new TaskId { Id = 0, Partition = 0 }); mockContext.Setup(c => c.Metrics).Returns(metricsRegistry); mockContext.Setup(c => c.Timestamp).Returns(DateTime.Now.GetMilliseconds()); var date = DateTime.Now.AddDays(-1); var store = new InMemoryWindowStore("store", TimeSpan.Zero, (long)defaultSize.TotalMilliseconds); store.Init(mockContext.Object, null); store.Put(new Bytes(new byte[1] { 13 }), new byte[0], date.GetMilliseconds()); Assert.AreEqual(0, store.All().ToList().Count); }