public ValueAndTimestamp <KeyValuePair <K1, V1> > Get(K key) { ValueAndTimestamp <V> valueAndTimestamp = parentTableGetter.Get(key); var v = mapper.Apply(key, valueAndTimestamp != null ? valueAndTimestamp.Value : default); return(ValueAndTimestamp <KeyValuePair <K1, V1> > .Make(v, valueAndTimestamp == null?context.Timestamp : valueAndTimestamp.Timestamp)); }
public void TestFacadeFetchRangeKeyTwoElement() { DateTime dt = DateTime.Now; store.Put( "coucou", ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()), dt.GetMilliseconds()); store.Put( "coucou-bis", ValueAndTimestamp <int> .Make(500, dt.AddMilliseconds(400).GetMilliseconds()), dt.AddMilliseconds(100).GetMilliseconds()); var enumerator = facade.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(5)); Assert.IsNotNull(enumerator); var list = enumerator.ToList(); Assert.AreEqual(2, list.Count); Assert.AreEqual(120, list[0].Value); Assert.AreEqual("coucou", list[0].Key.Key); Assert.AreEqual(TimeSpan.FromSeconds(2), list[0].Key.Window.TotalTime); Assert.AreEqual(dt.GetMilliseconds(), list[0].Key.Window.StartMs); Assert.AreEqual(500, list[1].Value); Assert.AreEqual("coucou-bis", list[1].Key.Key); Assert.AreEqual(TimeSpan.FromSeconds(2), list[1].Key.Window.TotalTime); Assert.AreEqual(dt.AddMilliseconds(100).GetMilliseconds(), list[1].Key.Window.StartMs); }
public void FetchAllTest() { InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); var dt = DateTime.Now; var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds()); store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds()); var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2)); Assert.IsNotNull(result); var items = result.ToList(); Assert.AreEqual(2, items.Count); var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test")); var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2")); Assert.IsNotNull(c1); Assert.IsNotNull(c2); Assert.AreEqual("coucou1", c1.Value); Assert.AreEqual("coucou2", c2.Value); }
public override void Process(K key, V value) { // If the key or value is null we don't need to proceed if (key == null || value == null) { log.Warn($"Skipping record due to null key or value. key=[{key}] value=[{value}] topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]"); return; } ValueAndTimestamp <V> oldAggAndTimestamp = store.Get(key); V oldAgg, newAgg; long newTimestamp; if (oldAggAndTimestamp == null) { oldAgg = default; newAgg = value; newTimestamp = Context.Timestamp; } else { oldAgg = oldAggAndTimestamp.Value; newAgg = reducer.Apply(oldAgg, value); newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } store.Put(key, ValueAndTimestamp <V> .Make(newAgg, newTimestamp)); tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
public override void Process(K key, V value) { LogProcessingKeyValue(key, value); if (key == null) { log.Warn($"Skipping record due to null key. topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]"); return; } if (queryableName != null) { ValueAndTimestamp <V> oldValueAndTimestamp = store.Get(key); V oldValue; if (oldValueAndTimestamp != null) { oldValue = oldValueAndTimestamp.Value; if (Context.Timestamp < oldValueAndTimestamp.Timestamp) { log.Warn($"Detected out-of-order KTable update for {store.Name} at offset {Context.Offset}, partition {Context.Partition}."); } } else { oldValue = default(V); } store.Put(key, ValueAndTimestamp <V> .Make(value, Context.Timestamp)); tupleForwarder.MaybeForward(key, value, oldValue); } else { this.Forward <K, Change <V> >(key, new Change <V>(default(V), value)); } }
public override void Process(K key, V value) { if (key == null || value == null) { log.Warn($"Skipping record due to null key or value. key=[{key}] value=[{value}] topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]"); return; } ValueAndTimestamp <T> oldAggAndTimestamp = store.Get(key); T oldAgg, newAgg; long newTimestamp; if (oldAggAndTimestamp == null) { oldAgg = initializer.Apply(); newTimestamp = Context.Timestamp; } else { oldAgg = oldAggAndTimestamp.Value; newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } newAgg = aggregator.Apply(key, value, oldAgg); store.Put(key, ValueAndTimestamp <T> .Make(newAgg, newTimestamp)); tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
public void ResetTest() { InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store"); InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store"); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext())); store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext())); var enumerator = composite.Range("test", "test2"); Assert.IsNotNull(enumerator); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual("test", enumerator.PeekNextKey()); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual("test2", enumerator.PeekNextKey()); Assert.IsFalse(enumerator.MoveNext()); enumerator.Reset(); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual("test", enumerator.PeekNextKey()); Assert.IsTrue(enumerator.MoveNext()); Assert.AreEqual("test2", enumerator.PeekNextKey()); Assert.IsFalse(enumerator.MoveNext()); }
public void ReverseAllTest() { InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store"); InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store"); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext())); store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1bis", dt), new SerializationContext())); store2.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext())); store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2bis", dt), new SerializationContext())); var result = composite.ReverseAll().ToList(); Assert.IsNotNull(result); Assert.AreEqual(4, result.Count); Assert.AreEqual("test2", result[0].Key); Assert.AreEqual("coucou1bis", result[0].Value); Assert.AreEqual("test", result[1].Key); Assert.AreEqual("coucou1", result[1].Value); Assert.AreEqual("test2", result[2].Key); Assert.AreEqual("coucou2bis", result[2].Value); Assert.AreEqual("test", result[3].Key); Assert.AreEqual("coucou2", result[3].Value); }
public override void Process(K key, Change <V> value) { if (key == null) { throw new StreamsException($"Record key for KTable aggregate operator with state {queryableStoreName} should not be null."); } ValueAndTimestamp <T> oldAggAndTimestamp = store.Get(key); T oldAgg = oldAggAndTimestamp != null ? oldAggAndTimestamp.Value : default; T intermediateAgg; long newTimestamp = Context.Timestamp; // first try to remove the old value if (oldAggAndTimestamp != null && value.OldValue != null && oldAgg != null) { intermediateAgg = remove.Apply(key, value.OldValue, oldAgg); newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } else { intermediateAgg = oldAgg; } // then try to add the new value T newAgg; if (value.NewValue != null) { T initializedAgg; if (intermediateAgg == null) { initializedAgg = initializer.Apply(); } else { initializedAgg = intermediateAgg; } newAgg = add.Apply(key, value.NewValue, initializedAgg); if (oldAggAndTimestamp != null) { newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } } else { newAgg = intermediateAgg; } // update the store with the new value store.Put(key, ValueAndTimestamp <T> .Make(newAgg, newTimestamp)); tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
private ValueAndTimestamp <VR> ComputeValue(K key, ValueAndTimestamp <V> valueAndTimestamp) { VR newValue = default(VR); long timestamp = 0; if (valueAndTimestamp != null) { newValue = mapper.Apply(key, valueAndTimestamp.Value); timestamp = valueAndTimestamp.Timestamp; } return(ValueAndTimestamp <VR> .Make(newValue, timestamp)); }
public void TestFacadeFetchKeyByTime() { DateTime dt = DateTime.Now; store.Put( "coucou", ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()), dt.GetMilliseconds()); var r = facade.Fetch("coucou", dt.GetMilliseconds()); Assert.IsNotNull(r); Assert.AreEqual(120, r); }
public override void Process(K key, Change <V> value) { // the keys should never be null if (key == null) { throw new StreamsException($"Record key for KTable reduce operator with state {queryableStoreName} should not be null."); } ValueAndTimestamp <V> oldAggAndTimestamp = store.Get(key); V oldAgg = oldAggAndTimestamp != null ? oldAggAndTimestamp.Value : default; V intermediateAgg; long newTimestamp = Context.Timestamp; // first try to remove the old value if (oldAggAndTimestamp != null && value.OldValue != null && oldAgg != null) { intermediateAgg = substractor.Apply(oldAgg, value.OldValue); newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } else { intermediateAgg = oldAgg; } // then try to add the new value V newAgg; if (value.NewValue != null) { if (intermediateAgg == null) { newAgg = value.NewValue; } else { newAgg = adder.Apply(intermediateAgg, value.NewValue); } if (oldAggAndTimestamp != null) { newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } } else { newAgg = intermediateAgg; } // update the store with the new value store.Put(key, ValueAndTimestamp <V> .Make(newAgg, newTimestamp)); tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
public override ValueAndTimestamp <V> Deserialize(byte[] data) { using (var stream = new MemoryStream(data)) using (var reader = new BinaryReader(stream)) { long t = reader.ReadInt64(); int length = reader.ReadInt32(); byte[] d = reader.ReadBytes(length); V v = InnerSerdes.Deserialize(d); ValueAndTimestamp <V> obj = ValueAndTimestamp <V> .Make(v, t); return(obj); } }
public void CountTest() { InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store"); InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store"); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext())); store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext())); Assert.AreEqual(2, composite.ApproximateNumEntries()); }
public void DeserializeData() { long millie = DateTime.Now.Millisecond; string s = "coucou"; var stringSerdes = new StringSerDes(); var serdes = new ValueAndTimestampSerDes <string>(stringSerdes); var data = ValueAndTimestamp <string> .Make(s, millie); var r = serdes.Deserialize(serdes.Serialize(data)); Assert.IsNotNull(r); Assert.AreEqual(s, r.Value); Assert.AreEqual(millie, r.Timestamp); }
public override void Process(K key, Change <V> value) { if (!string.IsNullOrEmpty(queryableStoreName)) { store.Put(key, ValueAndTimestamp <V> .Make(value.NewValue, Context.Timestamp)); tupleForwarder.MaybeForward(key, value.NewValue, sendOldValues ? value.OldValue : default); } else { if (sendOldValues) { Forward(key, value); } else { Forward(key, new Change <V>(default, value.NewValue));
public void FetchTest() { InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt); var result = composite.Fetch("test", dt); Assert.IsNotNull(result); Assert.AreEqual("coucou1", result); }
public void GetTest() { InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store"); InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store"); var dt = DateTime.Now.GetMilliseconds(); var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes()); var bytes = new Bytes(Encoding.UTF8.GetBytes("test")); var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2); var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store"); store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt))); var result = composite.Get("test"); Assert.IsNotNull(result); Assert.AreEqual("coucou1", result); }
public override void Process(K key, Change <V> value) { LogProcessingKeyValue(key, value); VR newValue = ComputeValue(key, value.NewValue); VR oldValue = sendOldValues ? ComputeValue(key, value.OldValue) : default(VR); if (queryableStoreName != null) { store.Put(key, ValueAndTimestamp <VR> .Make(newValue, Context.Timestamp)); tupleForwarder.MaybeForward(key, newValue, oldValue); } else { Forward(key, new Change <VR>(oldValue, newValue)); } }
public void SerializeData() { long millie = DateTime.Now.Millisecond; string s = "coucou"; var stringSerdes = new StringSerDes(); var serdes = new ValueAndTimestampSerDes <string>(stringSerdes); var data = ValueAndTimestamp <string> .Make(s, millie); var r = serdes.Serialize(data, new Confluent.Kafka.SerializationContext()); Assert.IsNotNull(r); Assert.Greater(r.Length, 0); var r2 = serdes.Deserialize(r, new Confluent.Kafka.SerializationContext()); Assert.AreEqual(s, r2.Value); Assert.AreEqual(millie, r2.Timestamp); }
public override ValueAndTimestamp <V> Deserialize(byte[] data, SerializationContext context) { if (data != null) { using var stream = new MemoryStream(data); using var reader = new BinaryReader(stream); long t = reader.ReadInt64(); int length = reader.ReadInt32(); byte[] d = reader.ReadBytes(length); V v = InnerSerdes.Deserialize(d, context); ValueAndTimestamp <V> obj = ValueAndTimestamp <V> .Make(v, t); return(obj); } else { return(null); } }
public ValueAndTimestamp <VR> Get(K key) { ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key); ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key); VR newValue = default; V1 value1 = valueAndTimestamp1 == null ? default : valueAndTimestamp1.Value; long ts1 = valueAndTimestamp1 == null ? -1 : valueAndTimestamp1.Timestamp; V2 value2 = valueAndTimestamp2 == null ? default : valueAndTimestamp2.Value; long ts2 = valueAndTimestamp2 == null ? -1 : valueAndTimestamp2.Timestamp; if (value1 != null || value2 != null) { newValue = joiner.Apply(value1, value2); } return(ValueAndTimestamp <VR> .Make(newValue, Math.Max(ts1, ts2))); }
public override void Process(K key, Change <V> change) { LogProcessingKeyValue(key, change); V newValue = ComputeValue(key, change.NewValue); V oldValue = sendOldValues ? ComputeValue(key, change.OldValue) : default(V); if (sendOldValues && oldValue == null && newValue == null) { return; // unnecessary to forward here. } if (queryableStoreName != null) { store.Put(key, ValueAndTimestamp <V> .Make(newValue, Context.Timestamp)); //TODO : tupleForwarder.maybeForward(key, newValue, oldValue); } else { this.Forward(key, new Change <V>(oldValue, newValue)); } }
public void TestFacadeFetchRangeKeyOneElement() { DateTime dt = DateTime.Now; store.Put( "coucou", ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()), dt.GetMilliseconds()); store.Put( "coucou", ValueAndTimestamp <int> .Make(500, dt.AddMilliseconds(400).GetMilliseconds()), dt.GetMilliseconds()); var enumerator = facade.Fetch("coucou", dt.AddSeconds(-2), dt.AddSeconds(5)); Assert.IsNotNull(enumerator); var list = enumerator.ToList(); Assert.AreEqual(1, list.Count); Assert.AreEqual(500, list[0].Value); Assert.AreEqual(dt.GetMilliseconds(), list[0].Key); }
public void ChangelogPut() { var consumerConfig = new ConsumerConfig(); consumerConfig.GroupId = "test-result-store-changelog"; var consumer = kafkaSupplier.GetConsumer(consumerConfig, null); consumer.Subscribe("test-store-changelog"); var message = new Message <byte[], byte[]> { Headers = new Headers(), Timestamp = new Timestamp(DateTime.Now) }; var consumerResult = new ConsumeResult <byte[], byte[]> { Message = message, Offset = 0, Topic = "test-store", Partition = 0 }; context.SetRecordMetaData(consumerResult); store.Put(CreateKey("test"), CreateValue(ValueAndTimestamp <string> .Make("value", 0)), 0); store.Put(CreateKey("test2"), null, 0); var r = consumer.Consume(); Assert.AreEqual("test", FromKey(r.Message.Key).Key); Assert.AreEqual("value", FromValue(r.Message.Value)); r = consumer.Consume(); Assert.AreEqual("test2", FromKey(r.Message.Key).Key); Assert.IsNull(r.Message.Value); consumer.Dispose(); }
public override void Process(K key, V value) { if (key == null) { log.LogWarning($"Skipping record due to null key.value =[{value}] topic =[{Context.RecordContext.Topic}] partition =[{Context.RecordContext.Partition}] offset =[{Context.RecordContext.Offset }]"); droppedRecordsSensor.Record(); return; } observedStreamTime = Math.Max(observedStreamTime, Context.Timestamp); long closeTime = observedStreamTime - windowOptions.GracePeriodMs; var matchedWindows = windowOptions.WindowsFor(Context.Timestamp); foreach (var entry in matchedWindows) { long windowStart = entry.Key, windowEnd = entry.Value.EndMs; if (windowEnd > closeTime) { var oldAggAndTimestamp = windowStore.Fetch(key, windowStart); Agg oldAgg = oldAggAndTimestamp == null ? default : oldAggAndTimestamp.Value; long newTs; Agg newAgg; if (oldAggAndTimestamp == null) { oldAgg = initializer.Apply(); newTs = Context.Timestamp; } else { newTs = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp); } newAgg = aggregator.Apply(key, value, oldAgg); windowStore.Put(key, ValueAndTimestamp <Agg> .Make(newAgg, newTs), windowStart); tupleForwarder.MaybeForward(new Windowed <K>(key, entry.Value), newAgg, sendOldValues ? oldAgg : default, newTs);
public ValueAndTimestamp <VR> Get(K key) { ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key); if (valueAndTimestamp1 != null) { ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key); if (valueAndTimestamp2 != null) { return(ValueAndTimestamp <VR> .Make( joiner.Apply(valueAndTimestamp1.Value, valueAndTimestamp2.Value), Math.Max(valueAndTimestamp1.Timestamp, valueAndTimestamp2.Timestamp))); } else { return(null); } } else { return(null); } }
private byte[] CreateValue(string value) => valueAndTimestampSerDes.Serialize(ValueAndTimestamp <string> .Make(value, DateTime.Now.GetMilliseconds()), new SerializationContext());