public override void Process(K1 key, V1 value)
{
// If the key or value is null we don't need to proceed
if (key == null || value == null)
{
log.Warn($"Skipping record due to null key or value. key=[{key}] value=[{value}] topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]");
return;
}
else
{
K2 mappedKey = mapper.Apply(key, value);
V2 value2 = mappedKey == null ? null : ValueAndTimestamp.GetValueOrNull(valueGetter.Get(mappedKey));
if (leftJoin || value2 != null)
{
Forward(key, joiner.Apply(value, value2));
}
}
}
public void SerializeData()
{
long millie = DateTime.Now.Millisecond;
string s = "coucou";
var stringSerdes = new StringSerDes();
var serdes = new ValueAndTimestampSerDes <string>(stringSerdes);
var data = ValueAndTimestamp <string> .Make(s, millie);
var r = serdes.Serialize(data);
Assert.IsNotNull(r);
Assert.Greater(r.Length, 0);
var r2 = serdes.Deserialize(r);
Assert.AreEqual(s, r2.Value);
Assert.AreEqual(millie, r2.Timestamp);
}
public ValueAndTimestamp <VR> Get(K key)
{
ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key);
if (valueAndTimestamp2 != null)
{
ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key);
long resultTimestamp;
if (valueAndTimestamp1 != null)
{
resultTimestamp = Math.Max(valueAndTimestamp2.Timestamp, valueAndTimestamp1.Timestamp);
}
else
{
resultTimestamp = valueAndTimestamp2.Timestamp;
}
return(ValueAndTimestamp <VR> .Make(
joiner.Apply(valueAndTimestamp1 != null ? valueAndTimestamp1.Value : default, valueAndTimestamp2.Value),
public override ValueAndTimestamp <V> Deserialize(byte[] data, SerializationContext context)
{
if (data != null)
{
using var stream = new MemoryStream(data);
using var reader = new BinaryReader(stream);
long t = reader.ReadInt64();
int length = reader.ReadInt32();
byte[] d = reader.ReadBytes(length);
V v = InnerSerdes.Deserialize(d, context);
ValueAndTimestamp <V> obj = ValueAndTimestamp <V> .Make(v, t);
return(obj);
}
else
{
return(null);
}
}
public void GetTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider =
new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1,
store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
var result = composite.Get("test");
Assert.IsNotNull(result);
Assert.AreEqual("coucou1", result);
}
public override byte[] Serialize(ValueAndTimestamp <V> data, SerializationContext context)
{
if (data != null)
{
using var stream = new MemoryStream();
using var writer = new BinaryWriter(stream);
byte[] innerobj = InnerSerdes.Serialize(data.Value, context);
writer.Write(data.Timestamp);
writer.Write(innerobj.Length);
writer.Write(innerobj);
writer.Flush();
return(stream.ToArray());
}
else
{
return(null);
}
}
public ValueAndTimestamp <VR> Get(K key)
{
ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key);
ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key);
VR newValue = default;
V1 value1 = valueAndTimestamp1 == null ? default : valueAndTimestamp1.Value;
long ts1 = valueAndTimestamp1 == null ? -1 : valueAndTimestamp1.Timestamp;
V2 value2 = valueAndTimestamp2 == null ? default : valueAndTimestamp2.Value;
long ts2 = valueAndTimestamp2 == null ? -1 : valueAndTimestamp2.Timestamp;
if (value1 != null || value2 != null)
{
newValue = joiner.Apply(value1, value2);
}
return(ValueAndTimestamp <VR> .Make(newValue, Math.Max(ts1, ts2)));
}
public void AllTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt)));
store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt)));
var result = composite.All().ToList();
Assert.IsNotNull(result);
Assert.AreEqual(2, result.Count);
Assert.AreEqual("coucou1", result.FirstOrDefault(k => k.Key.Equals("test")).Value);
Assert.AreEqual("coucou2", result.FirstOrDefault(k => k.Key.Equals("test2")).Value);
}
public void Fetch2Test()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now;
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
var result = composite.Fetch("test", dt.AddSeconds(-2), dt.AddSeconds(2));
Assert.IsNotNull(result);
var items = result.ToList();
Assert.AreEqual(1, items.Count);
var c1 = items.FirstOrDefault(kp => kp.Value.Equals("coucou1"));
Assert.IsNotNull(c1);
}
public override void Process(K key, Change <V> change)
{
LogProcessingKeyValue(key, change);
V newValue = ComputeValue(key, change.NewValue);
V oldValue = sendOldValues ? ComputeValue(key, change.OldValue) : default(V);
if (sendOldValues && oldValue == null && newValue == null)
{
return; // unnecessary to forward here.
}
if (queryableStoreName != null)
{
store.Put(key, ValueAndTimestamp <V> .Make(newValue, Context.Timestamp));
//TODO : tupleForwarder.maybeForward(key, newValue, oldValue);
}
else
{
this.Forward(key, new Change <V>(oldValue, newValue));
}
}
public void TestFacadeFetchRangeKeyOneElement()
{
DateTime dt = DateTime.Now;
store.Put(
"coucou",
ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()),
dt.GetMilliseconds());
store.Put(
"coucou",
ValueAndTimestamp <int> .Make(500, dt.AddMilliseconds(400).GetMilliseconds()),
dt.GetMilliseconds());
var enumerator = facade.Fetch("coucou", dt.AddSeconds(-2), dt.AddSeconds(5));
Assert.IsNotNull(enumerator);
var list = enumerator.ToList();
Assert.AreEqual(1, list.Count);
Assert.AreEqual(500, list[0].Value);
Assert.AreEqual(dt.GetMilliseconds(), list[0].Key);
}
public void ChangelogPut()
{
var consumerConfig = new ConsumerConfig();
consumerConfig.GroupId = "test-result-store-changelog";
var consumer = kafkaSupplier.GetConsumer(consumerConfig, null);
consumer.Subscribe("test-store-changelog");
var message = new Message <byte[], byte[]>
{
Headers = new Headers(),
Timestamp = new Timestamp(DateTime.Now)
};
var consumerResult = new ConsumeResult <byte[], byte[]>
{
Message = message,
Offset = 0,
Topic = "test-store",
Partition = 0
};
context.SetRecordMetaData(consumerResult);
store.Put(CreateKey("test"), CreateValue(ValueAndTimestamp <string> .Make("value", 0)), 0);
store.Put(CreateKey("test2"), null, 0);
var r = consumer.Consume();
Assert.AreEqual("test", FromKey(r.Message.Key).Key);
Assert.AreEqual("value", FromValue(r.Message.Value));
r = consumer.Consume();
Assert.AreEqual("test2", FromKey(r.Message.Key).Key);
Assert.IsNull(r.Message.Value);
consumer.Dispose();
}
public override void Process(K key, V value)
{
if (key == null)
{
log.LogWarning($"Skipping record due to null key.value =[{value}] topic =[{Context.RecordContext.Topic}] partition =[{Context.RecordContext.Partition}] offset =[{Context.RecordContext.Offset }]");
droppedRecordsSensor.Record();
return;
}
observedStreamTime = Math.Max(observedStreamTime, Context.Timestamp);
long closeTime = observedStreamTime - windowOptions.GracePeriodMs;
var matchedWindows = windowOptions.WindowsFor(Context.Timestamp);
foreach (var entry in matchedWindows)
{
long windowStart = entry.Key, windowEnd = entry.Value.EndMs;
if (windowEnd > closeTime)
{
var oldAggAndTimestamp = windowStore.Fetch(key, windowStart);
Agg oldAgg = oldAggAndTimestamp == null ? default : oldAggAndTimestamp.Value;
long newTs;
Agg newAgg;
if (oldAggAndTimestamp == null)
{
oldAgg = initializer.Apply();
newTs = Context.Timestamp;
}
else
{
newTs = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
}
newAgg = aggregator.Apply(key, value, oldAgg);
windowStore.Put(key, ValueAndTimestamp <Agg> .Make(newAgg, newTs), windowStart);
tupleForwarder.MaybeForward(new Windowed <K>(key, entry.Value), newAgg, sendOldValues ? oldAgg : default, newTs);
public ValueAndTimestamp <VR> Get(K key)
{
ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key);
if (valueAndTimestamp1 != null)
{
ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key);
if (valueAndTimestamp2 != null)
{
return(ValueAndTimestamp <VR> .Make(
joiner.Apply(valueAndTimestamp1.Value, valueAndTimestamp2.Value),
Math.Max(valueAndTimestamp1.Timestamp, valueAndTimestamp2.Timestamp)));
}
else
{
return(null);
}
}
else
{
return(null);
}
}
private byte[] GetValueBytes(ValueAndTimestamp <V> value) => this.valueSerdes.Serialize(value);
private byte[] CreateValue(string value) => valueAndTimestampSerDes.Serialize(ValueAndTimestamp <string> .Make(value, DateTime.Now.GetMilliseconds()), new SerializationContext());
public ValueAndTimestamp<KeyValuePair<K1, V1>> Get(K key)
{
ValueAndTimestamp<V> valueAndTimestamp = parentTableGetter.Get(key);
var v = mapper.Apply(key, valueAndTimestamp != null ? valueAndTimestamp.Value : default);
return ValueAndTimestamp<KeyValuePair<K1, V1>>.Make(v, valueAndTimestamp == null ? context.Timestamp : valueAndTimestamp.Timestamp);
}
public static V GetValueOrNull <V>(ValueAndTimestamp <V> valueAndTimestamp)
where V : class
{
return(valueAndTimestamp == null ? null : valueAndTimestamp.Value);
}
public static V GetValueOrNull <V>(ValueAndTimestamp <V> valueAndTimestamp)
{
return(valueAndTimestamp == null ? default : valueAndTimestamp.Value);
}
public void Put(K key, ValueAndTimestamp <V> value) => wrapped.Put(GetKeyBytes(key), GetValueBytes(value));
public ValueAndTimestamp <V> PutIfAbsent(K key, ValueAndTimestamp <V> value) => FromValue(wrapped.PutIfAbsent(GetKeyBytes(key), GetValueBytes(value)));
private byte[] CreateValue(ValueAndTimestamp <string> value) => valueAndTsSerDes.Serialize(value, new SerializationContext());
