public ValueAndTimestamp <KeyValuePair <K1, V1> > Get(K key)
            {
                ValueAndTimestamp <V> valueAndTimestamp = parentTableGetter.Get(key);
                var v = mapper.Apply(key, valueAndTimestamp != null ? valueAndTimestamp.Value : default);

                return(ValueAndTimestamp <KeyValuePair <K1, V1> > .Make(v, valueAndTimestamp == null?context.Timestamp : valueAndTimestamp.Timestamp));
            }
Esempio n. 2
0
        public void TestFacadeFetchRangeKeyTwoElement()
        {
            DateTime dt = DateTime.Now;

            store.Put(
                "coucou",
                ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()),
                dt.GetMilliseconds());
            store.Put(
                "coucou-bis",
                ValueAndTimestamp <int> .Make(500, dt.AddMilliseconds(400).GetMilliseconds()),
                dt.AddMilliseconds(100).GetMilliseconds());
            var enumerator = facade.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(5));

            Assert.IsNotNull(enumerator);
            var list = enumerator.ToList();

            Assert.AreEqual(2, list.Count);
            Assert.AreEqual(120, list[0].Value);
            Assert.AreEqual("coucou", list[0].Key.Key);
            Assert.AreEqual(TimeSpan.FromSeconds(2), list[0].Key.Window.TotalTime);
            Assert.AreEqual(dt.GetMilliseconds(), list[0].Key.Window.StartMs);
            Assert.AreEqual(500, list[1].Value);
            Assert.AreEqual("coucou-bis", list[1].Key.Key);
            Assert.AreEqual(TimeSpan.FromSeconds(2), list[1].Key.Window.TotalTime);
            Assert.AreEqual(dt.AddMilliseconds(100).GetMilliseconds(), list[1].Key.Window.StartMs);
        }
Esempio n. 3
0
        public void FetchAllTest()
        {
            InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
            InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
            var dt          = DateTime.Now;
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var bytes2      = new Bytes(Encoding.UTF8.GetBytes("test2"));
            var provider    = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
            var composite   = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");

            store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
            store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
            var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2));

            Assert.IsNotNull(result);
            var items = result.ToList();

            Assert.AreEqual(2, items.Count);
            var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test"));
            var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2"));

            Assert.IsNotNull(c1);
            Assert.IsNotNull(c2);
            Assert.AreEqual("coucou1", c1.Value);
            Assert.AreEqual("coucou2", c2.Value);
        }
        public override void Process(K key, V value)
        {
            // If the key or value is null we don't need to proceed
            if (key == null || value == null)
            {
                log.Warn($"Skipping record due to null key or value. key=[{key}] value=[{value}] topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]");
                return;
            }

            ValueAndTimestamp <V> oldAggAndTimestamp = store.Get(key);
            V    oldAgg, newAgg;
            long newTimestamp;

            if (oldAggAndTimestamp == null)
            {
                oldAgg       = default;
                newAgg       = value;
                newTimestamp = Context.Timestamp;
            }
            else
            {
                oldAgg       = oldAggAndTimestamp.Value;
                newAgg       = reducer.Apply(oldAgg, value);
                newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
            }

            store.Put(key, ValueAndTimestamp <V> .Make(newAgg, newTimestamp));
            tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
Esempio n. 5
0
        public override void Process(K key, V value)
        {
            LogProcessingKeyValue(key, value);
            if (key == null)
            {
                log.Warn($"Skipping record due to null key. topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]");
                return;
            }

            if (queryableName != null)
            {
                ValueAndTimestamp <V> oldValueAndTimestamp = store.Get(key);
                V oldValue;
                if (oldValueAndTimestamp != null)
                {
                    oldValue = oldValueAndTimestamp.Value;
                    if (Context.Timestamp < oldValueAndTimestamp.Timestamp)
                    {
                        log.Warn($"Detected out-of-order KTable update for {store.Name} at offset {Context.Offset}, partition {Context.Partition}.");
                    }
                }
                else
                {
                    oldValue = default(V);
                }
                store.Put(key, ValueAndTimestamp <V> .Make(value, Context.Timestamp));
                tupleForwarder.MaybeForward(key, value, oldValue);
            }
            else
            {
                this.Forward <K, Change <V> >(key, new Change <V>(default(V), value));
            }
        }
Esempio n. 6
0
        public override void Process(K key, V value)
        {
            if (key == null || value == null)
            {
                log.Warn($"Skipping record due to null key or value. key=[{key}] value=[{value}] topic=[{Context.Topic}] partition=[{Context.Partition}] offset=[{Context.Offset}]");
                return;
            }

            ValueAndTimestamp <T> oldAggAndTimestamp = store.Get(key);
            T    oldAgg, newAgg;
            long newTimestamp;

            if (oldAggAndTimestamp == null)
            {
                oldAgg       = initializer.Apply();
                newTimestamp = Context.Timestamp;
            }
            else
            {
                oldAgg       = oldAggAndTimestamp.Value;
                newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
            }

            newAgg = aggregator.Apply(key, value, oldAgg);

            store.Put(key, ValueAndTimestamp <T> .Make(newAgg, newTimestamp));
            tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
        public void ResetTest()
        {
            InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
            InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
            var dt          = DateTime.Now.GetMilliseconds();
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var bytes2      = new Bytes(Encoding.UTF8.GetBytes("test2"));
            var provider    =
                new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1,
                                                       store2);
            var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");

            store1.Put(bytes,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
            store2.Put(bytes2,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
            var enumerator = composite.Range("test", "test2");

            Assert.IsNotNull(enumerator);
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual("test", enumerator.PeekNextKey());
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual("test2", enumerator.PeekNextKey());
            Assert.IsFalse(enumerator.MoveNext());
            enumerator.Reset();
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual("test", enumerator.PeekNextKey());
            Assert.IsTrue(enumerator.MoveNext());
            Assert.AreEqual("test2", enumerator.PeekNextKey());
            Assert.IsFalse(enumerator.MoveNext());
        }
        public void ReverseAllTest()
        {
            InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
            InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
            var dt          = DateTime.Now.GetMilliseconds();
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var bytes2      = new Bytes(Encoding.UTF8.GetBytes("test2"));
            var provider    =
                new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1,
                                                       store2);
            var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");

            store1.Put(bytes,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
            store1.Put(bytes2,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1bis", dt), new SerializationContext()));
            store2.Put(bytes,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
            store2.Put(bytes2,
                       valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2bis", dt), new SerializationContext()));
            var result = composite.ReverseAll().ToList();

            Assert.IsNotNull(result);
            Assert.AreEqual(4, result.Count);
            Assert.AreEqual("test2", result[0].Key);
            Assert.AreEqual("coucou1bis", result[0].Value);
            Assert.AreEqual("test", result[1].Key);
            Assert.AreEqual("coucou1", result[1].Value);
            Assert.AreEqual("test2", result[2].Key);
            Assert.AreEqual("coucou2bis", result[2].Value);
            Assert.AreEqual("test", result[3].Key);
            Assert.AreEqual("coucou2", result[3].Value);
        }
Esempio n. 9
0
        public override void Process(K key, Change <V> value)
        {
            if (key == null)
            {
                throw new StreamsException($"Record key for KTable aggregate operator with state {queryableStoreName} should not be null.");
            }

            ValueAndTimestamp <T> oldAggAndTimestamp = store.Get(key);
            T    oldAgg = oldAggAndTimestamp != null ? oldAggAndTimestamp.Value : default;
            T    intermediateAgg;
            long newTimestamp = Context.Timestamp;

            // first try to remove the old value
            if (oldAggAndTimestamp != null && value.OldValue != null && oldAgg != null)
            {
                intermediateAgg = remove.Apply(key, value.OldValue, oldAgg);
                newTimestamp    = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
            }
            else
            {
                intermediateAgg = oldAgg;
            }

            // then try to add the new value
            T newAgg;

            if (value.NewValue != null)
            {
                T initializedAgg;
                if (intermediateAgg == null)
                {
                    initializedAgg = initializer.Apply();
                }
                else
                {
                    initializedAgg = intermediateAgg;
                }

                newAgg = add.Apply(key, value.NewValue, initializedAgg);
                if (oldAggAndTimestamp != null)
                {
                    newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
                }
            }
            else
            {
                newAgg = intermediateAgg;
            }

            // update the store with the new value
            store.Put(key, ValueAndTimestamp <T> .Make(newAgg, newTimestamp));
            tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
            private ValueAndTimestamp <VR> ComputeValue(K key, ValueAndTimestamp <V> valueAndTimestamp)
            {
                VR   newValue  = default(VR);
                long timestamp = 0;

                if (valueAndTimestamp != null)
                {
                    newValue  = mapper.Apply(key, valueAndTimestamp.Value);
                    timestamp = valueAndTimestamp.Timestamp;
                }

                return(ValueAndTimestamp <VR> .Make(newValue, timestamp));
            }
Esempio n. 11
0
        public void TestFacadeFetchKeyByTime()
        {
            DateTime dt = DateTime.Now;

            store.Put(
                "coucou",
                ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()),
                dt.GetMilliseconds());
            var r = facade.Fetch("coucou", dt.GetMilliseconds());

            Assert.IsNotNull(r);
            Assert.AreEqual(120, r);
        }
        public override void Process(K key, Change <V> value)
        {
            // the keys should never be null
            if (key == null)
            {
                throw new StreamsException($"Record key for KTable reduce operator with state {queryableStoreName} should not be null.");
            }

            ValueAndTimestamp <V> oldAggAndTimestamp = store.Get(key);
            V    oldAgg = oldAggAndTimestamp != null ? oldAggAndTimestamp.Value : default;
            V    intermediateAgg;
            long newTimestamp = Context.Timestamp;

            // first try to remove the old value
            if (oldAggAndTimestamp != null && value.OldValue != null && oldAgg != null)
            {
                intermediateAgg = substractor.Apply(oldAgg, value.OldValue);
                newTimestamp    = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
            }
            else
            {
                intermediateAgg = oldAgg;
            }

            // then try to add the new value
            V newAgg;

            if (value.NewValue != null)
            {
                if (intermediateAgg == null)
                {
                    newAgg = value.NewValue;
                }
                else
                {
                    newAgg = adder.Apply(intermediateAgg, value.NewValue);
                }

                if (oldAggAndTimestamp != null)
                {
                    newTimestamp = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
                }
            }
            else
            {
                newAgg = intermediateAgg;
            }

            // update the store with the new value
            store.Put(key, ValueAndTimestamp <V> .Make(newAgg, newTimestamp));
            tupleForwarder.MaybeForward(key, newAgg, sendOldValues ? oldAgg : default, newTimestamp);
        public override ValueAndTimestamp <V> Deserialize(byte[] data)
        {
            using (var stream = new MemoryStream(data))
                using (var reader = new BinaryReader(stream))
                {
                    long   t                  = reader.ReadInt64();
                    int    length             = reader.ReadInt32();
                    byte[] d                  = reader.ReadBytes(length);
                    V      v                  = InnerSerdes.Deserialize(d);
                    ValueAndTimestamp <V> obj = ValueAndTimestamp <V> .Make(v, t);

                    return(obj);
                }
        }
        public void CountTest()
        {
            InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
            InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
            var dt          = DateTime.Now.GetMilliseconds();
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var bytes2      = new Bytes(Encoding.UTF8.GetBytes("test2"));
            var provider    = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
            var composite   = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");

            store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
            store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
            Assert.AreEqual(2, composite.ApproximateNumEntries());
        }
        public void DeserializeData()
        {
            long   millie = DateTime.Now.Millisecond;
            string s      = "coucou";

            var stringSerdes = new StringSerDes();
            var serdes       = new ValueAndTimestampSerDes <string>(stringSerdes);
            var data         = ValueAndTimestamp <string> .Make(s, millie);

            var r = serdes.Deserialize(serdes.Serialize(data));

            Assert.IsNotNull(r);
            Assert.AreEqual(s, r.Value);
            Assert.AreEqual(millie, r.Timestamp);
        }
 public override void Process(K key, Change <V> value)
 {
     if (!string.IsNullOrEmpty(queryableStoreName))
     {
         store.Put(key, ValueAndTimestamp <V> .Make(value.NewValue, Context.Timestamp));
         tupleForwarder.MaybeForward(key, value.NewValue, sendOldValues ? value.OldValue : default);
     }
     else
     {
         if (sendOldValues)
         {
             Forward(key, value);
         }
         else
         {
             Forward(key, new Change <V>(default, value.NewValue));
Esempio n. 17
0
        public void FetchTest()
        {
            InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
            InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
            var dt          = DateTime.Now.GetMilliseconds();
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var provider    = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
            var composite   = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");

            store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt);
            var result = composite.Fetch("test", dt);

            Assert.IsNotNull(result);
            Assert.AreEqual("coucou1", result);
        }
Esempio n. 18
0
        public void GetTest()
        {
            InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
            InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
            var dt          = DateTime.Now.GetMilliseconds();
            var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
            var bytes       = new Bytes(Encoding.UTF8.GetBytes("test"));
            var provider    = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
            var composite   = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");

            store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt)));
            var result = composite.Get("test");

            Assert.IsNotNull(result);
            Assert.AreEqual("coucou1", result);
        }
        public override void Process(K key, Change <V> value)
        {
            LogProcessingKeyValue(key, value);
            VR newValue = ComputeValue(key, value.NewValue);
            VR oldValue = sendOldValues ? ComputeValue(key, value.OldValue) : default(VR);

            if (queryableStoreName != null)
            {
                store.Put(key, ValueAndTimestamp <VR> .Make(newValue, Context.Timestamp));
                tupleForwarder.MaybeForward(key, newValue, oldValue);
            }
            else
            {
                Forward(key, new Change <VR>(oldValue, newValue));
            }
        }
Esempio n. 20
0
        public void SerializeData()
        {
            long   millie = DateTime.Now.Millisecond;
            string s      = "coucou";

            var stringSerdes = new StringSerDes();
            var serdes       = new ValueAndTimestampSerDes <string>(stringSerdes);
            var data         = ValueAndTimestamp <string> .Make(s, millie);

            var r = serdes.Serialize(data, new Confluent.Kafka.SerializationContext());

            Assert.IsNotNull(r);
            Assert.Greater(r.Length, 0);
            var r2 = serdes.Deserialize(r, new Confluent.Kafka.SerializationContext());

            Assert.AreEqual(s, r2.Value);
            Assert.AreEqual(millie, r2.Timestamp);
        }
        public override ValueAndTimestamp <V> Deserialize(byte[] data, SerializationContext context)
        {
            if (data != null)
            {
                using var stream = new MemoryStream(data);
                using var reader = new BinaryReader(stream);
                long   t                  = reader.ReadInt64();
                int    length             = reader.ReadInt32();
                byte[] d                  = reader.ReadBytes(length);
                V      v                  = InnerSerdes.Deserialize(d, context);
                ValueAndTimestamp <V> obj = ValueAndTimestamp <V> .Make(v, t);

                return(obj);
            }
            else
            {
                return(null);
            }
        }
            public ValueAndTimestamp <VR> Get(K key)
            {
                ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key);
                ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key);

                VR newValue = default;

                V1 value1                       = valueAndTimestamp1 == null ? default : valueAndTimestamp1.Value;
                                       long ts1 = valueAndTimestamp1 == null ? -1 : valueAndTimestamp1.Timestamp;

                                       V2 value2                       = valueAndTimestamp2 == null ? default : valueAndTimestamp2.Value;
                                                              long ts2 = valueAndTimestamp2 == null ? -1 : valueAndTimestamp2.Timestamp;

                                                              if (value1 != null || value2 != null)
                                                              {
                                                                  newValue = joiner.Apply(value1, value2);
                                                              }

                                                              return(ValueAndTimestamp <VR> .Make(newValue, Math.Max(ts1, ts2)));
            }
        public override void Process(K key, Change <V> change)
        {
            LogProcessingKeyValue(key, change);
            V newValue = ComputeValue(key, change.NewValue);
            V oldValue = sendOldValues ? ComputeValue(key, change.OldValue) : default(V);

            if (sendOldValues && oldValue == null && newValue == null)
            {
                return; // unnecessary to forward here.
            }

            if (queryableStoreName != null)
            {
                store.Put(key, ValueAndTimestamp <V> .Make(newValue, Context.Timestamp));
                //TODO : tupleForwarder.maybeForward(key, newValue, oldValue);
            }
            else
            {
                this.Forward(key, new Change <V>(oldValue, newValue));
            }
        }
Esempio n. 24
0
        public void TestFacadeFetchRangeKeyOneElement()
        {
            DateTime dt = DateTime.Now;

            store.Put(
                "coucou",
                ValueAndTimestamp <int> .Make(120, dt.AddMilliseconds(100).GetMilliseconds()),
                dt.GetMilliseconds());
            store.Put(
                "coucou",
                ValueAndTimestamp <int> .Make(500, dt.AddMilliseconds(400).GetMilliseconds()),
                dt.GetMilliseconds());
            var enumerator = facade.Fetch("coucou", dt.AddSeconds(-2), dt.AddSeconds(5));

            Assert.IsNotNull(enumerator);
            var list = enumerator.ToList();

            Assert.AreEqual(1, list.Count);
            Assert.AreEqual(500, list[0].Value);
            Assert.AreEqual(dt.GetMilliseconds(), list[0].Key);
        }
Esempio n. 25
0
        public void ChangelogPut()
        {
            var consumerConfig = new ConsumerConfig();

            consumerConfig.GroupId = "test-result-store-changelog";
            var consumer = kafkaSupplier.GetConsumer(consumerConfig, null);

            consumer.Subscribe("test-store-changelog");

            var message = new Message <byte[], byte[]>
            {
                Headers   = new Headers(),
                Timestamp = new Timestamp(DateTime.Now)
            };

            var consumerResult = new ConsumeResult <byte[], byte[]>
            {
                Message   = message,
                Offset    = 0,
                Topic     = "test-store",
                Partition = 0
            };

            context.SetRecordMetaData(consumerResult);

            store.Put(CreateKey("test"), CreateValue(ValueAndTimestamp <string> .Make("value", 0)), 0);
            store.Put(CreateKey("test2"), null, 0);

            var r = consumer.Consume();

            Assert.AreEqual("test", FromKey(r.Message.Key).Key);
            Assert.AreEqual("value", FromValue(r.Message.Value));

            r = consumer.Consume();

            Assert.AreEqual("test2", FromKey(r.Message.Key).Key);
            Assert.IsNull(r.Message.Value);

            consumer.Dispose();
        }
        public override void Process(K key, V value)
        {
            if (key == null)
            {
                log.LogWarning($"Skipping record due to null key.value =[{value}] topic =[{Context.RecordContext.Topic}] partition =[{Context.RecordContext.Partition}] offset =[{Context.RecordContext.Offset }]");
                droppedRecordsSensor.Record();
                return;
            }

            observedStreamTime = Math.Max(observedStreamTime, Context.Timestamp);
            long closeTime      = observedStreamTime - windowOptions.GracePeriodMs;
            var  matchedWindows = windowOptions.WindowsFor(Context.Timestamp);

            foreach (var entry in matchedWindows)
            {
                long windowStart = entry.Key, windowEnd = entry.Value.EndMs;
                if (windowEnd > closeTime)
                {
                    var oldAggAndTimestamp = windowStore.Fetch(key, windowStart);

                    Agg oldAgg = oldAggAndTimestamp == null ? default : oldAggAndTimestamp.Value;
                                 long newTs;
                                 Agg  newAgg;

                                 if (oldAggAndTimestamp == null)
                                 {
                                     oldAgg = initializer.Apply();
                                     newTs  = Context.Timestamp;
                                 }
                                 else
                                 {
                                     newTs = Math.Max(Context.Timestamp, oldAggAndTimestamp.Timestamp);
                                 }

                                 newAgg = aggregator.Apply(key, value, oldAgg);
                                 windowStore.Put(key, ValueAndTimestamp <Agg> .Make(newAgg, newTs), windowStart);
                                 tupleForwarder.MaybeForward(new Windowed <K>(key, entry.Value), newAgg, sendOldValues ? oldAgg : default, newTs);
Esempio n. 27
0
            public ValueAndTimestamp <VR> Get(K key)
            {
                ValueAndTimestamp <V1> valueAndTimestamp1 = iKTableValueGetter1.Get(key);

                if (valueAndTimestamp1 != null)
                {
                    ValueAndTimestamp <V2> valueAndTimestamp2 = iKTableValueGetter2.Get(key);

                    if (valueAndTimestamp2 != null)
                    {
                        return(ValueAndTimestamp <VR> .Make(
                                   joiner.Apply(valueAndTimestamp1.Value, valueAndTimestamp2.Value),
                                   Math.Max(valueAndTimestamp1.Timestamp, valueAndTimestamp2.Timestamp)));
                    }
                    else
                    {
                        return(null);
                    }
                }
                else
                {
                    return(null);
                }
            }
 private byte[] CreateValue(string value)
 => valueAndTimestampSerDes.Serialize(ValueAndTimestamp <string> .Make(value, DateTime.Now.GetMilliseconds()), new SerializationContext());