public void FetchAllTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now;
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
store1.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt.GetMilliseconds()), new SerializationContext()), dt.GetMilliseconds());
var result = composite.FetchAll(dt.AddSeconds(-2), dt.AddSeconds(2));
Assert.IsNotNull(result);
var items = result.ToList();
Assert.AreEqual(2, items.Count);
var c1 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test"));
var c2 = items.FirstOrDefault(kp => kp.Key.Key.Equals("test2"));
Assert.IsNotNull(c1);
Assert.IsNotNull(c2);
Assert.AreEqual("coucou1", c1.Value);
Assert.AreEqual("coucou2", c2.Value);
}
public void ResetTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider =
new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1,
store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
store2.Put(bytes2,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
var enumerator = composite.Range("test", "test2");
Assert.IsNotNull(enumerator);
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual("test", enumerator.PeekNextKey());
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual("test2", enumerator.PeekNextKey());
Assert.IsFalse(enumerator.MoveNext());
enumerator.Reset();
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual("test", enumerator.PeekNextKey());
Assert.IsTrue(enumerator.MoveNext());
Assert.AreEqual("test2", enumerator.PeekNextKey());
Assert.IsFalse(enumerator.MoveNext());
}
public void ReverseAllTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider =
new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1,
store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
store1.Put(bytes2,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1bis", dt), new SerializationContext()));
store2.Put(bytes,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
store2.Put(bytes2,
valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2bis", dt), new SerializationContext()));
var result = composite.ReverseAll().ToList();
Assert.IsNotNull(result);
Assert.AreEqual(4, result.Count);
Assert.AreEqual("test2", result[0].Key);
Assert.AreEqual("coucou1bis", result[0].Value);
Assert.AreEqual("test", result[1].Key);
Assert.AreEqual("coucou1", result[1].Value);
Assert.AreEqual("test2", result[2].Key);
Assert.AreEqual("coucou2bis", result[2].Value);
Assert.AreEqual("test", result[3].Key);
Assert.AreEqual("coucou2", result[3].Value);
}
public void DeserializeNullData()
{
var stringSerdes = new StringSerDes();
var serdes = new ValueAndTimestampSerDes <string>(stringSerdes);
var r = serdes.Deserialize(null);
Assert.IsNull(r);
}
public void DeserializeNullData()
{
var stringSerdes = new StringSerDes();
var serdes = new ValueAndTimestampSerDes <string>(stringSerdes);
var r = serdes.Deserialize(null, new Confluent.Kafka.SerializationContext());
Assert.IsNull(r);
}
protected override void Publish(Bytes key, byte[] valueAndTs)
{
if (valueAndTs != null)
{
(long ts, byte[] data) = ValueAndTimestampSerDes.Extract(valueAndTs);
context.Log(Name, key, data, ts);
}
else
{
context.Log(Name, key, null, context.RecordContext.Timestamp);
}
}
private void Publish(Bytes key, byte[] value)
{
if (value == null)
{
context.Log(Name, key, null, context.RecordContext.Timestamp);
}
else
{
(long ts, byte[] data) = ValueAndTimestampSerDes.Extract(value);
context.Log(Name, key, data, ts);
}
}
public void CountTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var bytes2 = new Bytes(Encoding.UTF8.GetBytes("test2"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()));
store2.Put(bytes2, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou2", dt), new SerializationContext()));
Assert.AreEqual(2, composite.ApproximateNumEntries());
}
public void DeserializeData()
{
long millie = DateTime.Now.Millisecond;
string s = "coucou";
var stringSerdes = new StringSerDes();
var serdes = new ValueAndTimestampSerDes <string>(stringSerdes);
var data = ValueAndTimestamp <string> .Make(s, millie);
var r = serdes.Deserialize(serdes.Serialize(data));
Assert.IsNotNull(r);
Assert.AreEqual(s, r.Value);
Assert.AreEqual(millie, r.Timestamp);
}
public void FetchTest()
{
InMemoryWindowStore store1 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
InMemoryWindowStore store2 = new InMemoryWindowStore("store", TimeSpan.FromDays(1), 1000 * 10);
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyWindowStore <string, string>(provider, new WindowStoreType <string, string>(), "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt), new SerializationContext()), dt);
var result = composite.Fetch("test", dt);
Assert.IsNotNull(result);
Assert.AreEqual("coucou1", result);
}
public void GetTest()
{
InMemoryKeyValueStore store1 = new InMemoryKeyValueStore("store");
InMemoryKeyValueStore store2 = new InMemoryKeyValueStore("store");
var dt = DateTime.Now.GetMilliseconds();
var valueSerdes = new ValueAndTimestampSerDes <string>(new StringSerDes());
var bytes = new Bytes(Encoding.UTF8.GetBytes("test"));
var provider = new MockStateProvider <string, string>(1000 * 10, new StringSerDes(), new StringSerDes(), store1, store2);
var composite = new CompositeReadOnlyKeyValueStore <string, string>(provider, storeType, "store");
store1.Put(bytes, valueSerdes.Serialize(ValueAndTimestamp <string> .Make("coucou1", dt)));
var result = composite.Get("test");
Assert.IsNotNull(result);
Assert.AreEqual("coucou1", result);
}
public void Begin()
{
valueAndTimestampSerDes = new ValueAndTimestampSerDes <string>(stringSerDes);
config = new StreamConfig();
config.ApplicationId = $"unit-test-changelogging-tkv";
id = new TaskId {
Id = 0, Partition = 0
};
partition = new TopicPartition("source", 0);
kafkaSupplier = new SyncKafkaSupplier();
var producerConfig = new ProducerConfig();
producerConfig.ClientId = "producer-1";
var producerClient = kafkaSupplier.GetProducer(producerConfig);
recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG));
recordCollector.Init(ref producerClient);
var changelogsTopics = new Dictionary <string, string> {
{ "test-store", "test-store-changelog" }
};
stateManager = new ProcessorStateManager(
id,
new List <TopicPartition> {
partition
},
changelogsTopics,
new MockChangelogRegister(),
new MockOffsetCheckpointManager());
task = new Mock <AbstractTask>();
task.Setup(k => k.Id).Returns(id);
context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry());
context.UseRecordCollector(recordCollector);
var inmemorystore = new InMemoryKeyValueStore("test-store");
store = new ChangeLoggingTimestampedKeyValueBytesStore(inmemorystore);
store.Init(context, store);
}
public void SerializeData()
{
long millie = DateTime.Now.Millisecond;
string s = "coucou";
var stringSerdes = new StringSerDes();
var serdes = new ValueAndTimestampSerDes <string>(stringSerdes);
var data = ValueAndTimestamp <string> .Make(s, millie);
var r = serdes.Serialize(data, new Confluent.Kafka.SerializationContext());
Assert.IsNotNull(r);
Assert.Greater(r.Length, 0);
var r2 = serdes.Deserialize(r, new Confluent.Kafka.SerializationContext());
Assert.AreEqual(s, r2.Value);
Assert.AreEqual(millie, r2.Timestamp);
}