public void KTableSource()
{
var builder = new StreamBuilder();
builder.Table("table-topic", RocksDb <string, string> .As <StringSerDes, StringSerDes>("table-topic-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-map";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("table-topic");
inputTopic.PipeInput("key1", "1");
inputTopic.PipeInput("key2", "2");
var store = driver.GetKeyValueStore <string, string>("table-topic-store");
Assert.IsNotNull(store);
var resultK1 = store.Get("key1");
var resultK2 = store.Get("key2");
Assert.AreEqual("1", resultK1);
Assert.AreEqual("2", resultK2);
}
config.RemoveRocksDbFolderForTest();
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.BootstrapServers = "localhost:9092";
config.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Earliest;
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
StreamBuilder builder = new StreamBuilder();
builder
.Table("table", RocksDb <string, string> .As("table-store").WithLoggingEnabled())
.ToStream()
.Print(Printed <string, string> .ToOut());
Topology t = builder.Build();
KafkaStream stream = new KafkaStream(t, config);
await stream.StartAsync(stoppingToken);
}
public void Init()
{
token1 = new System.Threading.CancellationTokenSource();
token2 = new System.Threading.CancellationTokenSource();
config.ApplicationId = "test-stream-thread";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE;
config.PollMs = 10;
mockKafkaSupplier = new MockKafkaSupplier(2, 0);
var builder = new StreamBuilder();
builder.Table("topic", RocksDb <string, string> .As("store").WithLoggingEnabled());
var topo = builder.Build();
topo.Builder.RewriteTopology(config);
topo.Builder.BuildTopology();
thread1 = StreamThread.Create(
"thread-0", "c0",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
0) as StreamThread;
thread2 = StreamThread.Create(
"thread-1", "c1",
topo.Builder, new StreamMetricsRegistry(), config,
mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")),
1) as StreamThread;
var internalTopicManager =
new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")));
InternalTopicManagerUtils
.New()
.CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter();
}
public void GlobalTableWithStoreQueryableWithRocksDb()
{
var builder = new StreamBuilder();
var table = builder.GlobalTable("topic", RocksDb <string, string> .As("global-store"));
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-globaltable";
config.UseRandomRocksDbConfigForTest();
Topology t = builder.Build();
using (var driver = new TopologyTestDriver(t, config))
{
var inputTopic = driver.CreateInputTopic <string, string>("topic");
inputTopic.PipeInput("test", "coucou");
var store = driver.GetKeyValueStore <string, string>("global-store");
var ele = store.Get("test");
Assert.IsNotNull(ele);
Assert.AreEqual("coucou", ele);
}
config.RemoveRocksDbFolderForTest();
}
public void StreamTaskWrittingCheckpoint()
{
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-app";
config.StateDir = Path.Combine(".", Guid.NewGuid().ToString());
var serdes = new StringSerDes();
var builder = new StreamBuilder();
var table = builder.Table("topic", RocksDb <string, string> .As("store").WithLoggingEnabled());
TaskId id = new TaskId {
Id = 0, Partition = 0
};
var topology = builder.Build();
topology.Builder.RewriteTopology(config);
var processorTopology = topology.Builder.BuildTopology(id);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var part = new TopicPartition("topic", 0);
StreamTask task = new StreamTask(
"thread-0",
id,
new List <TopicPartition> {
part
},
processorTopology,
consumer,
config,
supplier,
producer,
new MockChangelogRegister()
, new StreamMetricsRegistry());
task.GroupMetadata = consumer as SyncConsumer;
task.InitializeStateStores();
task.InitializeTopology();
task.RestorationIfNeeded();
task.CompleteRestoration();
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < 5; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds()));
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
Assert.IsTrue(task.CommitNeeded);
task.Commit();
}
task.MayWriteCheckpoint(true);
messages = new List <ConsumeResult <byte[], byte[]> >();
for (int i = 0; i < StateManagerTools.OFFSET_DELTA_THRESHOLD_FOR_CHECKPOINT + 10; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
}
task.MayWriteCheckpoint(false);
var lines = File.ReadAllLines(Path.Combine(config.StateDir, config.ApplicationId, "0-0", ".checkpoint"));
Assert.AreEqual(3, lines.Length);
Assert.AreEqual("test-app-store-changelog 0 10014", lines[2]);
task.Suspend();
task.Close();
Directory.Delete(config.StateDir, true);
}
private void TaskManagerRestorationChangelog(bool persistenStateStore = false)
{
var stateDir = Path.Combine(".", Guid.NewGuid().ToString());
var config = new StreamConfig <StringSerDes, StringSerDes>();
config.ApplicationId = "test-restoration-changelog-app";
config.StateDir = stateDir;
var builder = new StreamBuilder();
builder.Table("topic",
persistenStateStore
? RocksDb <string, string> .As("store").WithLoggingEnabled(null)
: InMemory <string, string> .As("store").WithLoggingEnabled(null));
var serdes = new StringSerDes();
var topology = builder.Build();
topology.Builder.RewriteTopology(config);
var supplier = new SyncKafkaSupplier();
var producer = supplier.GetProducer(config.ToProducerConfig());
var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null);
var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig());
var storeChangelogReader =
new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry());
var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer,
storeChangelogReader, new StreamMetricsRegistry());
var taskManager = new TaskManager(topology.Builder, taskCreator,
supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader);
var part = new TopicPartition("topic", 0);
taskManager.CreateTasks(
new List <TopicPartition>
{
part
});
var task = taskManager.ActiveTaskFor(part);
IDictionary <TaskId, ITask> tasks = new Dictionary <TaskId, ITask>();
tasks.Add(task.Id, task);
taskManager.TryToCompleteRestoration();
storeChangelogReader.Restore();
Assert.IsTrue(taskManager.TryToCompleteRestoration());
List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >();
int offset = 0;
for (int i = 0; i < 5; ++i)
{
messages.Add(
new ConsumeResult <byte[], byte[]>
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize($"key{i + 1}", new SerializationContext()),
Value = serdes.Serialize($"value{i + 1}", new SerializationContext())
},
TopicPartitionOffset = new TopicPartitionOffset(part, offset++)
});
}
task.AddRecords(messages);
// Process messages
while (task.CanProcess(DateTime.Now.GetMilliseconds()))
{
Assert.IsTrue(task.Process());
}
taskManager.CommitAll();
// Simulate Close + new open
taskManager.Close();
restoreConsumer.Resume(new TopicPartition("test-restoration-changelog-app-store-changelog", 0).ToSingle());
taskManager.CreateTasks(
new List <TopicPartition>
{
part
});
task = taskManager.ActiveTaskFor(part);
tasks = new Dictionary <TaskId, ITask>();
tasks.Add(task.Id, task);
Assert.IsFalse(taskManager.TryToCompleteRestoration());
storeChangelogReader.Restore();
Assert.IsTrue(taskManager.TryToCompleteRestoration());
var store = task.GetStore("store");
var items = (store as ITimestampedKeyValueStore <string, string>).All().ToList();
Assert.AreEqual(5, items.Count);
taskManager.Close();
if (persistenStateStore)
{
Directory.Delete(stateDir, true);
}
}