public void ApplyInternalChangelogTopicsWithExistingTopics() { AdminClientConfig config = new AdminClientConfig(); config.BootstrapServers = "localhost:9092"; StreamConfig config2 = new StreamConfig(); DefaultTopicManager manager = new DefaultTopicManager(config2, kafkaSupplier.GetAdmin(config)); ((SyncProducer)kafkaSupplier.GetProducer(new ProducerConfig())).CreateTopic("topic"); IDictionary <string, InternalTopicConfig> topics = new Dictionary <string, InternalTopicConfig>(); topics.Add("topic", new UnwindowedChangelogTopicConfig { Name = "topic", NumberPartitions = 1 }); topics.Add("topic1", new UnwindowedChangelogTopicConfig { Name = "topic1", NumberPartitions = 1 }); var r = manager.ApplyAsync(0, topics).GetAwaiter().GetResult().ToList(); Assert.AreEqual(1, r.Count); Assert.AreEqual("topic1", r[0]); }
public void Initialize() { streamMetricsRegistry = new StreamMetricsRegistry(Guid.NewGuid().ToString(), MetricsRecordingLevel.DEBUG); config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; config.CommitIntervalMs = 1; var builder = new StreamBuilder(); var stream = builder.Stream <string, string>("topic"); stream.GroupByKey().Count(); stream.To("topic2"); var topo = builder.Build(); id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = builder.Build().Builder.BuildTopology(id); syncKafkaSupplier = new SyncKafkaSupplier(); var producer = syncKafkaSupplier.GetProducer(config.ToProducerConfig()); var consumer = syncKafkaSupplier.GetConsumer(config.ToConsumerConfig(), null); topicPartition = new TopicPartition("topic", 0); task = new StreamTask( threadId, id, new List <TopicPartition> { topicPartition }, processorTopology, consumer, config, syncKafkaSupplier, null, new MockChangelogRegister(), streamMetricsRegistry); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); var activeRestorationSensor = streamMetricsRegistry.GetSensors().FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_RESTORATION))); Assert.AreEqual(1, activeRestorationSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_RESTORATION, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); task.CompleteRestoration(); Assert.AreEqual(0, activeRestorationSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_RESTORATION, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); }
public void TaskManagerCommitWithoutCommitNeeed() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.CommitAll()); taskManager.Close(); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig <StringSerDes, StringSerDes>(); var serdes = new StringSerDes(); config.ApplicationId = "test-count"; var builder = new StreamBuilder(); Materialized <string, long, IKeyValueStore <Bytes, byte[]> > m = null; builder .Table <string, string>("topic") .GroupBy((k, v) => KeyValuePair.Create(k.ToUpper(), v)) .Count(m); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); Assert.AreEqual(2, task.Context.States.StateStoreNames.Count()); var nameStore1 = task.Context.States.StateStoreNames.ElementAt(0); var nameStore2 = task.Context.States.StateStoreNames.ElementAt(1); Assert.IsNotNull(nameStore1); Assert.IsNotNull(nameStore2); Assert.AreNotEqual(string.Empty, nameStore1); Assert.AreNotEqual(string.Empty, nameStore2); var store1 = task.GetStore(nameStore1); var store2 = task.GetStore(nameStore2); Assert.IsInstanceOf <TimestampedKeyValueStore <string, string> >(store1); Assert.IsInstanceOf <TimestampedKeyValueStore <string, long> >(store2); Assert.AreEqual(0, (store1 as TimestampedKeyValueStore <string, string>).ApproximateNumEntries()); Assert.AreEqual(0, (store2 as TimestampedKeyValueStore <string, long>).ApproximateNumEntries()); }
public void GetCurrentPartitionMetadataTests() { var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.FollowMetadata = true; var configConsumer = config.Clone(); configConsumer.ApplicationId = "test-consumer"; int?h = null; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .MapValues((v) => { h = StreamizMetadata.GetCurrentPartitionMetadata(); return(v); }) .To("output"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(configConsumer.ToConsumerConfig(), null); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); consumer.Subscribe("output"); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); } while (result == null); source.Cancel(); thread.Dispose(); Assert.NotNull(h); Assert.AreEqual(0, h); }
public void GetElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); var source = new CancellationTokenSource(); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; stream.Start(source.Token); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1"), Value = serdes.Serialize("coucou") }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); Assert.AreEqual(1, store.ApproximateNumEntries()); var item = store.Get("key1"); Assert.IsNotNull(item); Assert.AreEqual("coucou", item); } source.Cancel(); stream.Close(); }
public async Task BuildGlobalStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 1; var builder = new StreamBuilder(); builder.GlobalTable <string, string>("test", InMemory <string, string> .As("store")); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(new ProducerConfig()); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var stringSerdes = new StringSerDes(); producer.Produce("test", new Message <byte[], byte[]> { Key = stringSerdes.Serialize("key", new SerializationContext()), Value = stringSerdes.Serialize("value", new SerializationContext()) }); Thread.Sleep(250); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); Assert.AreEqual(1, store.ApproximateNumEntries()); } stream.Dispose(); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig <StringSerDes, StringSerDes>(); var serdes = new StringSerDes(); config.ApplicationId = "test-window-count"; var builder = new StreamBuilder(); Materialized <string, long, IWindowStore <Bytes, byte[]> > m = null; builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Count(m); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null, new MockChangelogRegister(), new StreamMetricsRegistry()); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); task.CompleteRestoration(); Assert.AreEqual(1, task.Context.States.StateStoreNames.Count()); var nameStore = task.Context.States.StateStoreNames.ElementAt(0); Assert.IsNotNull(nameStore); Assert.AreNotEqual(string.Empty, nameStore); var store = task.GetStore(nameStore); Assert.IsInstanceOf <ITimestampedWindowStore <string, long> >(store); Assert.AreEqual(0, (store as ITimestampedWindowStore <string, long>).All().ToList().Count); }
public void TaskManagerAssignedUnknownPartitions() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer, storeChangelogReader, new StreamMetricsRegistry()); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1) }); taskManager.RevokeTasks( new List <TopicPartition> { new TopicPartition("topic", 1) }); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2) }); taskManager.TryToCompleteRestoration(); Assert.AreEqual(3, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); taskManager.Close(); }
public void StreamThreadCommitIntervalWorkflow() { var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.CommitIntervalMs = 1; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); var offsets = thread.GetCommittedOffsets(new List <TopicPartition> { new TopicPartition("topic", 0) }, TimeSpan.FromSeconds(10)).ToList(); Assert.AreEqual(1, offsets.Count); Assert.AreEqual(1, offsets[0].Offset.Value); Assert.AreEqual(0, offsets[0].TopicPartition.Partition.Value); Assert.AreEqual("topic", offsets[0].Topic); source.Cancel(); thread.Dispose(); }
public void RestoreFromRecovery() { storeChangelogReader.Restore(); stateMgr.Checkpoint(); stateMgr.Close(); store.Init(context, store); stateMgr.InitializeOffsetsFromCheckpoint(); var producer = supplier.GetProducer(config.ToProducerConfig()); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key3", "value3")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key4", "value4")); restoreConsumer.Resume(new TopicPartition(changelogTopic, 0).ToSingle()); storeChangelogReader.Restore(); stateMgr.Checkpoint(); stateMgr.Close(); File.Delete(Path.Combine(config.StateDir, config.ApplicationId, "0-0", ".checkpoint")); store.Init(context, store); stateMgr.InitializeOffsetsFromCheckpoint(); restoreConsumer.Resume(new TopicPartition(changelogTopic, 0).ToSingle()); storeChangelogReader.Restore(); Assert.AreEqual(4, store.All().ToList().Count); var metadata = storeChangelogReader.GetMetadata(new TopicPartition(changelogTopic, 0)); Assert.IsNotNull(metadata); Assert.AreEqual(ChangelogState.COMPLETED, metadata.ChangelogState); Assert.AreEqual(3, metadata.RestoreEndOffset); Assert.AreEqual(2, metadata.TotalRestored); }
public void Init() { stateDir = Path.Combine(".", Guid.NewGuid().ToString()); config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-storechangelog-app"; config.StateDir = stateDir; config.PollMs = 100; supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var topicPart = new TopicPartition("topic", 0); changelogTopic = "store-changelog-topic"; var changelogsTopics = new Dictionary <string, string>(); changelogsTopics.Add("store", changelogTopic); var id = new TaskId { Id = 0, Partition = 0 }; store = new RocksDbKeyValueStore("store"); storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); stateMgr = new ProcessorStateManager( id, topicPart.ToSingle(), changelogsTopics, storeChangelogReader, new OffsetCheckpointFile(Path.Combine(config.StateDir, config.ApplicationId, $"{id.Id}-{id.Partition}")) ); Mock <AbstractTask> moq = new Mock <AbstractTask>(); moq.Setup(t => t.Id).Returns(new TaskId { Id = 0, Partition = 0 }); context = new ProcessorContext(moq.Object, config, stateMgr, new StreamMetricsRegistry()); store.Init(context, store); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key1", "value1")); producer.Produce(changelogTopic, CreateMessage(changelogTopic, "key2", "value2")); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig <StringSerDes, StringSerDes>(); var serdes = new StringSerDes(); config.ApplicationId = "test-window-count"; var builder = new StreamBuilder(); Materialized <string, int, IWindowStore <Bytes, byte[]> > m = null; builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Aggregate( () => 0, (k, v, agg) => Math.Max(v.Length, agg), m); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null); task.GroupMetadata = consumer as SyncConsumer; Assert.Throws <StreamsException>(() => task.InitializeStateStores()); }
public void RestoreFromCheckpoint() { var serdes = new StringSerDes(); byte[] GetBytes(string content) => serdes.Serialize(content, SerializationContext.Empty); kvStoreMock .Setup(s => s.Init(context, It.IsAny <IStateStore>())) .Callback((ProcessorContext c, IStateStore store) => { stateManager.Register(store, (k, v, t) => { }); }); otherStoreMock .Setup(s => s.Init(context, It.IsAny <IStateStore>())) .Callback((ProcessorContext c, IStateStore store) => { stateManager.Register(store, (k, v, t) => { }); }); StringBuilder sb = new StringBuilder(); sb.AppendLine("0").AppendLine("1").AppendLine("kv-store-topic 0 10"); var producer = mockKafkaSupplier.GetProducer(new ProducerConfig()); for (int i = 0; i < 10; ++i) { producer.Produce("other-store-topic", new Message <byte[], byte[]>() { Key = GetBytes($"key{i}"), Value = GetBytes($"value{i}") }); } Directory.CreateDirectory(Path.Combine(streamConfigMock.Object.StateDir, streamConfigMock.Object.ApplicationId, "global")); File.WriteAllText( Path.Combine(streamConfigMock.Object.StateDir, streamConfigMock.Object.ApplicationId, "global", ".checkpoint"), sb.ToString()); stateManager.Initialize(); Assert.AreEqual(10, stateManager.ChangelogOffsets[new TopicPartition("kv-store-topic", 0)]); Assert.AreEqual(9, stateManager.ChangelogOffsets[new TopicPartition("other-store-topic", 0)]); Directory.Delete(Path.Combine(streamConfigMock.Object.StateDir), true); }
public void Begin() { valueAndTimestampSerDes = new ValueAndTimestampSerDes <string>(stringSerDes); config = new StreamConfig(); config.ApplicationId = $"unit-test-changelogging-tkv"; id = new TaskId { Id = 0, Partition = 0 }; partition = new TopicPartition("source", 0); kafkaSupplier = new SyncKafkaSupplier(); var producerConfig = new ProducerConfig(); producerConfig.ClientId = "producer-1"; var producerClient = kafkaSupplier.GetProducer(producerConfig); recordCollector = new RecordCollector("p-1", config, id, new NoRunnableSensor("s", "s", MetricsRecordingLevel.DEBUG)); recordCollector.Init(ref producerClient); var changelogsTopics = new Dictionary <string, string> { { "test-store", "test-store-changelog" } }; stateManager = new ProcessorStateManager( id, new List <TopicPartition> { partition }, changelogsTopics, new MockChangelogRegister(), new MockOffsetCheckpointManager()); task = new Mock <AbstractTask>(); task.Setup(k => k.Id).Returns(id); context = new ProcessorContext(task.Object, config, stateManager, new StreamMetricsRegistry()); context.UseRecordCollector(recordCollector); var inmemorystore = new InMemoryKeyValueStore("test-store"); store = new ChangeLoggingTimestampedKeyValueBytesStore(inmemorystore); store.Init(context, store); }
public void WithNullMaterialize() { // CERTIFIED THAT SAME IF Materialize is null, a state store exist for count processor with a generated namestore var config = new StreamConfig(); var serdes = new StringSerDes(); config.ApplicationId = "test-window-reduce"; var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(2000)) .Reduce((v1, v2) => v1.Length > v2.Length ? v1 : v2); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null, new MockChangelogRegister(), new StreamMetricsRegistry()); task.GroupMetadata = consumer as SyncConsumer; Assert.Throws <StreamsException>(() => task.InitializeStateStores()); }
public static UnassignedStreamTask Create(TaskId id) { var config = new StreamConfig(); config.ApplicationId = "un-assigned-stream-task"; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); return(new UnassignedStreamTask( "un-assigned-stream-task", id, new List <TopicPartition>(), ProcessorTopology.EMPTY, consumer, config, supplier, producer)); }
public static UnassignedStreamTask Create() { var config = new StreamConfig(); config.ApplicationId = "un-assigned-stream-task"; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); return(new UnassignedStreamTask( "un-assigned-stream-task", new TaskId { Id = -1, Partition = -1 }, new List <TopicPartition>(), null, consumer, config, supplier, producer)); }
public async Task GetRangeKVStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.KeyValueStore <string, string>())); Assert.IsNotNull(store); var list = store.Range("key1", "key2").ToList(); Assert.AreEqual(1, list.Count); var item = list[0]; Assert.IsNotNull(item); Assert.AreEqual("coucou", item.Value); Assert.AreEqual("key1", item.Key); } stream.Dispose(); }
public void TaskManagerCommit() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); var part = new TopicPartition("topic", 0); var task = taskManager.ActiveTaskFor(part); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } // ONLY ONE TASK HAVE BEEN RECORDS Assert.AreEqual(1, taskManager.CommitAll()); // CHECK IN TOPIC topic2 consumer.Subscribe("topic2"); List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >(); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); if (result != null) { results.Add(result); consumer.Commit(result); } } while (result != null); Assert.AreEqual(5, results.Count); for (int i = 0; i < 5; ++i) { Assert.AreEqual($"KEY{i + 1}", serdes.Deserialize(results[i].Message.Key, new SerializationContext())); Assert.AreEqual($"VALUE{i+1}", serdes.Deserialize(results[i].Message.Value, new SerializationContext())); } // NO RECORD IN THIS TASKS part = new TopicPartition("topic", 2); task = taskManager.ActiveTaskFor(part); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.Process()); taskManager.Close(); }
public void StreamThreadNormalWorkflow() { bool metricsReporterCalled = false; List <ThreadState> allStates = new List <ThreadState>(); var expectedStates = new List <ThreadState> { ThreadState.CREATED, ThreadState.STARTING, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PENDING_SHUTDOWN, ThreadState.DEAD }; var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.MetricsReporter = (sensor) => { metricsReporterCalled = true; }; config.AddOrUpdate(StreamConfig.metricsIntervalMsCst, 10); var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; allStates.Add(thread.State); thread.StateChanged += (t, o, n) => { Assert.IsInstanceOf <ThreadState>(n); allStates.Add(n as ThreadState); }; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); source.Cancel(); thread.Dispose(); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); Assert.AreEqual(expectedStates, allStates); Assert.IsTrue(metricsReporterCalled); }
//[Test] // TODO : fix that public void WorkflowCompleteBufferedRecordsTest() { int maxBuffered = 10; var token = new System.Threading.CancellationTokenSource(); var serdes = new StringSerDes(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-group"; config.MaxTaskIdleMs = (long)TimeSpan.FromSeconds(100).TotalMilliseconds; config.BufferedRecordsPerPartition = maxBuffered; config.PollMs = 10; var builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("topic1"); var stream2 = builder.Stream <string, string>("topic2"); stream1 .Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("output"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(token.Token); for (int i = 0; i < maxBuffered + 1; ++i) { producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{i}", new SerializationContext()) }); } // CONSUME PAUSE AFTER maxBuffered + 1 messages System.Threading.Thread.Sleep(50); // Add one message more with consumer in stream thread in pause producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{maxBuffered+1}", new SerializationContext()) }); Assert.AreEqual(1, thread.ActiveTasks.Count()); var task = thread.ActiveTasks.ToArray()[0]; Assert.IsNotNull(task.Grouper); Assert.IsFalse(task.Grouper.AllPartitionsBuffered); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered()); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered(new TopicPartition("topic1", 0))); Assert.AreEqual(0, task.Grouper.NumBuffered(new TopicPartition("topic2", 0))); producer.Produce("topic2", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"test", new SerializationContext()) }); List <ConsumeResult <byte[], byte[]> > records = new List <ConsumeResult <byte[], byte[]> >(); do { records.AddRange(consumer.ConsumeRecords(TimeSpan.FromMilliseconds(100)).ToList()); } while (records.Count() <= 12); Assert.AreEqual(maxBuffered + 2, records.Count()); for (int i = 0; i < maxBuffered + 2; ++i) { var message = records.ToArray()[i]; Assert.AreEqual("key", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.IsTrue(serdes.Deserialize(message.Message.Value, new SerializationContext()).Contains($"coucou{i}-")); } token.Cancel(); thread.Dispose(); }
public void TaskMetricsTest() { var serdes = new StringSerDes(); var cloneConfig = config.Clone(); cloneConfig.ApplicationId = "consume-test"; var producer = syncKafkaSupplier.GetProducer(cloneConfig.ToProducerConfig()); var consumer = syncKafkaSupplier.GetConsumer(cloneConfig.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); int nbMessage = 1000; // produce 1000 messages to input topic List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < nbMessage; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, offset++) }); } task.AddRecords(messages); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } var messagesSink = new List <ConsumeResult <byte[], byte[]> >(); AssertExtensions.WaitUntil(() => { messagesSink.AddRange(consumer.ConsumeRecords(TimeSpan.FromSeconds(1))); return(messagesSink.Count < nbMessage); }, TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); long now = DateTime.Now.GetMilliseconds(); var sensors = streamMetricsRegistry.GetThreadScopeSensor(threadId); var processorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS))); Assert.AreEqual(2, processorSensor.Metrics.Count()); Assert.AreEqual(nbMessage, processorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.IsTrue( (double)(processorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value) > 0d); var enforcedProcessorSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ENFORCED_PROCESSING))); Assert.AreEqual(2, enforcedProcessorSensor.Metrics.Count()); Assert.AreEqual(0, enforcedProcessorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.AreEqual(0, enforcedProcessorSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ENFORCED_PROCESSING + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var processLatency = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.PROCESS_LATENCY))); Assert.AreEqual(2, processLatency.Metrics.Count()); Assert.IsTrue( (double)processLatency.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.AVG_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); Assert.IsTrue( (double)processLatency.Metrics[MetricName.NameAndGroup( TaskMetrics.PROCESS_LATENCY + StreamMetricsRegistry.MAX_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); var commitSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.COMMIT))); Assert.AreEqual(2, commitSensor.Metrics.Count()); Assert.AreEqual(nbMessage, commitSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.COMMIT + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.IsTrue( (double)commitSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.COMMIT + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value > 0d); var droppedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.DROPPED_RECORDS))); Assert.AreEqual(2, droppedRecordSensor.Metrics.Count()); Assert.AreEqual(0, droppedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.TOTAL_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); Assert.AreEqual(0, droppedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.DROPPED_RECORDS + StreamMetricsRegistry.RATE_SUFFIX, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var activeBufferedRecordSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT))); Assert.AreEqual(1, activeBufferedRecordSensor.Metrics.Count()); Assert.AreEqual(0, activeBufferedRecordSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_TASK_PREFIX + TaskMetrics.BUFFER_COUNT, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var restorationRecordsSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.RESTORATION_RECORDS))); Assert.AreEqual(1, restorationRecordsSensor.Metrics.Count()); Assert.AreEqual(0, restorationRecordsSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.RESTORATION_RECORDS, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); var activeRestorationSensor = sensors.FirstOrDefault(s => s.Name.Equals(GetSensorName(TaskMetrics.ACTIVE_RESTORATION))); Assert.AreEqual(1, activeRestorationSensor.Metrics.Count()); Assert.AreEqual(0, activeRestorationSensor.Metrics[MetricName.NameAndGroup( TaskMetrics.ACTIVE_RESTORATION, StreamMetricsRegistry.TASK_LEVEL_GROUP)].Value); }
public void StreamTaskWrittingCheckpoint() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.StateDir = Path.Combine(".", Guid.NewGuid().ToString()); var serdes = new StringSerDes(); var builder = new StreamBuilder(); var table = builder.Table("topic", RocksDb <string, string> .As("store").WithLoggingEnabled()); TaskId id = new TaskId { Id = 0, Partition = 0 }; var topology = builder.Build(); topology.Builder.RewriteTopology(config); var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, producer, new MockChangelogRegister() , new StreamMetricsRegistry()); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); task.CompleteRestoration(); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } task.MayWriteCheckpoint(true); messages = new List <ConsumeResult <byte[], byte[]> >(); for (int i = 0; i < StateManagerTools.OFFSET_DELTA_THRESHOLD_FOR_CHECKPOINT + 10; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } task.MayWriteCheckpoint(false); var lines = File.ReadAllLines(Path.Combine(config.StateDir, config.ApplicationId, "0-0", ".checkpoint")); Assert.AreEqual(3, lines.Length); Assert.AreEqual("test-app-store-changelog 0 10014", lines[2]); task.Suspend(); task.Close(); Directory.Delete(config.StateDir, true); }
public void StreamTaskSuspendResume() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Table <string, string>("topic", InMemory <string, string> .As("store").WithLoggingDisabled()) .MapValues((k, v) => v.ToUpper()) .ToStream() .To("topic2"); TaskId id = new TaskId { Id = 0, Partition = 0 }; var topology = builder.Build(); var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, producer, new MockChangelogRegister() , new StreamMetricsRegistry()); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); task.RestorationIfNeeded(); task.CompleteRestoration(); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } Assert.IsNotNull(task.GetStore("store")); task.Suspend(); Assert.IsNull(task.GetStore("store")); task.Resume(); task.RestorationIfNeeded(); Assert.IsNotNull(task.GetStore("store")); task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } // CHECK IN TOPIC topic2 consumer.Subscribe("topic2"); List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >(); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); if (result != null) { results.Add(result); consumer.Commit(result); } } while (result != null); Assert.AreEqual(10, results.Count); task.Close(); }
public async Task GetWindowElementInStateStore() { var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .GroupByKey() .WindowedBy(TumblingWindowOptions.Of(TimeSpan.FromMinutes(1))) .Count(InMemoryWindows <string, long> .As("store")); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); dt = DateTime.Now; producer.Produce("test", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()), Timestamp = new Confluent.Kafka.Timestamp(dt) }); Thread.Sleep(50); var store = stream.Store(StoreQueryParameters.FromNameAndType("store", QueryableStoreTypes.WindowStore <string, long>())); Assert.IsNotNull(store); var @enum = store.All(); Assert.AreEqual(1, store.All().ToList().Count); var item = store.Fetch("key1", dt.AddMinutes(-1), dt.AddMinutes(1)); Assert.IsNotNull(item); Assert.IsTrue(item.MoveNext()); Assert.IsTrue(item.Current.HasValue); Assert.AreEqual(1, item.Current.Value.Value); item.Dispose(); } stream.Dispose(); }
public async Task exec(IConfiguration config, IServiceProvider services) { Console.WriteLine("Process"); var destTopic = config["spring.cloud.stream.bindings.output.destination"]; Console.WriteLine(destTopic); using (var scope = services.CreateScope()) { this._dataService = scope.ServiceProvider .GetRequiredService <IDataService>(); bool isRunningState = false; var timeout = TimeSpan.FromSeconds(10); DateTime dt = DateTime.Now; Order[] capture = this._dataService.readData(); // Inyectamos los datos obtenidos al Stream var sConfig = new StreamConfig <StringSerDes, StringSerDes>(); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; sConfig.SchemaRegistryUrl = config["SchemaRegistryUrl"]; sConfig.AutoRegisterSchemas = true; sConfig.NumStreamThreads = 10; sConfig.Acks = Acks.All; sConfig.AddConsumerConfig("allow.auto.create.topics", "true"); sConfig.InnerExceptionHandler = (e) => ExceptionHandlerResponse.CONTINUE; var schemaRegistryClient = new CachedSchemaRegistryClient (new SchemaRegistryConfig { Url = sConfig.SchemaRegistryUrl }); var supplier = new SyncKafkaSupplier(new KafkaLoggerAdapter(sConfig)); var producerConfig = sConfig.ToProducerConfig(); var adminConfig = sConfig.ToAdminConfig(sConfig.ApplicationId); var admin = supplier.GetAdmin(adminConfig); // try // { // var topic = new TopicSpecification // { // Name = destTopic, // NumPartitions = 1, // ReplicationFactor = 3 // }; // var topicProduct = new TopicSpecification // { // Name = "product-external", // NumPartitions = 1, // ReplicationFactor = 3 // }; // IList<TopicSpecification> topics = new List<TopicSpecification>(); // topics.Add(topic); // topics.Add(topicProduct); // await admin.CreateTopicsAsync(topics); // } // catch (Exception topicExists) // { // Console.WriteLine("Topic alreade exists"); // Console.Write(topicExists); // } var producer = supplier.GetProducer(producerConfig); StreamBuilder builder = new StreamBuilder(); var serdes = new SchemaAvroSerDes <Order>(); var keySerdes = new Int32SerDes(); builder.Table(destTopic, keySerdes, serdes, InMemory <int, Order> .As(config["table"])); var t = builder.Build(); KafkaStream stream = new KafkaStream(t, sConfig, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } if (isRunningState) { // //create a well formatted Endpoint in external topic var endpProducer = new ProducerBuilder <byte[], Endpoint>(producerConfig) .SetValueSerializer(new AvroSerializer <Endpoint>(schemaRegistryClient, new AvroSerializerConfig { AutoRegisterSchemas = true }).AsSyncOverAsync()).Build(); //create a well formatted Product in external topic var productProducer = new ProducerBuilder <byte[], Product>(producerConfig) .SetValueSerializer(new AvroSerializer <Product>(schemaRegistryClient, new AvroSerializerConfig { AutoRegisterSchemas = true }).AsSyncOverAsync()).Build(); for (int k = 1; k < 10; k++) { endpProducer.Produce("api-endpoints", new Message <byte[], Endpoint> { Key = new Int32SerDes().Serialize(k, new SerializationContext()), Value = new Endpoint { endpoint_id = ("endpoint" + k), endpoint_url = ("http://endpoint" + k + "/"), http_method = "POST" } }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Endpoint Message sent !"); } }); productProducer.Produce("product-external", new Message <byte[], Product> { Key = new Int32SerDes().Serialize(1, new SerializationContext()), Value = new Product { name = "Producto de Software", price = 1234.5F, product_id = 3 } }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Product Message sent !"); } }); } Thread.Sleep(10); for (int k = 1; k < 10; k++) { producer.Produce(destTopic, new Confluent.Kafka.Message <byte[], byte[]> { Key = keySerdes.Serialize(k, new SerializationContext()), Value = serdes.Serialize(new Order { order_id = k, price = 123.5F, product_id = k }, new SerializationContext()) }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Order Message sent !"); } }); } Thread.Sleep(50); } } }
public void StandardWorkflowTaskManager() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); for (int i = 0; i < 4; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } // Revoked 2 partitions taskManager.RevokeTasks(new List <TopicPartition> { new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(2, taskManager.ActiveTasks.Count()); Assert.AreEqual(2, taskManager.RevokedTasks.Count()); for (int i = 0; i < 2; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } var taskFailed = taskManager.ActiveTaskFor(new TopicPartition("topic", 2)); Assert.IsNull(taskFailed); taskManager.Close(); Assert.AreEqual(0, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); }
public async Task KafkaStreamDeserializationExceptionHandlerContinueTest() { var _return = new List <KeyValuePair <string, string> >(); var timeout = TimeSpan.FromSeconds(10); bool isRunningState = false; DateTime dt = DateTime.Now; var config = new StreamConfig <SerdesThrowException, SerdesThrowException>(); config.ApplicationId = "test"; config.BootstrapServers = "127.0.0.1"; config.PollMs = 10; config.DeserializationExceptionHandler += (c, r, e) => ExceptionHandlerResponse.CONTINUE; var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var builder = new StreamBuilder(); builder .Stream <string, string>("test") .Peek((k, v) => _return.Add(KeyValuePair.Create(k, v))); var t = builder.Build(); var stream = new KafkaStream(t, config, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } Assert.IsTrue(isRunningState); if (isRunningState) { var serdes = new StringSerDes(); dt = DateTime.Now; producer.Produce("test", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("k", new SerializationContext()), Value = serdes.Serialize("test", new SerializationContext()), Timestamp = new Confluent.Kafka.Timestamp(dt) }); producer.Produce("test", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("v", new SerializationContext()), Value = serdes.Serialize("test2", new SerializationContext()), Timestamp = new Confluent.Kafka.Timestamp(dt) }); Thread.Sleep(1000); var expected = new List <KeyValuePair <string, string> >(); expected.Add(KeyValuePair.Create("v", "test2")); Assert.AreEqual(expected, _return); } stream.Dispose(); }
public void StreamTaskWithEXACTLY_ONCE() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; config.Guarantee = ProcessingGuarantee.EXACTLY_ONCE; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); TaskId id = new TaskId { Id = 0, Partition = 0 }; var processorTopology = topology.Builder.BuildTopology(id); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var part = new TopicPartition("topic", 0); StreamTask task = new StreamTask( "thread-0", id, new List <TopicPartition> { part }, processorTopology, consumer, config, supplier, null); task.GroupMetadata = consumer as SyncConsumer; task.InitializeStateStores(); task.InitializeTopology(); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); Assert.IsTrue(task.CommitNeeded); task.Commit(); } // CHECK IN TOPIC topic2 consumer.Subscribe("topic2"); List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >(); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); if (result != null) { results.Add(result); consumer.Commit(result); } } while (result != null); Assert.AreEqual(5, results.Count); for (int i = 0; i < 5; ++i) { Assert.AreEqual($"KEY{i + 1}", serdes.Deserialize(results[i].Message.Key, new SerializationContext())); Assert.AreEqual($"VALUE{i + 1}", serdes.Deserialize(results[i].Message.Value, new SerializationContext())); } task.Close(); }