public void Init() { config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; mockKafkaSupplier = new MockKafkaSupplier(4); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); thread1 = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; }
public void Initialize() { streamMetricsRegistry = new StreamMetricsRegistry(Guid.NewGuid().ToString(), MetricsRecordingLevel.INFO); config.ApplicationId = "test-stream-thread"; config.StateDir = Guid.NewGuid().ToString(); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; config.MaxPollRecords = 10; config.CommitIntervalMs = 10; config.MetricsRecording = MetricsRecordingLevel.INFO; mockKafkaSupplier = new MockKafkaSupplier(numberPartitions); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); thread = StreamThread.Create( threadId, "c1", topo.Builder, streamMetricsRegistry, config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; }
public void CheckSetStateStartingWithDeadThread() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; // MUST BE IN CREATED STATE Assert.AreEqual(ThreadState.CREATED, thread.State); thread.SetState(ThreadState.STARTING); thread.SetState(ThreadState.PENDING_SHUTDOWN); thread.SetState(ThreadState.DEAD); thread.Start(default);
public void GetCurrentPartitionMetadataTests() { var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.FollowMetadata = true; var configConsumer = config.Clone(); configConsumer.ApplicationId = "test-consumer"; int?h = null; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder .Stream <string, string>("topic") .MapValues((v) => { h = StreamizMetadata.GetCurrentPartitionMetadata(); return(v); }) .To("output"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(configConsumer.ToConsumerConfig(), null); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); consumer.Subscribe("output"); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); } while (result == null); source.Cancel(); thread.Dispose(); Assert.NotNull(h); Assert.AreEqual(0, h); }
public void TaskManagerCommitWithoutCommitNeeed() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.CommitAll()); taskManager.Close(); }
public void CreateAdminClient() { var supplier = new DefaultKafkaClientSupplier(new KafkaLoggerAdapter(config)); var adminClient = supplier.GetAdmin(config.ToAdminConfig("admin")); Assert.IsNotNull(adminClient); Assert.AreEqual("admin", adminClient.Name.Split("#")[0]); }
public void StreamThreadCommitIntervalWorkflow() { var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.CommitIntervalMs = 1; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); var offsets = thread.GetCommittedOffsets(new List <TopicPartition> { new TopicPartition("topic", 0) }, TimeSpan.FromSeconds(10)).ToList(); Assert.AreEqual(1, offsets.Count); Assert.AreEqual(1, offsets[0].Offset.Value); Assert.AreEqual(0, offsets[0].TopicPartition.Partition.Value); Assert.AreEqual("topic", offsets[0].Topic); source.Cancel(); thread.Dispose(); }
public void TaskManagerAssignedUnknownPartitions() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer, storeChangelogReader, new StreamMetricsRegistry()); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1) }); taskManager.RevokeTasks( new List <TopicPartition> { new TopicPartition("topic", 1) }); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2) }); taskManager.TryToCompleteRestoration(); Assert.AreEqual(3, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); taskManager.Close(); }
public void Init() { token1 = new System.Threading.CancellationTokenSource(); token2 = new System.Threading.CancellationTokenSource(); config.ApplicationId = "test-stream-thread"; config.StateDir = Path.Combine(".", Guid.NewGuid().ToString()); config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 10; mockKafkaSupplier = new MockKafkaSupplier(2, 0); var builder = new StreamBuilder(); builder.Table("topic", InMemory <string, string> .As("store").WithLoggingEnabled()); var topo = builder.Build(); topo.Builder.RewriteTopology(config); topo.Builder.BuildTopology(); thread1 = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, new StreamMetricsRegistry(), config, mockKafkaSupplier, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; var internalTopicManager = new DefaultTopicManager(config, mockKafkaSupplier.GetAdmin(config.ToAdminConfig("admin"))); InternalTopicManagerUtils .New() .CreateInternalTopicsAsync(internalTopicManager, topo.Builder).GetAwaiter(); }
public void StreamAddCorrectConfig() { var stream = new StreamConfig(); stream.ApplicationId = "unittest"; stream.AddConfig("sasl.password", "coucou"); var adminConfig = stream.ToAdminConfig("admin"); var consumerConfig = stream.ToConsumerConfig(); var producerConfig = stream.ToProducerConfig(); var globalConfig = stream.ToGlobalConsumerConfig("global"); Assert.AreEqual("coucou", adminConfig.SaslPassword); Assert.AreEqual("coucou", consumerConfig.SaslPassword); Assert.AreEqual("coucou", producerConfig.SaslPassword); Assert.AreEqual("coucou", globalConfig.SaslPassword); }
public void CreateStreamThread() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; Assert.AreEqual("thread-0", thread.Name); }
public void CheckIncorrectStateTransition() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; // MUST BE IN CREATED STATE Assert.AreEqual(ThreadState.CREATED, thread.State); Assert.Throws <StreamsException>(() => thread.SetState(ThreadState.DEAD)); }
public void CheckSetStateWithoutStateChangedHandler() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; // MUST BE IN CREATED STATE Assert.AreEqual(ThreadState.CREATED, thread.State); thread.SetState(ThreadState.STARTING); Assert.AreEqual(ThreadState.STARTING, thread.State); }
public void StreamCompleteConfigAllProperty() { var stream = new StreamConfig(); stream.ApplicationId = "test"; stream.Acks = Confluent.Kafka.Acks.All; stream.ApiVersionFallbackMs = 1; stream.ApiVersionRequest = false; stream.ApiVersionRequestTimeoutMs = 100; stream.AutoOffsetReset = Confluent.Kafka.AutoOffsetReset.Latest; stream.BatchNumMessages = 42; stream.BootstrapServers = "127.0.0.1:9092"; stream.BrokerAddressFamily = Confluent.Kafka.BrokerAddressFamily.V4; stream.BrokerAddressTtl = 100; stream.BrokerVersionFallback = "0.12.0"; stream.CheckCrcs = true; stream.ClientId = "test-client"; stream.ClientRack = "1"; stream.CommitIntervalMs = 300; stream.CompressionLevel = 2; stream.CompressionType = Confluent.Kafka.CompressionType.Snappy; stream.ConsumeResultFields = "all"; stream.CoordinatorQueryIntervalMs = 300; stream.Debug = "all"; stream.DeliveryReportFields = "key"; stream.EnableAutoOffsetStore = false; stream.EnableBackgroundPoll = false; stream.EnableDeliveryReports = false; stream.EnableGaplessGuarantee = false; stream.EnableIdempotence = true; stream.EnablePartitionEof = true; stream.EnableSaslOauthbearerUnsecureJwt = true; stream.EnableSslCertificateVerification = false; stream.FetchErrorBackoffMs = 10; stream.FetchMaxBytes = 10; stream.FetchMinBytes = 10; stream.FetchWaitMaxMs = 10; stream.GroupProtocolType = "?"; stream.HeartbeatIntervalMs = 4000; stream.InternalTerminationSignal = 1; stream.IsolationLevel = Confluent.Kafka.IsolationLevel.ReadCommitted; stream.LingerMs = 12; stream.LogConnectionClose = false; stream.LogQueue = true; stream.LogThreadName = false; stream.MaxInFlight = 12; stream.MaxPartitionFetchBytes = 500; stream.MaxPollIntervalMs = 400; stream.MessageCopyMaxBytes = 40; stream.MessageMaxBytes = 500; stream.MessageSendMaxRetries = 4; stream.MessageTimeoutMs = 600; stream.MetadataMaxAgeMs = 6; stream.MetadataRequestTimeoutMs = 83; stream.PartitionAssignmentStrategy = Confluent.Kafka.PartitionAssignmentStrategy.RoundRobin; stream.Partitioner = Confluent.Kafka.Partitioner.Murmur2Random; stream.PluginLibraryPaths = "D:"; stream.QueueBufferingBackpressureThreshold = 10; stream.QueueBufferingMaxKbytes = 400; stream.QueueBufferingMaxMessages = 5; stream.QueuedMaxMessagesKbytes = 800; stream.QueuedMinMessages = 1; stream.ReceiveMessageMaxBytes = 1000; stream.ReconnectBackoffMaxMs = 9000; stream.ReconnectBackoffMs = 8000; stream.RequestTimeoutMs = 16600; stream.RetryBackoffMs = 600; stream.SaslKerberosKeytab = "test"; stream.SaslKerberosKinitCmd = "test"; stream.SaslKerberosMinTimeBeforeRelogin = 600; stream.SaslKerberosPrincipal = "Princiapl"; stream.SaslKerberosServiceName = "kerberos"; stream.SaslMechanism = Confluent.Kafka.SaslMechanism.ScramSha512; stream.SaslOauthbearerConfig = "ouath"; stream.SaslPassword = "******"; stream.SaslUsername = "******"; stream.SecurityProtocol = Confluent.Kafka.SecurityProtocol.SaslPlaintext; stream.SessionTimeoutMs = 1000; stream.SocketKeepaliveEnable = true; stream.SocketMaxFails = 2; stream.SocketNagleDisable = true; stream.SocketReceiveBufferBytes = 50000; stream.SocketSendBufferBytes = 50000; stream.SocketTimeoutMs = 6000; stream.SslCaLocation = "D:"; stream.SslCertificateLocation = "D:"; stream.SslCertificatePem = "D:"; stream.SslCipherSuites = "ciphers"; stream.SslCrlLocation = "D:"; stream.SslCurvesList = ""; stream.SslEndpointIdentificationAlgorithm = Confluent.Kafka.SslEndpointIdentificationAlgorithm.Https; stream.SslKeyLocation = "C:"; stream.SslKeyPassword = "******"; stream.SslKeyPem = "pem"; stream.SslKeystoreLocation = "J:"; stream.SslKeystorePassword = "******"; stream.SslSigalgsList = "oepn"; stream.StatisticsIntervalMs = 14; stream.TopicBlacklist = "*"; stream.TopicMetadataRefreshFastIntervalMs = 500; stream.TopicMetadataRefreshIntervalMs = 200; stream.TopicMetadataRefreshSparse = false; stream.TransactionalId = "transac"; stream.TransactionTimeout = TimeSpan.FromSeconds(1); stream.TransactionTimeoutMs = 400; var producerConfig = stream.ToProducerConfig(); var consumerConfig = stream.ToConsumerConfig(); var globalConfig = stream.ToGlobalConsumerConfig("global"); var adminConfig = stream.ToAdminConfig("admin"); #region ProducerConfig Assert.AreEqual(Confluent.Kafka.Acks.All, producerConfig.Acks); Assert.AreEqual(1, producerConfig.ApiVersionFallbackMs); Assert.AreEqual(false, producerConfig.ApiVersionRequest); Assert.AreEqual(100, producerConfig.ApiVersionRequestTimeoutMs); Assert.AreEqual(42, producerConfig.BatchNumMessages); Assert.AreEqual("127.0.0.1:9092", producerConfig.BootstrapServers); Assert.AreEqual(Confluent.Kafka.BrokerAddressFamily.V4, producerConfig.BrokerAddressFamily); Assert.AreEqual(100, producerConfig.BrokerAddressTtl); Assert.AreEqual("0.12.0", producerConfig.BrokerVersionFallback); Assert.AreEqual("test-client", producerConfig.ClientId); Assert.AreEqual("1", producerConfig.ClientRack); Assert.AreEqual(2, producerConfig.CompressionLevel); Assert.AreEqual(Confluent.Kafka.CompressionType.Snappy, producerConfig.CompressionType); Assert.AreEqual("all", producerConfig.Debug); Assert.AreEqual("key", producerConfig.DeliveryReportFields); Assert.AreEqual(false, producerConfig.EnableBackgroundPoll); Assert.AreEqual(false, producerConfig.EnableDeliveryReports); Assert.AreEqual(false, producerConfig.EnableGaplessGuarantee); Assert.AreEqual(true, producerConfig.EnableIdempotence); Assert.AreEqual(true, producerConfig.EnableSaslOauthbearerUnsecureJwt); Assert.AreEqual(false, producerConfig.EnableSslCertificateVerification); Assert.AreEqual(1, producerConfig.InternalTerminationSignal); Assert.AreEqual(12, producerConfig.LingerMs); Assert.AreEqual(false, producerConfig.LogConnectionClose); Assert.AreEqual(true, producerConfig.LogQueue); Assert.AreEqual(false, producerConfig.LogThreadName); Assert.AreEqual(12, producerConfig.MaxInFlight); Assert.AreEqual(40, producerConfig.MessageCopyMaxBytes); Assert.AreEqual(500, producerConfig.MessageMaxBytes); Assert.AreEqual(4, producerConfig.MessageSendMaxRetries); Assert.AreEqual(600, producerConfig.MessageTimeoutMs); Assert.AreEqual(6, producerConfig.MetadataMaxAgeMs); Assert.AreEqual(83, producerConfig.MetadataRequestTimeoutMs); Assert.AreEqual(Confluent.Kafka.Partitioner.Murmur2Random, producerConfig.Partitioner); Assert.AreEqual("D:", producerConfig.PluginLibraryPaths); Assert.AreEqual(10, producerConfig.QueueBufferingBackpressureThreshold); Assert.AreEqual(400, producerConfig.QueueBufferingMaxKbytes); Assert.AreEqual(5, producerConfig.QueueBufferingMaxMessages); Assert.AreEqual(1000, producerConfig.ReceiveMessageMaxBytes); Assert.AreEqual(9000, producerConfig.ReconnectBackoffMaxMs); Assert.AreEqual(8000, producerConfig.ReconnectBackoffMs); Assert.AreEqual(16600, producerConfig.RequestTimeoutMs); Assert.AreEqual(600, producerConfig.RetryBackoffMs); Assert.AreEqual("test", producerConfig.SaslKerberosKeytab); Assert.AreEqual("test", producerConfig.SaslKerberosKinitCmd); Assert.AreEqual(600, producerConfig.SaslKerberosMinTimeBeforeRelogin); Assert.AreEqual("Princiapl", producerConfig.SaslKerberosPrincipal); Assert.AreEqual("kerberos", producerConfig.SaslKerberosServiceName); Assert.AreEqual(Confluent.Kafka.SaslMechanism.ScramSha512, producerConfig.SaslMechanism); Assert.AreEqual("ouath", producerConfig.SaslOauthbearerConfig); Assert.AreEqual("test", producerConfig.SaslPassword); Assert.AreEqual("admin", producerConfig.SaslUsername); Assert.AreEqual(Confluent.Kafka.SecurityProtocol.SaslPlaintext, producerConfig.SecurityProtocol); Assert.AreEqual(true, producerConfig.SocketKeepaliveEnable); Assert.AreEqual(2, producerConfig.SocketMaxFails); Assert.AreEqual(true, producerConfig.SocketNagleDisable); Assert.AreEqual(50000, producerConfig.SocketReceiveBufferBytes); Assert.AreEqual(50000, producerConfig.SocketSendBufferBytes); Assert.AreEqual(6000, producerConfig.SocketTimeoutMs); Assert.AreEqual("D:", producerConfig.SslCaLocation); Assert.AreEqual("D:", producerConfig.SslCertificateLocation); Assert.AreEqual("D:", producerConfig.SslCertificatePem); Assert.AreEqual("ciphers", producerConfig.SslCipherSuites); Assert.AreEqual("D:", producerConfig.SslCrlLocation); Assert.AreEqual("", producerConfig.SslCurvesList); Assert.AreEqual(Confluent.Kafka.SslEndpointIdentificationAlgorithm.Https, producerConfig.SslEndpointIdentificationAlgorithm); Assert.AreEqual("C:", producerConfig.SslKeyLocation); Assert.AreEqual("test", producerConfig.SslKeyPassword); Assert.AreEqual("pem", producerConfig.SslKeyPem); Assert.AreEqual("J:", producerConfig.SslKeystoreLocation); Assert.AreEqual("password", producerConfig.SslKeystorePassword); Assert.AreEqual("oepn", producerConfig.SslSigalgsList); Assert.AreEqual(14, producerConfig.StatisticsIntervalMs); Assert.AreEqual("*", producerConfig.TopicBlacklist); Assert.AreEqual(500, producerConfig.TopicMetadataRefreshFastIntervalMs); Assert.AreEqual(200, producerConfig.TopicMetadataRefreshIntervalMs); Assert.AreEqual(false, producerConfig.TopicMetadataRefreshSparse); Assert.AreEqual("transac", producerConfig.TransactionalId); Assert.AreEqual(400, producerConfig.TransactionTimeoutMs); #endregion #region ConsumerConfig Assert.AreEqual(Confluent.Kafka.Acks.All, consumerConfig.Acks); Assert.AreEqual(1, consumerConfig.ApiVersionFallbackMs); Assert.AreEqual(false, consumerConfig.ApiVersionRequest); Assert.AreEqual(100, consumerConfig.ApiVersionRequestTimeoutMs); Assert.AreEqual(Confluent.Kafka.AutoOffsetReset.Latest, consumerConfig.AutoOffsetReset); Assert.AreEqual("127.0.0.1:9092", consumerConfig.BootstrapServers); Assert.AreEqual(Confluent.Kafka.BrokerAddressFamily.V4, consumerConfig.BrokerAddressFamily); Assert.AreEqual(100, consumerConfig.BrokerAddressTtl); Assert.AreEqual("0.12.0", consumerConfig.BrokerVersionFallback); Assert.AreEqual(true, consumerConfig.CheckCrcs); Assert.AreEqual("test-client", consumerConfig.ClientId); Assert.AreEqual("1", consumerConfig.ClientRack); Assert.AreEqual(300, consumerConfig.CoordinatorQueryIntervalMs); Assert.AreEqual("all", consumerConfig.Debug); Assert.AreEqual(false, consumerConfig.EnableAutoOffsetStore); Assert.AreEqual(true, consumerConfig.EnablePartitionEof); Assert.AreEqual(true, consumerConfig.EnableSaslOauthbearerUnsecureJwt); Assert.AreEqual(false, consumerConfig.EnableSslCertificateVerification); Assert.AreEqual(10, consumerConfig.FetchErrorBackoffMs); Assert.AreEqual(10, consumerConfig.FetchMaxBytes); Assert.AreEqual(10, consumerConfig.FetchMinBytes); Assert.AreEqual(10, consumerConfig.FetchWaitMaxMs); Assert.AreEqual("?", consumerConfig.GroupProtocolType); Assert.AreEqual(4000, consumerConfig.HeartbeatIntervalMs); Assert.AreEqual(1, consumerConfig.InternalTerminationSignal); Assert.AreEqual(Confluent.Kafka.IsolationLevel.ReadCommitted, consumerConfig.IsolationLevel); Assert.AreEqual(false, consumerConfig.LogConnectionClose); Assert.AreEqual(true, consumerConfig.LogQueue); Assert.AreEqual(false, consumerConfig.LogThreadName); Assert.AreEqual(12, consumerConfig.MaxInFlight); Assert.AreEqual(500, consumerConfig.MaxPartitionFetchBytes); Assert.AreEqual(400, consumerConfig.MaxPollIntervalMs); Assert.AreEqual(40, consumerConfig.MessageCopyMaxBytes); Assert.AreEqual(500, consumerConfig.MessageMaxBytes); Assert.AreEqual(6, consumerConfig.MetadataMaxAgeMs); Assert.AreEqual(83, consumerConfig.MetadataRequestTimeoutMs); Assert.AreEqual(Confluent.Kafka.PartitionAssignmentStrategy.RoundRobin, consumerConfig.PartitionAssignmentStrategy); Assert.AreEqual("D:", consumerConfig.PluginLibraryPaths); Assert.AreEqual(800, consumerConfig.QueuedMaxMessagesKbytes); Assert.AreEqual(1, consumerConfig.QueuedMinMessages); Assert.AreEqual(1000, consumerConfig.ReceiveMessageMaxBytes); Assert.AreEqual(9000, consumerConfig.ReconnectBackoffMaxMs); Assert.AreEqual(8000, consumerConfig.ReconnectBackoffMs); Assert.AreEqual("test", consumerConfig.SaslKerberosKeytab); Assert.AreEqual("test", consumerConfig.SaslKerberosKinitCmd); Assert.AreEqual(600, consumerConfig.SaslKerberosMinTimeBeforeRelogin); Assert.AreEqual("Princiapl", consumerConfig.SaslKerberosPrincipal); Assert.AreEqual("kerberos", consumerConfig.SaslKerberosServiceName); Assert.AreEqual(Confluent.Kafka.SaslMechanism.ScramSha512, consumerConfig.SaslMechanism); Assert.AreEqual("ouath", consumerConfig.SaslOauthbearerConfig); Assert.AreEqual("test", consumerConfig.SaslPassword); Assert.AreEqual("admin", consumerConfig.SaslUsername); Assert.AreEqual(Confluent.Kafka.SecurityProtocol.SaslPlaintext, consumerConfig.SecurityProtocol); Assert.AreEqual(1000, consumerConfig.SessionTimeoutMs); Assert.AreEqual(true, consumerConfig.SocketKeepaliveEnable); Assert.AreEqual(2, consumerConfig.SocketMaxFails); Assert.AreEqual(true, consumerConfig.SocketNagleDisable); Assert.AreEqual(50000, consumerConfig.SocketReceiveBufferBytes); Assert.AreEqual(50000, consumerConfig.SocketSendBufferBytes); Assert.AreEqual(6000, consumerConfig.SocketTimeoutMs); Assert.AreEqual("D:", consumerConfig.SslCaLocation); Assert.AreEqual("D:", consumerConfig.SslCertificateLocation); Assert.AreEqual("D:", consumerConfig.SslCertificatePem); Assert.AreEqual("ciphers", consumerConfig.SslCipherSuites); Assert.AreEqual("D:", consumerConfig.SslCrlLocation); Assert.AreEqual("", consumerConfig.SslCurvesList); Assert.AreEqual(Confluent.Kafka.SslEndpointIdentificationAlgorithm.Https, consumerConfig.SslEndpointIdentificationAlgorithm); Assert.AreEqual("C:", consumerConfig.SslKeyLocation); Assert.AreEqual("test", consumerConfig.SslKeyPassword); Assert.AreEqual("pem", consumerConfig.SslKeyPem); Assert.AreEqual("J:", consumerConfig.SslKeystoreLocation); Assert.AreEqual("password", consumerConfig.SslKeystorePassword); Assert.AreEqual("oepn", consumerConfig.SslSigalgsList); Assert.AreEqual(14, consumerConfig.StatisticsIntervalMs); Assert.AreEqual("*", consumerConfig.TopicBlacklist); Assert.AreEqual(500, consumerConfig.TopicMetadataRefreshFastIntervalMs); Assert.AreEqual(200, consumerConfig.TopicMetadataRefreshIntervalMs); Assert.AreEqual(false, consumerConfig.TopicMetadataRefreshSparse); #endregion #region GlobalConfig Assert.AreEqual(Confluent.Kafka.Acks.All, globalConfig.Acks); Assert.AreEqual(1, globalConfig.ApiVersionFallbackMs); Assert.AreEqual(false, globalConfig.ApiVersionRequest); Assert.AreEqual(100, globalConfig.ApiVersionRequestTimeoutMs); Assert.AreEqual(Confluent.Kafka.AutoOffsetReset.Latest, globalConfig.AutoOffsetReset); Assert.AreEqual("127.0.0.1:9092", globalConfig.BootstrapServers); Assert.AreEqual(Confluent.Kafka.BrokerAddressFamily.V4, globalConfig.BrokerAddressFamily); Assert.AreEqual(100, globalConfig.BrokerAddressTtl); Assert.AreEqual("0.12.0", globalConfig.BrokerVersionFallback); Assert.AreEqual(true, globalConfig.CheckCrcs); Assert.AreEqual("global", globalConfig.ClientId); Assert.AreEqual("1", globalConfig.ClientRack); Assert.AreEqual(300, globalConfig.CoordinatorQueryIntervalMs); Assert.AreEqual("all", globalConfig.Debug); Assert.AreEqual(false, globalConfig.EnableAutoOffsetStore); Assert.AreEqual(true, globalConfig.EnablePartitionEof); Assert.AreEqual(true, globalConfig.EnableSaslOauthbearerUnsecureJwt); Assert.AreEqual(false, globalConfig.EnableSslCertificateVerification); Assert.AreEqual(10, globalConfig.FetchErrorBackoffMs); Assert.AreEqual(10, globalConfig.FetchMaxBytes); Assert.AreEqual(10, globalConfig.FetchMinBytes); Assert.AreEqual(10, globalConfig.FetchWaitMaxMs); Assert.AreEqual("?", globalConfig.GroupProtocolType); Assert.AreEqual(4000, globalConfig.HeartbeatIntervalMs); Assert.AreEqual(1, globalConfig.InternalTerminationSignal); Assert.AreEqual(Confluent.Kafka.IsolationLevel.ReadCommitted, globalConfig.IsolationLevel); Assert.AreEqual(false, globalConfig.LogConnectionClose); Assert.AreEqual(true, globalConfig.LogQueue); Assert.AreEqual(false, globalConfig.LogThreadName); Assert.AreEqual(12, globalConfig.MaxInFlight); Assert.AreEqual(500, globalConfig.MaxPartitionFetchBytes); Assert.AreEqual(400, globalConfig.MaxPollIntervalMs); Assert.AreEqual(40, globalConfig.MessageCopyMaxBytes); Assert.AreEqual(500, globalConfig.MessageMaxBytes); Assert.AreEqual(6, globalConfig.MetadataMaxAgeMs); Assert.AreEqual(83, globalConfig.MetadataRequestTimeoutMs); Assert.AreEqual(Confluent.Kafka.PartitionAssignmentStrategy.RoundRobin, globalConfig.PartitionAssignmentStrategy); Assert.AreEqual("D:", globalConfig.PluginLibraryPaths); Assert.AreEqual(800, globalConfig.QueuedMaxMessagesKbytes); Assert.AreEqual(1, globalConfig.QueuedMinMessages); Assert.AreEqual(1000, globalConfig.ReceiveMessageMaxBytes); Assert.AreEqual(9000, globalConfig.ReconnectBackoffMaxMs); Assert.AreEqual(8000, globalConfig.ReconnectBackoffMs); Assert.AreEqual("test", globalConfig.SaslKerberosKeytab); Assert.AreEqual("test", globalConfig.SaslKerberosKinitCmd); Assert.AreEqual(600, globalConfig.SaslKerberosMinTimeBeforeRelogin); Assert.AreEqual("Princiapl", globalConfig.SaslKerberosPrincipal); Assert.AreEqual("kerberos", globalConfig.SaslKerberosServiceName); Assert.AreEqual(Confluent.Kafka.SaslMechanism.ScramSha512, globalConfig.SaslMechanism); Assert.AreEqual("ouath", globalConfig.SaslOauthbearerConfig); Assert.AreEqual("test", globalConfig.SaslPassword); Assert.AreEqual("admin", globalConfig.SaslUsername); Assert.AreEqual(Confluent.Kafka.SecurityProtocol.SaslPlaintext, globalConfig.SecurityProtocol); Assert.AreEqual(1000, globalConfig.SessionTimeoutMs); Assert.AreEqual(true, globalConfig.SocketKeepaliveEnable); Assert.AreEqual(2, globalConfig.SocketMaxFails); Assert.AreEqual(true, globalConfig.SocketNagleDisable); Assert.AreEqual(50000, globalConfig.SocketReceiveBufferBytes); Assert.AreEqual(50000, globalConfig.SocketSendBufferBytes); Assert.AreEqual(6000, globalConfig.SocketTimeoutMs); Assert.AreEqual("D:", globalConfig.SslCaLocation); Assert.AreEqual("D:", globalConfig.SslCertificateLocation); Assert.AreEqual("D:", globalConfig.SslCertificatePem); Assert.AreEqual("ciphers", globalConfig.SslCipherSuites); Assert.AreEqual("D:", globalConfig.SslCrlLocation); Assert.AreEqual("", globalConfig.SslCurvesList); Assert.AreEqual(Confluent.Kafka.SslEndpointIdentificationAlgorithm.Https, globalConfig.SslEndpointIdentificationAlgorithm); Assert.AreEqual("C:", globalConfig.SslKeyLocation); Assert.AreEqual("test", globalConfig.SslKeyPassword); Assert.AreEqual("pem", globalConfig.SslKeyPem); Assert.AreEqual("J:", globalConfig.SslKeystoreLocation); Assert.AreEqual("password", globalConfig.SslKeystorePassword); Assert.AreEqual("oepn", globalConfig.SslSigalgsList); Assert.AreEqual(14, globalConfig.StatisticsIntervalMs); Assert.AreEqual("*", globalConfig.TopicBlacklist); Assert.AreEqual(500, globalConfig.TopicMetadataRefreshFastIntervalMs); Assert.AreEqual(200, globalConfig.TopicMetadataRefreshIntervalMs); Assert.AreEqual(false, globalConfig.TopicMetadataRefreshSparse); #endregion #region AdminConfig Assert.AreEqual(Confluent.Kafka.Acks.All, adminConfig.Acks); Assert.AreEqual(1, adminConfig.ApiVersionFallbackMs); Assert.AreEqual(false, adminConfig.ApiVersionRequest); Assert.AreEqual(100, adminConfig.ApiVersionRequestTimeoutMs); Assert.AreEqual("127.0.0.1:9092", adminConfig.BootstrapServers); Assert.AreEqual(Confluent.Kafka.BrokerAddressFamily.V4, adminConfig.BrokerAddressFamily); Assert.AreEqual(100, adminConfig.BrokerAddressTtl); Assert.AreEqual("0.12.0", adminConfig.BrokerVersionFallback); Assert.AreEqual("admin", adminConfig.ClientId); Assert.AreEqual("1", adminConfig.ClientRack); Assert.AreEqual("all", adminConfig.Debug); Assert.AreEqual(true, adminConfig.EnableSaslOauthbearerUnsecureJwt); Assert.AreEqual(false, adminConfig.EnableSslCertificateVerification); Assert.AreEqual(1, adminConfig.InternalTerminationSignal); Assert.AreEqual(false, adminConfig.LogConnectionClose); Assert.AreEqual(true, adminConfig.LogQueue); Assert.AreEqual(false, adminConfig.LogThreadName); Assert.AreEqual(12, adminConfig.MaxInFlight); Assert.AreEqual(40, adminConfig.MessageCopyMaxBytes); Assert.AreEqual(500, adminConfig.MessageMaxBytes); Assert.AreEqual(6, adminConfig.MetadataMaxAgeMs); Assert.AreEqual(83, adminConfig.MetadataRequestTimeoutMs); Assert.AreEqual("D:", adminConfig.PluginLibraryPaths); Assert.AreEqual(1000, adminConfig.ReceiveMessageMaxBytes); Assert.AreEqual(9000, adminConfig.ReconnectBackoffMaxMs); Assert.AreEqual(8000, adminConfig.ReconnectBackoffMs); Assert.AreEqual("test", adminConfig.SaslKerberosKeytab); Assert.AreEqual("test", adminConfig.SaslKerberosKinitCmd); Assert.AreEqual(600, adminConfig.SaslKerberosMinTimeBeforeRelogin); Assert.AreEqual("Princiapl", adminConfig.SaslKerberosPrincipal); Assert.AreEqual("kerberos", adminConfig.SaslKerberosServiceName); Assert.AreEqual(Confluent.Kafka.SaslMechanism.ScramSha512, adminConfig.SaslMechanism); Assert.AreEqual("ouath", adminConfig.SaslOauthbearerConfig); Assert.AreEqual("test", adminConfig.SaslPassword); Assert.AreEqual("admin", adminConfig.SaslUsername); Assert.AreEqual(Confluent.Kafka.SecurityProtocol.SaslPlaintext, adminConfig.SecurityProtocol); Assert.AreEqual(true, adminConfig.SocketKeepaliveEnable); Assert.AreEqual(2, adminConfig.SocketMaxFails); Assert.AreEqual(true, adminConfig.SocketNagleDisable); Assert.AreEqual(50000, adminConfig.SocketReceiveBufferBytes); Assert.AreEqual(50000, adminConfig.SocketSendBufferBytes); Assert.AreEqual(6000, adminConfig.SocketTimeoutMs); Assert.AreEqual("D:", adminConfig.SslCaLocation); Assert.AreEqual("D:", adminConfig.SslCertificateLocation); Assert.AreEqual("D:", adminConfig.SslCertificatePem); Assert.AreEqual("ciphers", adminConfig.SslCipherSuites); Assert.AreEqual("D:", adminConfig.SslCrlLocation); Assert.AreEqual("", adminConfig.SslCurvesList); Assert.AreEqual(Confluent.Kafka.SslEndpointIdentificationAlgorithm.Https, adminConfig.SslEndpointIdentificationAlgorithm); Assert.AreEqual("C:", adminConfig.SslKeyLocation); Assert.AreEqual("test", adminConfig.SslKeyPassword); Assert.AreEqual("pem", adminConfig.SslKeyPem); Assert.AreEqual("J:", adminConfig.SslKeystoreLocation); Assert.AreEqual("password", adminConfig.SslKeystorePassword); Assert.AreEqual("oepn", adminConfig.SslSigalgsList); Assert.AreEqual(14, adminConfig.StatisticsIntervalMs); Assert.AreEqual("*", adminConfig.TopicBlacklist); Assert.AreEqual(500, adminConfig.TopicMetadataRefreshFastIntervalMs); Assert.AreEqual(200, adminConfig.TopicMetadataRefreshIntervalMs); Assert.AreEqual(false, adminConfig.TopicMetadataRefreshSparse); #endregion }
public void TaskManagerCommit() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic") .Map((k, v) => KeyValuePair.Create(k.ToUpper(), v.ToUpper())) .To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); var part = new TopicPartition("topic", 0); var task = taskManager.ActiveTaskFor(part); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); Assert.IsTrue(task.CanProcess(DateTime.Now.GetMilliseconds())); while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } // ONLY ONE TASK HAVE BEEN RECORDS Assert.AreEqual(1, taskManager.CommitAll()); // CHECK IN TOPIC topic2 consumer.Subscribe("topic2"); List <ConsumeResult <byte[], byte[]> > results = new List <ConsumeResult <byte[], byte[]> >(); ConsumeResult <byte[], byte[]> result = null; do { result = consumer.Consume(100); if (result != null) { results.Add(result); consumer.Commit(result); } } while (result != null); Assert.AreEqual(5, results.Count); for (int i = 0; i < 5; ++i) { Assert.AreEqual($"KEY{i + 1}", serdes.Deserialize(results[i].Message.Key, new SerializationContext())); Assert.AreEqual($"VALUE{i+1}", serdes.Deserialize(results[i].Message.Value, new SerializationContext())); } // NO RECORD IN THIS TASKS part = new TopicPartition("topic", 2); task = taskManager.ActiveTaskFor(part); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.Process()); taskManager.Close(); }
public async Task exec(IConfiguration config, IServiceProvider services) { Console.WriteLine("Process"); var destTopic = config["spring.cloud.stream.bindings.output.destination"]; Console.WriteLine(destTopic); using (var scope = services.CreateScope()) { this._dataService = scope.ServiceProvider .GetRequiredService <IDataService>(); bool isRunningState = false; var timeout = TimeSpan.FromSeconds(10); DateTime dt = DateTime.Now; Order[] capture = this._dataService.readData(); // Inyectamos los datos obtenidos al Stream var sConfig = new StreamConfig <StringSerDes, StringSerDes>(); sConfig.ApplicationId = config["SPRING_CLOUD_APPLICATION_GUID"]; sConfig.BootstrapServers = config["SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS"]; sConfig.SchemaRegistryUrl = config["SchemaRegistryUrl"]; sConfig.AutoRegisterSchemas = true; sConfig.NumStreamThreads = 10; sConfig.Acks = Acks.All; sConfig.AddConsumerConfig("allow.auto.create.topics", "true"); sConfig.InnerExceptionHandler = (e) => ExceptionHandlerResponse.CONTINUE; var schemaRegistryClient = new CachedSchemaRegistryClient (new SchemaRegistryConfig { Url = sConfig.SchemaRegistryUrl }); var supplier = new SyncKafkaSupplier(new KafkaLoggerAdapter(sConfig)); var producerConfig = sConfig.ToProducerConfig(); var adminConfig = sConfig.ToAdminConfig(sConfig.ApplicationId); var admin = supplier.GetAdmin(adminConfig); // try // { // var topic = new TopicSpecification // { // Name = destTopic, // NumPartitions = 1, // ReplicationFactor = 3 // }; // var topicProduct = new TopicSpecification // { // Name = "product-external", // NumPartitions = 1, // ReplicationFactor = 3 // }; // IList<TopicSpecification> topics = new List<TopicSpecification>(); // topics.Add(topic); // topics.Add(topicProduct); // await admin.CreateTopicsAsync(topics); // } // catch (Exception topicExists) // { // Console.WriteLine("Topic alreade exists"); // Console.Write(topicExists); // } var producer = supplier.GetProducer(producerConfig); StreamBuilder builder = new StreamBuilder(); var serdes = new SchemaAvroSerDes <Order>(); var keySerdes = new Int32SerDes(); builder.Table(destTopic, keySerdes, serdes, InMemory <int, Order> .As(config["table"])); var t = builder.Build(); KafkaStream stream = new KafkaStream(t, sConfig, supplier); stream.StateChanged += (old, @new) => { if (@new.Equals(KafkaStream.State.RUNNING)) { isRunningState = true; } }; await stream.StartAsync(); while (!isRunningState) { Thread.Sleep(250); if (DateTime.Now > dt + timeout) { break; } } if (isRunningState) { // //create a well formatted Endpoint in external topic var endpProducer = new ProducerBuilder <byte[], Endpoint>(producerConfig) .SetValueSerializer(new AvroSerializer <Endpoint>(schemaRegistryClient, new AvroSerializerConfig { AutoRegisterSchemas = true }).AsSyncOverAsync()).Build(); //create a well formatted Product in external topic var productProducer = new ProducerBuilder <byte[], Product>(producerConfig) .SetValueSerializer(new AvroSerializer <Product>(schemaRegistryClient, new AvroSerializerConfig { AutoRegisterSchemas = true }).AsSyncOverAsync()).Build(); for (int k = 1; k < 10; k++) { endpProducer.Produce("api-endpoints", new Message <byte[], Endpoint> { Key = new Int32SerDes().Serialize(k, new SerializationContext()), Value = new Endpoint { endpoint_id = ("endpoint" + k), endpoint_url = ("http://endpoint" + k + "/"), http_method = "POST" } }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Endpoint Message sent !"); } }); productProducer.Produce("product-external", new Message <byte[], Product> { Key = new Int32SerDes().Serialize(1, new SerializationContext()), Value = new Product { name = "Producto de Software", price = 1234.5F, product_id = 3 } }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Product Message sent !"); } }); } Thread.Sleep(10); for (int k = 1; k < 10; k++) { producer.Produce(destTopic, new Confluent.Kafka.Message <byte[], byte[]> { Key = keySerdes.Serialize(k, new SerializationContext()), Value = serdes.Serialize(new Order { order_id = k, price = 123.5F, product_id = k }, new SerializationContext()) }, (d) => { if (d.Status == PersistenceStatus.Persisted) { Console.WriteLine("Order Message sent !"); } }); } Thread.Sleep(50); } } }
public void StandardWorkflowTaskManager() { var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-app"; var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topology = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer); taskManager.CreateTasks( new List <TopicPartition> { new TopicPartition("topic", 0), new TopicPartition("topic", 1), new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(4, taskManager.ActiveTasks.Count()); for (int i = 0; i < 4; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } // Revoked 2 partitions taskManager.RevokeTasks(new List <TopicPartition> { new TopicPartition("topic", 2), new TopicPartition("topic", 3), }); Assert.AreEqual(2, taskManager.ActiveTasks.Count()); Assert.AreEqual(2, taskManager.RevokedTasks.Count()); for (int i = 0; i < 2; ++i) { var task = taskManager.ActiveTaskFor(new TopicPartition("topic", i)); Assert.IsNotNull(task); Assert.AreEqual("test-app", task.ApplicationId); Assert.IsFalse(task.CanProcess(DateTime.Now.GetMilliseconds())); Assert.IsFalse(task.CommitNeeded); Assert.IsFalse(task.HasStateStores); } var taskFailed = taskManager.ActiveTaskFor(new TopicPartition("topic", 2)); Assert.IsNull(taskFailed); taskManager.Close(); Assert.AreEqual(0, taskManager.ActiveTasks.Count()); Assert.AreEqual(0, taskManager.RevokedTasks.Count()); }
//[Test] // TODO : fix that public void WorkflowCompleteBufferedRecordsTest() { int maxBuffered = 10; var token = new System.Threading.CancellationTokenSource(); var serdes = new StringSerDes(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-group"; config.MaxTaskIdleMs = (long)TimeSpan.FromSeconds(100).TotalMilliseconds; config.BufferedRecordsPerPartition = maxBuffered; config.PollMs = 10; var builder = new StreamBuilder(); var stream1 = builder.Stream <string, string>("topic1"); var stream2 = builder.Stream <string, string>("topic2"); stream1 .Join(stream2, (v1, v2) => $"{v1}-{v2}", JoinWindowOptions.Of(TimeSpan.FromSeconds(10))) .To("output"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("output"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; thread.Start(token.Token); for (int i = 0; i < maxBuffered + 1; ++i) { producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{i}", new SerializationContext()) }); } // CONSUME PAUSE AFTER maxBuffered + 1 messages System.Threading.Thread.Sleep(50); // Add one message more with consumer in stream thread in pause producer.Produce("topic1", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"coucou{maxBuffered+1}", new SerializationContext()) }); Assert.AreEqual(1, thread.ActiveTasks.Count()); var task = thread.ActiveTasks.ToArray()[0]; Assert.IsNotNull(task.Grouper); Assert.IsFalse(task.Grouper.AllPartitionsBuffered); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered()); Assert.AreEqual(maxBuffered + 1, task.Grouper.NumBuffered(new TopicPartition("topic1", 0))); Assert.AreEqual(0, task.Grouper.NumBuffered(new TopicPartition("topic2", 0))); producer.Produce("topic2", new Message <byte[], byte[]> { Key = serdes.Serialize("key", new SerializationContext()), Value = serdes.Serialize($"test", new SerializationContext()) }); List <ConsumeResult <byte[], byte[]> > records = new List <ConsumeResult <byte[], byte[]> >(); do { records.AddRange(consumer.ConsumeRecords(TimeSpan.FromMilliseconds(100)).ToList()); } while (records.Count() <= 12); Assert.AreEqual(maxBuffered + 2, records.Count()); for (int i = 0; i < maxBuffered + 2; ++i) { var message = records.ToArray()[i]; Assert.AreEqual("key", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.IsTrue(serdes.Deserialize(message.Message.Value, new SerializationContext()).Contains($"coucou{i}-")); } token.Cancel(); thread.Dispose(); }
public void StreamThreadNormalWorkflowWithRebalancing() { List <ThreadState> allStates = new List <ThreadState>(); var expectedStates = new List <ThreadState> { ThreadState.CREATED, ThreadState.STARTING, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PARTITIONS_REVOKED, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PENDING_SHUTDOWN, ThreadState.DEAD }; var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; var consumeConfig = config.Clone(); consumeConfig.ApplicationId = "consume-test"; var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new MockKafkaSupplier(4); var producer = supplier.GetProducer(consumeConfig.ToProducerConfig()); var consumer = supplier.GetConsumer(consumeConfig.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; allStates.Add(thread.State); thread.StateChanged += (t, o, n) => { Assert.IsInstanceOf <ThreadState>(n); allStates.Add(n as ThreadState); }; thread.Start(source.Token); // WAIT PARTITONS ASSIGNED System.Threading.Thread.Sleep(50); var thread2 = StreamThread.Create( "thread-1", "c1", topo.Builder, config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 1) as StreamThread; thread2.Start(source.Token); // WAIT PARTITONS REBALANCING System.Threading.Thread.Sleep(50); producer.Produce("topic", new Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); // 2 CONSUMER FOR THE SAME GROUP ID => TOPIC WITH 4 PARTITIONS Assert.AreEqual(2, thread.ActiveTasks.Count()); Assert.AreEqual(2, thread2.ActiveTasks.Count()); source.Cancel(); thread.Dispose(); thread2.Dispose(); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); Assert.AreEqual(expectedStates, allStates); // Destroy in memory cluster supplier.Destroy(); }
public void StreamThreadNormalWorkflow() { bool metricsReporterCalled = false; List <ThreadState> allStates = new List <ThreadState>(); var expectedStates = new List <ThreadState> { ThreadState.CREATED, ThreadState.STARTING, ThreadState.PARTITIONS_ASSIGNED, ThreadState.RUNNING, ThreadState.PENDING_SHUTDOWN, ThreadState.DEAD }; var source = new System.Threading.CancellationTokenSource(); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test"; config.Guarantee = ProcessingGuarantee.AT_LEAST_ONCE; config.PollMs = 1; config.MetricsReporter = (sensor) => { metricsReporterCalled = true; }; config.AddOrUpdate(StreamConfig.metricsIntervalMsCst, 10); var serdes = new StringSerDes(); var builder = new StreamBuilder(); builder.Stream <string, string>("topic").To("topic2"); var topo = builder.Build(); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig("test-consum"), null); consumer.Subscribe("topic2"); var thread = StreamThread.Create( "thread-0", "c0", topo.Builder, new StreamMetricsRegistry(), config, supplier, supplier.GetAdmin(config.ToAdminConfig("admin")), 0) as StreamThread; allStates.Add(thread.State); thread.StateChanged += (t, o, n) => { Assert.IsInstanceOf <ThreadState>(n); allStates.Add(n as ThreadState); }; thread.Start(source.Token); producer.Produce("topic", new Confluent.Kafka.Message <byte[], byte[]> { Key = serdes.Serialize("key1", new SerializationContext()), Value = serdes.Serialize("coucou", new SerializationContext()) }); //WAIT STREAMTHREAD PROCESS MESSAGE System.Threading.Thread.Sleep(100); var message = consumer.Consume(100); source.Cancel(); thread.Dispose(); Assert.AreEqual("key1", serdes.Deserialize(message.Message.Key, new SerializationContext())); Assert.AreEqual("coucou", serdes.Deserialize(message.Message.Value, new SerializationContext())); Assert.AreEqual(expectedStates, allStates); Assert.IsTrue(metricsReporterCalled); }
private void TaskManagerRestorationChangelog(bool persistenStateStore = false) { var stateDir = Path.Combine(".", Guid.NewGuid().ToString()); var config = new StreamConfig <StringSerDes, StringSerDes>(); config.ApplicationId = "test-restoration-changelog-app"; config.StateDir = stateDir; var builder = new StreamBuilder(); builder.Table("topic", persistenStateStore ? RocksDb <string, string> .As("store").WithLoggingEnabled(null) : InMemory <string, string> .As("store").WithLoggingEnabled(null)); var serdes = new StringSerDes(); var topology = builder.Build(); topology.Builder.RewriteTopology(config); var supplier = new SyncKafkaSupplier(); var producer = supplier.GetProducer(config.ToProducerConfig()); var consumer = supplier.GetConsumer(config.ToConsumerConfig(), null); var restoreConsumer = supplier.GetRestoreConsumer(config.ToConsumerConfig()); var storeChangelogReader = new StoreChangelogReader(config, restoreConsumer, "thread-0", new StreamMetricsRegistry()); var taskCreator = new TaskCreator(topology.Builder, config, "thread-0", supplier, producer, storeChangelogReader, new StreamMetricsRegistry()); var taskManager = new TaskManager(topology.Builder, taskCreator, supplier.GetAdmin(config.ToAdminConfig("admin")), consumer, storeChangelogReader); var part = new TopicPartition("topic", 0); taskManager.CreateTasks( new List <TopicPartition> { part }); var task = taskManager.ActiveTaskFor(part); IDictionary <TaskId, ITask> tasks = new Dictionary <TaskId, ITask>(); tasks.Add(task.Id, task); taskManager.TryToCompleteRestoration(); storeChangelogReader.Restore(); Assert.IsTrue(taskManager.TryToCompleteRestoration()); List <ConsumeResult <byte[], byte[]> > messages = new List <ConsumeResult <byte[], byte[]> >(); int offset = 0; for (int i = 0; i < 5; ++i) { messages.Add( new ConsumeResult <byte[], byte[]> { Message = new Message <byte[], byte[]> { Key = serdes.Serialize($"key{i + 1}", new SerializationContext()), Value = serdes.Serialize($"value{i + 1}", new SerializationContext()) }, TopicPartitionOffset = new TopicPartitionOffset(part, offset++) }); } task.AddRecords(messages); // Process messages while (task.CanProcess(DateTime.Now.GetMilliseconds())) { Assert.IsTrue(task.Process()); } taskManager.CommitAll(); // Simulate Close + new open taskManager.Close(); restoreConsumer.Resume(new TopicPartition("test-restoration-changelog-app-store-changelog", 0).ToSingle()); taskManager.CreateTasks( new List <TopicPartition> { part }); task = taskManager.ActiveTaskFor(part); tasks = new Dictionary <TaskId, ITask>(); tasks.Add(task.Id, task); Assert.IsFalse(taskManager.TryToCompleteRestoration()); storeChangelogReader.Restore(); Assert.IsTrue(taskManager.TryToCompleteRestoration()); var store = task.GetStore("store"); var items = (store as ITimestampedKeyValueStore <string, string>).All().ToList(); Assert.AreEqual(5, items.Count); taskManager.Close(); if (persistenStateStore) { Directory.Delete(stateDir, true); } }