High-level API configuration for the producer
Inheritance: ZKConfig, ISyncProducerConfigShared, IAsyncProducerConfigShared
 public AsyncProducerConfig(ProducerConfig config, string host, int port)
     : base(config, host, port)
 {
     this.QueueBufferingMaxMs = config.QueueBufferingMaxMs;
     this.QueueBufferingMaxMessages = config.QueueBufferingMaxMessages;
     this.QueueEnqueueTimeoutMs = config.QueueEnqueueTimeoutMs;
     this.BatchNumMessages = config.BatchNumMessages;
     this.Serializer = config.Serializer;
     this.KeySerializer = config.KeySerializer;
 }
        public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<string, SortedSet<Partition>> mappings;
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                WaitUntillIdle(client, 500);
                var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null);
                client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener);
                client.CreatePersistent(brokerPath, true);
                client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                WaitUntillIdle(client, 500);
                Assert.IsTrue(brokers.ContainsKey(2345));
                client.DeleteRecursive(brokerPath);
                WaitUntillIdle(client, 500);
                Assert.IsFalse(brokers.ContainsKey(2345));
            }
        }
        public void WhenSessionIsExpiredListenerRecreatesEphemeralNodes()
        {
            {
                var producerConfig = new ProducerConfig(clientConfig);
                IDictionary<string, SortedSet<Partition>> mappings;
                IDictionary<int, Broker> brokers;
                IDictionary<string, SortedSet<Partition>> mappings2;
                IDictionary<int, Broker> brokers2;
                using (IZooKeeperClient client = new ZooKeeperClient(
                    producerConfig.ZkConnect,
                    producerConfig.ZkSessionTimeoutMs,
                    ZooKeeperStringSerializer.Serializer))
                {
                    using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                    {
                        brokers = brokerPartitionInfo.GetAllBrokerInfo();
                        mappings =
                            ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                                "topicBrokerPartitions", brokerPartitionInfo);
                        Assert.NotNull(brokers);
                        Assert.Greater(brokers.Count, 0);
                        Assert.NotNull(mappings);
                        Assert.Greater(mappings.Count, 0);
                        client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                        WaitUntillIdle(client, 3000);
                        brokers2 = brokerPartitionInfo.GetAllBrokerInfo();
                        mappings2 =
                            ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                                "topicBrokerPartitions", brokerPartitionInfo);
                    }
                }

                Assert.NotNull(brokers2);
                Assert.Greater(brokers2.Count, 0);
                Assert.NotNull(mappings2);
                Assert.Greater(mappings2.Count, 0);
                Assert.AreEqual(brokers.Count, brokers2.Count);
                Assert.AreEqual(mappings.Count, mappings2.Count);
            }
        }
        public void WhenNewTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<string, SortedSet<Partition>> mappings;
            string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;

            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                    client.CreatePersistent(topicPath, true);
                    WaitUntillIdle(client, 500);
                    client.UnsubscribeAll();
                    client.DeleteRecursive(topicPath);
                }
            }

            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
        }
        public void WhenNewBrokerInTopicIsAddedZKBrokerPartitionInfoUpdatesMappings()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<string, SortedSet<Partition>> mappings;
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic;
            string topicBrokerPath = topicPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    mappings =
                        ReflectionHelper.GetInstanceField<IDictionary<string, SortedSet<Partition>>>(
                            "topicBrokerPartitions", brokerPartitionInfo);
                    client.CreatePersistent(brokerPath, true);
                    client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                    client.CreatePersistent(topicPath, true);
                    WaitUntillIdle(client, 500);
                    Assert.IsTrue(brokers.ContainsKey(2345));
                    Assert.IsTrue(mappings.ContainsKey(CurrentTestTopic));
                    client.CreatePersistent(topicBrokerPath, true);
                    client.WriteData(topicBrokerPath, 5);
                    WaitUntillIdle(client, 500);
                    client.UnsubscribeAll();
                    client.DeleteRecursive(brokerPath);
                    client.DeleteRecursive(topicPath);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.NotNull(mappings);
            Assert.Greater(mappings.Count, 0);
            Assert.IsTrue(brokers.ContainsKey(2345));
            Assert.IsTrue(mappings.Keys.Contains(CurrentTestTopic));
            Assert.AreEqual(5, mappings[CurrentTestTopic].Count);
        }
        public void WhenChildIsCreatedChilListenerOnParentFires()
        {
            string myPath = "/" + Guid.NewGuid();
            var producerConfig = new ProducerConfig(clientConfig);
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                WaitUntillIdle(client, 500);
                client.Subscribe("/", this as IZooKeeperChildListener);
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.UnsubscribeAll();
                client.Delete(myPath);
            }

            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, "/");
            Assert.Greater(((ZooKeeperChildChangedEventArgs)e).Children.Count, 0);
            Assert.IsTrue(((ZooKeeperChildChangedEventArgs)e).Children.Contains(myPath.Replace("/", string.Empty)));
        }
        public void WhenDataChangedDataListenerFires()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            string myPath = "/" + Guid.NewGuid();
            string sourceData = "my test data";
            string resultData;
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.Subscribe(myPath, this as IZooKeeperDataListener);
                client.Subscribe(myPath, this as IZooKeeperChildListener);
                client.WriteData(myPath, sourceData);
                WaitUntillIdle(client, 500);
                client.UnsubscribeAll();
                resultData = client.ReadData<string>(myPath);
                client.Delete(myPath);
            }

            Assert.IsTrue(!string.IsNullOrEmpty(resultData));
            Assert.AreEqual(sourceData, resultData);
            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
            Assert.IsNotNull(((ZooKeeperDataChangedEventArgs)e).Data);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Data, sourceData);
        }
Exemple #8
0
 public void SetUp()
 {
     config = new ProducerConfig();
     config.BrokerPartitionInfo = "1:192.168.0.1:1234,2:192.168.0.2:3456";
 }
 public void ZooKeeperClientCreatesANewPathAndDeletesIt()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
     {
         client.Connect();
         string myPath = "/" + Guid.NewGuid();
         client.CreatePersistent(myPath, false);
         Assert.IsTrue(client.Exists(myPath));
         client.Delete(myPath);
         Assert.IsFalse(client.Exists(myPath));
     }
 }
 public void ZooKeeperClientCreatesAChildAndGetsChildren()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
     {
         client.Connect();
         string child = Guid.NewGuid().ToString();
         string myPath = "/" + child;
         client.CreatePersistent(myPath, false);
         IList<string> children = client.GetChildren("/", false);
         int countChildren = client.CountChildren("/");
         Assert.Greater(children.Count, 0);
         Assert.AreEqual(children.Count, countChildren);
         Assert.IsTrue(children.Contains(child));
         client.Delete(myPath);
     }
 }
 public void ZooKeeperClientChecksIfPathExists()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
     {
         client.Connect();
         Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false));
     }
 }
        public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            string myPath = "/" + Guid.NewGuid();
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                client.CreatePersistent(myPath, true);
                WaitUntillIdle(client, 500);
                client.Subscribe(myPath, this as IZooKeeperChildListener);
                client.Subscribe(myPath, this as IZooKeeperDataListener);
                client.Delete(myPath);
                WaitUntillIdle(client, 500);
            }

            Assert.AreEqual(2, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.ChildChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperChildChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperChildChangedEventArgs)e).Path, myPath);
            Assert.IsNull(((ZooKeeperChildChangedEventArgs)e).Children);
            e = this.events[1];
            Assert.AreEqual(ZooKeeperEventTypes.DataChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperDataChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperDataChangedEventArgs)e).Path, myPath);
            Assert.IsNull(((ZooKeeperDataChangedEventArgs)e).Data);
        }
        public void WhenStateChangedToExpiredStateAndSessionListenersFire()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Subscribe(this);
                client.Connect();
                WaitUntillIdle(client, 500);
                client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                WaitUntillIdle(client, 3000);
            }

            Assert.AreEqual(4, this.events.Count);
            ZooKeeperEventArgs e = this.events[1];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.Expired);
            e = this.events[2];
            Assert.AreEqual(ZooKeeperEventTypes.SessionCreated, e.Type);
            Assert.IsInstanceOf<ZooKeeperSessionCreatedEventArgs>(e);
            e = this.events[3];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
        }
        public void WhenStateChangedToConnectedStateListenerFires()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Subscribe(this);
                client.Connect();
                WaitUntillIdle(client, 500);
            }

            Assert.AreEqual(1, this.events.Count);
            ZooKeeperEventArgs e = this.events[0];
            Assert.AreEqual(ZooKeeperEventTypes.StateChanged, e.Type);
            Assert.IsInstanceOf<ZooKeeperStateChangedEventArgs>(e);
            Assert.AreEqual(((ZooKeeperStateChangedEventArgs)e).State, KeeperState.SyncConnected);
        }
        public void WhenSessionExpiredClientReconnects()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IZooKeeperConnection conn1;
            IZooKeeperConnection conn2;
            using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
            {
                client.Connect();
                conn1 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
                client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null));
                WaitUntillIdle(client, 1000);
                conn2 = ReflectionHelper.GetInstanceField<ZooKeeperConnection>("connection", client);
            }

            Assert.AreNotEqual(conn1, conn2);
        }
        public void WhenBrokerIsRemovedZKBrokerPartitionInfoUpdatesBrokersList()
        {
            var producerConfig = new ProducerConfig(clientConfig);
            IDictionary<int, Broker> brokers;
            string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345;
            using (IZooKeeperClient client = new ZooKeeperClient(
                producerConfig.ZkConnect,
                producerConfig.ZkSessionTimeoutMs,
                ZooKeeperStringSerializer.Serializer))
            {
                using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client))
                {
                    brokers = brokerPartitionInfo.GetAllBrokerInfo();
                    client.CreatePersistent(brokerPath, true);
                    client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102");
                    WaitUntillIdle(client, 500);
                    Assert.NotNull(brokers);
                    Assert.Greater(brokers.Count, 0);
                    Assert.IsTrue(brokers.ContainsKey(2345));
                    client.DeleteRecursive(brokerPath);
                    WaitUntillIdle(client, 500);
                }
            }

            Assert.NotNull(brokers);
            Assert.Greater(brokers.Count, 0);
            Assert.IsFalse(brokers.ContainsKey(2345));
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="ConfigBrokerPartitionInfo"/> class.
 /// </summary>
 /// <param name="config">The config.</param>
 public ConfigBrokerPartitionInfo(ProducerConfig config)
 {
     Guard.Assert<ArgumentNullException>(() => config != null);
     this.config = config;
     this.InitializeBrokers();
 }
 public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
     {
         client.Connect();
         var eventWorker = ReflectionHelper.GetInstanceField<Thread>("eventWorker", client);
         var zooKeeperWorker = ReflectionHelper.GetInstanceField<Thread>("zooKeeperEventWorker", client);
         Assert.NotNull(eventWorker);
         Assert.NotNull(zooKeeperWorker);
     }
 }
 public void ZooKeeperClientFailsWhenCreatedWithWrongConnectionInfo()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client = new ZooKeeperClient("random text", producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer))
     {
         Assert.Throws<FormatException>(client.Connect);
     }
 }
 public void WhenClientWillNotConnectWithinGivenTimeThrows()
 {
     var producerConfig = new ProducerConfig(clientConfig);
     using (IZooKeeperClient client =
         new ZooKeeperClient(
             producerConfig.ZkConnect,
             producerConfig.ZkSessionTimeoutMs,
             ZooKeeperStringSerializer.Serializer,
             1))
     {
         client.Connect();
     }
 }