internal static TopicPartitionState GetPartitionState(IZooKeeperClient zkClient, string topic, int partition) { var stateData = zkClient.ReadData <string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true); if (string.IsNullOrWhiteSpace(stateData)) { return(null); } TopicPartitionState partitionState; try { var ser = new JavaScriptSerializer(); var result = ser.Deserialize <Dictionary <string, object> >(stateData); partitionState = new TopicPartitionState() { Leader = int.Parse(result["leader"].ToString()), Leader_Epoch = int.Parse(result["leader_epoch"].ToString()), Controller_Epoch = int.Parse(result["controller_epoch"].ToString()), Verstion = int.Parse(result["version"].ToString()), }; var isrArr = result["isr"] as System.Collections.ArrayList; partitionState.Isr = isrArr != null?isrArr.Cast <int>().ToArray() : null; } catch (Exception exc) { Logger.WarnFormat("Unexpected error while trying to get topic partition state for topic '{0}' partition '{1}'. Error: {2} ", topic, partition, exc.FormatException()); return(null); } return(partitionState); }
internal static void CreateEphemeralPathExpectConflict(IZooKeeperClient zkClient, string path, string data) { try { CreateEphemeralPath(zkClient, path, data); } catch (KeeperException.NodeExistsException) { string storedData; try { storedData = zkClient.ReadData<string>(path); } catch (KeeperException.NoNodeException) { // the node disappeared; treat as if node existed and let caller handles this throw; } if (storedData == null || storedData != data) { Logger.InfoFormat(CultureInfo.CurrentCulture, "conflict in {0} data: {1} stored data: {2}", path, data, storedData); throw; } else { // otherwise, the creation succeeded, return normally Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} exits with value {1} during connection loss; this is ok", path, data); } } }
internal static IDictionary <string, IList <string> > GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable <string> topics) { var result = new Dictionary <string, IList <string> >(); foreach (string topic in topics) { var partList = new List <string>(); var brokers = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic); foreach (var broker in brokers) { var numberOfParts = int.Parse( zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic + "/" + broker), CultureInfo.CurrentCulture); for (int i = 0; i < numberOfParts; i++) { partList.Add(broker + "-" + i); } } partList.Sort(); result.Add(topic, partList); } return(result); }
public static IEnumerable <Broker> GetBrokerInfoFromIds(IZooKeeperClient zkClient, IEnumerable <int> brokerIds) { return(brokerIds.Select( brokerId => Broker.CreateBroker(brokerId, zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + brokerId)))); }
public static IEnumerable<Broker> GetBrokerInfoFromIds(IZooKeeperClient zkClient, IEnumerable<int> brokerIds) { return brokerIds.Select( brokerId => Broker.CreateBroker(brokerId, zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + brokerId))); }
public OffsetManager(KafkaSimpleManager <int, Message> manager) { _manager = manager; zkClient = new ZooKeeperClient(manager.Config.ZookeeperConfig.ZkConnect, manager.Config.ZookeeperConfig.ZkSessionTimeoutMs, ZooKeeperNetBinarySerializer.Serializer); zkClient.Connect(); }
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.WriteData(path, data); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { CreateParentPath(zkClient, path); try { zkClient.CreatePersistent(path, data); } catch (KeeperException e2) { if (e2.ErrorCode == KeeperException.Code.NODEEXISTS) { zkClient.WriteData(path, data); } else { throw; } } } else { throw; } } }
private void ConnectZk() { Logger.InfoFormat("Connecting to zookeeper instance at {0}", config.ZooKeeper.ZkConnect); zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs); zkClient.Connect(); }
private ServiceBeacon CreateBeacon(IZooKeeperClient zooKeeperClient, BuildContext context) { var settings = new ServiceBeaconSettings(); if (registrationDeniedFromNonActiveDatacenters) { settings.RegistrationAllowedProvider = LocalDatacenterIsActive(context.Datacenters); } settingsCustomization.Customize(settings); return(new ServiceBeacon( zooKeeperClient, s => { s.SetProperty(WellKnownApplicationIdentityProperties.Project, context.ApplicationIdentity.Project); s.SetProperty(WellKnownApplicationIdentityProperties.Subproject, context.ApplicationIdentity.Subproject); s.SetProperty(WellKnownApplicationIdentityProperties.Environment, context.ApplicationIdentity.Environment); s.SetProperty(WellKnownApplicationIdentityProperties.Application, context.ApplicationIdentity.Application); s.SetProperty(WellKnownApplicationIdentityProperties.Instance, context.ApplicationIdentity.Instance); replicaInfoCustomization.Customize(s); }, settings, context.Log)); }
internal static void CreateEphemeralPathExpectConflict(IZooKeeperClient zkClient, string path, string data) { try { CreateEphemeralPath(zkClient, path, data); } catch (KeeperException e) // KeeperException.NoNodeException) { if (e.ErrorCode == KeeperException.Code.NODEEXISTS) { string storedData; try { storedData = zkClient.ReadData <string>(path); } catch (Exception) { // the node disappeared; treat as if node existed and let caller handles this throw; } if (storedData == null || storedData != data) { Logger.InfoFormat("conflict in {0} data: {1} stored data: {2}", path, data, storedData); throw; } // otherwise, the creation succeeded, return normally Logger.InfoFormat("{0} exits with value {1} during connection loss; this is ok", path, data); } else { throw; } } }
public EnvironmentsStorage(IZooKeeperClient zooKeeperClient, ServiceDiscoveryPathHelper pathHelper, ActionsQueue eventsHandler, ILog log) { this.zooKeeperClient = zooKeeperClient; this.pathHelper = pathHelper; this.eventsHandler = eventsHandler; this.log = log; nodeWatcher = new AdHocNodeWatcher(OnNodeEvent); }
internal static void CreateParentPath(IZooKeeperClient zkClient, string path) { string parentDir = path.Substring(0, path.LastIndexOf('/')); if (parentDir.Length != 0) { zkClient.CreatePersistent(parentDir, true); } }
public ServiceBeacon( [NotNull] IZooKeeperClient zooKeeperClient, [CanBeNull] ReplicaInfoSetup replicaInfoSetup = null, [CanBeNull] ServiceBeaconSettings settings = null, [CanBeNull] ILog log = null) : this(zooKeeperClient, ReplicaInfoBuilder.Build(replicaInfoSetup, (settings ?? new ServiceBeaconSettings()).UseFQDN), settings, log) { }
/// <summary> /// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class. /// </summary> /// <param name="zkclient">The wrapper above ZooKeeper client.</param> public ZKBrokerPartitionInfo(IZooKeeperClient zkclient) { this.zkclient = zkclient; this.zkclient.Connect(); this.InitializeBrokers(); this.InitializeTopicBrokerPartitions(); this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, this.topicBrokerPartitions, this.brokers, this.callback); this.RegisterListeners(); }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfig config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos) { this.name = name; this.zkClient = zkClient; this.config = config; this.broker = broker; this.partitionTopicInfos = partitionTopicInfos; this.simpleConsumer = new Consumer(this.config); }
internal static void CreateParentPath(IZooKeeperClient zkClient, string path) { var parentDir = path.Substring(0, path.LastIndexOf('/')); if (parentDir.Length != 0) { zkClient.CreatePersistent(parentDir, true); } }
/// <summary> /// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class. /// </summary> /// <param name="zkclient">The wrapper above ZooKeeper client.</param> public ZKBrokerPartitionInfo(IZooKeeperClient zkclient) { this.zkclient = zkclient; this.zkclient.Connect(); this.InitializeBrokers(); this.InitializeTopicBrokerPartitions(); this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, this.topicBrokerPartitions, this.brokers, this.callback); this.RegisterListeners(); }
/// <summary> /// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class. /// </summary> /// <param name="zkclient">The wrapper above ZooKeeper client.</param> /// <param name="callback">The callback invoked when new broker is added..</param> public ZKBrokerPartitionInfo(IZooKeeperClient zkclient, Action <int, string, int> callback = null) { this.zkclient = zkclient; this.zkclient.Connect(); this.InitializeBrokers(); this.InitializeTopicBrokerPartitions(); this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, topicBrokerPartitions, brokers, callback); this.RegisterListeners(); }
internal static void WaitUntillIdle(IZooKeeperClient client, int timeout) { Thread.Sleep(timeout); int rest = timeout - client.IdleTime; while (rest > 0) { Thread.Sleep(rest); rest = timeout - client.IdleTime; } }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List <PartitionTopicInfo> partitionTopicInfos) { this.name = name; this.zkClient = zkClient; this.config = config; this.broker = broker; this.partitionTopicInfos = partitionTopicInfos; this.simpleConsumer = new Consumer(this.config, broker.Host, broker.Port); }
internal static void WaitUntillIdle(IZooKeeperClient client, int timeout) { Thread.Sleep(timeout); int rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout; while (rest > 0) { Thread.Sleep(rest); rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout; } }
internal static void DeletePath(IZooKeeperClient zkClient, string path) { try { zkClient.Delete(path); } catch (KeeperException.NoNodeException) { Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} deleted during connection loss; this is ok", path); } }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos, Action<PartitionTopicInfo> markPartitonWithError) { _name = name; _zkClient = zkClient; _config = config; _broker = broker; _partitionTopicInfos = partitionTopicInfos; _fetchBufferLength = config.MaxFetchBufferLength; _markPartitonWithError = markPartitonWithError; _simpleConsumer = new Consumer(_config, broker.Host, broker.Port); }
/// <summary> /// Initializes a new instance of the <see cref="Cluster"/> class. /// </summary> /// <param name="zkClient">IZooKeeperClient object</param> public Cluster(IZooKeeperClient zkClient) : this() { var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath); foreach (var node in nodes) { var brokerZkString = zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node); Broker broker = this.CreateBroker(node, brokerZkString); brokers[broker.Id] = broker; } }
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List <PartitionTopicInfo> partitionTopicInfos, Action <PartitionTopicInfo> markPartitonWithError) { _name = name; _zkClient = zkClient; _config = config; _broker = broker; _partitionTopicInfos = partitionTopicInfos; _fetchBufferLength = config.MaxFetchBufferLength; _markPartitonWithError = markPartitonWithError; _simpleConsumer = new Consumer(_config, broker.Host, broker.Port); }
public ServiceDiscoveryManager( [NotNull] IZooKeeperClient zooKeeperClient, [CanBeNull] ServiceDiscoveryManagerSettings settings = null, [CanBeNull] ILog log = null) { this.zooKeeperClient = zooKeeperClient ?? throw new ArgumentNullException(nameof(zooKeeperClient)); this.settings = settings ?? new ServiceDiscoveryManagerSettings(); this.log = (log ?? LogProvider.Get()).ForContext <ServiceDiscoveryManager>(); pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper); }
/// <summary> /// Initializes a new instance of the <see cref="Cluster"/> class. /// </summary> /// <param name="zkClient">IZooKeeperClient object</param> public Cluster(IZooKeeperClient zkClient) : this() { var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath); foreach (var node in nodes) { var brokerZkString = zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node); Broker broker = this.CreateBroker(node, brokerZkString); brokers[broker.Id] = broker; } }
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.CreateEphemeral(path, data); } catch (KeeperException.NoNodeException) { ZkUtils.CreateParentPath(zkClient, path); zkClient.CreateEphemeral(path, data); } }
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.CreateEphemeral(path, data); } catch (KeeperException.NoNodeException) { ZkUtils.CreateParentPath(zkClient, path); zkClient.CreateEphemeral(path, data); } }
private void ConnectZk() { Logger.Info("Enter connectZk()"); if (UseSharedStaticZookeeperClient) { Logger.Info("After check UseSharedStaticZookeeperClient"); if (zkClientStatic == null || zkClientStatic != null && zkClientStatic.GetClientState() != KeeperState.SyncConnected) { Logger.Info("After check UseSharedStaticZookeeperClient, will lock"); lock (zkClientStaticLock) { Logger.Info("got lock ... "); if (zkClientStatic == null || zkClientStatic != null && zkClientStatic.GetClientState() != KeeperState.SyncConnected) { Logger.InfoFormat("zkClientStatic: {0} will create one ...", zkClientStatic == null ? "null" : "not null"); Logger.InfoFormat("Connecting to zookeeper instance at {0} STATIC", config.ZooKeeper.ZkConnect); zkClientStatic = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs); zkClientStatic.Connect(); Logger.InfoFormat("Connecting to zookeeper instance at {0} STATIC. Done", config.ZooKeeper.ZkConnect); } Logger.Info("release lock ... "); } } Logger.InfoFormat("zkClientStatic: {0}", zkClientStatic == null ? "null" : "not null"); if (zkClientStatic != null) { Logger.InfoFormat("zkClientStatic.ClientState: {0}", zkClientStatic.GetClientState()); } } else { Logger.InfoFormat("Connecting to zookeeper instance at {0}", config.ZooKeeper.ZkConnect); if (zkClientInternal != null) { zkClientInternal.Dispose(); zkClientInternal = null; } zkClientInternal = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs); zkClientInternal.Connect(); } }
internal static int?GetLeaderForPartition(IZooKeeperClient zkClient, string topic, int partition) { var stateData = zkClient.ReadData <string>( GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true); if (string.IsNullOrWhiteSpace(stateData)) { return(null); } int leader; return(TryParsePartitionLeader(stateData, out leader) ? (int?)leader : null); }
public ServiceLocator( [NotNull] IZooKeeperClient zooKeeperClient, [CanBeNull] ServiceLocatorSettings settings = null, [CanBeNull] ILog log = null) { this.zooKeeperClient = zooKeeperClient; this.settings = settings ?? new ServiceLocatorSettings(); this.log = (log ?? LogProvider.Get()).ForContext <ServiceLocator>(); pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper); eventsQueue = new ActionsQueue(this.log); environmentsStorage = new EnvironmentsStorage(zooKeeperClient, pathHelper, eventsQueue, log); applicationsStorage = new ApplicationsStorage(zooKeeperClient, pathHelper, eventsQueue, log); }
/// <summary> /// Initializes a new instance of the <see cref="Cluster"/> class. /// </summary> /// <param name="zkClient">IZooKeeperClient object</param> public Cluster(IZooKeeperClient zkClient) { var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath); foreach (var node in nodes) { var brokerZkString = zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node); Broker broker = this.CreateBroker(node, brokerZkString); if (brokers.ContainsKey(broker.Id)) { brokers[broker.Id] = broker; } else { brokers.Add(broker.Id, broker); } } }
/// <summary> /// Initializes a new instance of the <see cref="Cluster"/> class. /// </summary> /// <param name="zkClient">IZooKeeperClient object</param> public Cluster(IZooKeeperClient zkClient) { var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath); foreach (var node in nodes) { var brokerZkString = zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node); Broker broker = this.CreateBroker(node, brokerZkString); if (brokers.ContainsKey(broker.Id)) { brokers[broker.Id] = broker; } else { brokers.Add(broker.Id, broker); } } }
internal static void DeletePath(IZooKeeperClient zkClient, string path) { try { zkClient.Delete(path); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { Logger.InfoFormat("{0} deleted during connection loss; this is ok", path); } else { throw; } } }
/// <summary> /// Initializes a new instance of the <see cref="BrokerTopicsListener"/> class. /// </summary> /// <param name="zkclient">The wrapper on ZooKeeper client.</param> /// <param name="actualBrokerTopicsPartitionsMap">The actual broker topics partitions map.</param> /// <param name="actualBrokerIdMap">The actual broker id map.</param> /// <param name="callback">The callback invoked after new broker is added.</param> public BrokerTopicsListener( IZooKeeperClient zkclient, IDictionary<string, SortedSet<Partition>> actualBrokerTopicsPartitionsMap, IDictionary<int, Broker> actualBrokerIdMap, Action<int, string, int> callback) { this.zkclient = zkclient; this.actualBrokerTopicsPartitionsMap = actualBrokerTopicsPartitionsMap; this.actualBrokerIdMap = actualBrokerIdMap; this.callback = callback; this.oldBrokerIdMap = new Dictionary<int, Broker>(this.actualBrokerIdMap); this.oldBrokerTopicsPartitionsMap = new Dictionary<string, SortedSet<Partition>>(this.actualBrokerTopicsPartitionsMap); Logger.Debug("Creating broker topics listener to watch the following paths - \n" + "/broker/topics, /broker/topics/topic, /broker/ids"); Logger.Debug("Initialized this broker topics listener with initial mapping of broker id to " + "partition id per topic with " + this.oldBrokerTopicsPartitionsMap.ToMultiString( x => x.Key + " --> " + x.Value.ToMultiString(y => y.ToString(), ","), "; ")); }
/// <summary> /// Initializes a new instance of the <see cref="BrokerTopicsListener"/> class. /// </summary> /// <param name="zkclient">The wrapper on ZooKeeper client.</param> /// <param name="actualBrokerTopicsPartitionsMap">The actual broker topics partitions map.</param> /// <param name="actualBrokerIdMap">The actual broker id map.</param> /// <param name="callback">The callback invoked after new broker is added.</param> public BrokerTopicsListener( IZooKeeperClient zkclient, IDictionary <string, SortedSet <Partition> > actualBrokerTopicsPartitionsMap, IDictionary <int, Broker> actualBrokerIdMap, Action <int, string, int> callback) { this.zkclient = zkclient; this.actualBrokerTopicsPartitionsMap = actualBrokerTopicsPartitionsMap; this.actualBrokerIdMap = actualBrokerIdMap; this.callback = callback; this.oldBrokerIdMap = new Dictionary <int, Broker>(this.actualBrokerIdMap); this.oldBrokerTopicsPartitionsMap = new Dictionary <string, SortedSet <Partition> >(this.actualBrokerTopicsPartitionsMap); Logger.Debug("Creating broker topics listener to watch the following paths - \n" + "/broker/topics, /broker/topics/topic, /broker/ids"); Logger.Debug("Initialized this broker topics listener with initial mapping of broker id to " + "partition id per topic with " + this.oldBrokerTopicsPartitionsMap.ToMultiString( x => x.Key + " --> " + x.Value.ToMultiString(y => y.ToString(), ","), "; ")); }
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.CreateEphemeral(path, data); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { CreateParentPath(zkClient, path); zkClient.CreateEphemeral(path, data); } else { throw; } } }
internal static IDictionary <string, IList <string> > GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable <string> topics) { var result = new Dictionary <string, IList <string> >(); foreach (string topic in topics) { var partitions = zkClient.GetChildrenParentMayNotExist(GetTopicPartitionsPath(topic)); if (partitions == null) { throw new NoPartitionsForTopicException(topic); } Logger.DebugFormat("children of /brokers/topics/{0} are {1}", topic, string.Join(",", partitions)); result.Add(topic, partitions != null ? partitions.OrderBy(x => x).ToList() : new List <string>()); } return(result); }
/// <summary> /// <para>Trying to update node data with optimistic concurrency strategy according to given <paramref name="request"/>.</para> /// <para>Check returned <see cref="UpdateDataResult"/> to see if operation was successful.</para> /// </summary> public static async Task <UpdateDataResult> UpdateDataAsync(this IZooKeeperClient zooKeeperClient, UpdateDataRequest request) { try { for (var i = 0; i < request.Attempts; i++) { var readResult = await zooKeeperClient.GetDataAsync(new GetDataRequest(request.Path)).ConfigureAwait(false); if (!readResult.IsSuccessful) { return(UpdateDataResult.Unsuccessful(readResult.Status, readResult.Path, readResult.Exception)); } var newData = request.Update(readResult.Data); if (ByteArrayKey.Equals(readResult.Data, newData)) { return(UpdateDataResult.Successful(readResult.Path)); } var setDataRequest = new SetDataRequest(request.Path, newData) { Version = readResult.Stat.Version }; var updateResult = await zooKeeperClient.SetDataAsync(setDataRequest).ConfigureAwait(false); if (updateResult.Status == ZooKeeperStatus.VersionsMismatch) { continue; } return(updateResult.IsSuccessful ? UpdateDataResult.Successful(updateResult.Path) : UpdateDataResult.Unsuccessful(updateResult.Status, updateResult.Path, updateResult.Exception)); } return(UpdateDataResult.Unsuccessful(ZooKeeperStatus.VersionsMismatch, request.Path, null)); } catch (Exception e) { return(UpdateDataResult.Unsuccessful(ZooKeeperStatus.UnknownError, request.Path, e)); } }
internal ZKRebalancerListener( ConsumerConfiguration config, string consumerIdString, IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry, IZooKeeperClient zkClient, ZookeeperConsumerConnector zkConsumerConnector, IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues, Fetcher fetcher, object syncLock) { this.syncLock = syncLock; this.consumerIdString = consumerIdString; this.config = config; this.topicRegistry = topicRegistry; this.zkClient = zkClient; this.dirs = new ZKGroupDirs(config.GroupId); this.zkConsumerConnector = zkConsumerConnector; this.queues = queues; this.fetcher = fetcher; }
internal ServiceBeacon( [NotNull] IZooKeeperClient zooKeeperClient, [NotNull] ReplicaInfo replicaInfo, [CanBeNull] ServiceBeaconSettings settings, [CanBeNull] ILog log) { this.zooKeeperClient = zooKeeperClient ?? throw new ArgumentNullException(nameof(settings)); this.replicaInfo = replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(settings)); this.settings = settings ?? new ServiceBeaconSettings(); this.log = (log ?? LogProvider.Get()).ForContext <ServiceBeacon>(); var pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper); environmentNodePath = pathHelper.BuildEnvironmentPath(replicaInfo.Environment); applicationNodePath = pathHelper.BuildApplicationPath(replicaInfo.Environment, replicaInfo.Application); replicaNodePath = pathHelper.BuildReplicaPath(replicaInfo.Environment, replicaInfo.Application, replicaInfo.Replica); replicaNodeData = ReplicaNodeDataSerializer.Serialize(replicaInfo); nodeWatcher = new AdHocNodeWatcher(OnNodeEvent); }
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.WriteData(path, data); } catch (KeeperException.NoNodeException) { CreateParentPath(zkClient, path); try { zkClient.CreatePersistent(path, data); } catch (KeeperException.NodeExistsException) { zkClient.WriteData(path, data); } } }
/// <summary> /// Initializes a new instance of the <see cref="Fetcher"/> class. /// </summary> /// <param name="config"> /// The consumer configuration. /// </param> /// <param name="zkClient"> /// The wrapper above ZooKeeper client. /// </param> public Fetcher(ConsumerConfig config, IZooKeeperClient zkClient) { this.config = config; this.zkClient = zkClient; }
public static IEnumerable<Broker> GetAllBrokersInCluster(IZooKeeperClient zkClient) { var brokerIds = zkClient.GetChildren(ZooKeeperClient.DefaultBrokerIdsPath).OrderBy(x => x).ToList(); return ZkUtils.GetBrokerInfoFromIds(zkClient, brokerIds.Select(x => int.Parse(x))); }
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.WriteData(path, data); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { CreateParentPath(zkClient, path); try { zkClient.CreatePersistent(path, data); } catch (KeeperException e2) { if (e2.ErrorCode == KeeperException.Code.NODEEXISTS) zkClient.WriteData(path, data); else throw; } } else throw; } }
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data) { try { zkClient.CreateEphemeral(path, data); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) { ZkUtils.CreateParentPath(zkClient, path); zkClient.CreateEphemeral(path, data); } else throw; } }
private void ConnectZk() { Logger.InfoFormat(CultureInfo.CurrentCulture, "Connecting to zookeeper instance at {0}", this.config.ZkConnect); this.zkClient = new ZooKeeperClient(this.config.ZkConnect, this.config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); this.zkClient.Connect(); }
internal static TopicPartitionState GetPartitionState(IZooKeeperClient zkClient, string topic, int partition) { var stateData = zkClient.ReadData<string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true); if (string.IsNullOrWhiteSpace(stateData)) { return null; } TopicPartitionState partitionState; try { var ser = new JavaScriptSerializer(); var result = ser.Deserialize<Dictionary<string, object>>(stateData); partitionState = new TopicPartitionState() { Leader = int.Parse(result["leader"].ToString()), Leader_Epoch = int.Parse(result["leader_epoch"].ToString()), Controller_Epoch = int.Parse(result["controller_epoch"].ToString()), Verstion = int.Parse(result["version"].ToString()), }; var isrArr = result["isr"] as System.Collections.ArrayList; partitionState.Isr = isrArr != null ? isrArr.Cast<int>().ToArray() : null; } catch (Exception exc) { Logger.WarnFormat("Unexpected error while trying to get topic partition state for topic '{0}' partition '{1}'. Error: {2} ", topic, partition, exc.FormatException()); return null; } return partitionState; }
internal static void DeletePath(IZooKeeperClient zkClient, string path) { try { zkClient.Delete(path); } catch (KeeperException e) { if (e.ErrorCode == KeeperException.Code.NONODE) Logger.InfoFormat("{0} deleted during connection loss; this is ok", path); else throw; } }
internal static IDictionary<string, IList<string>> GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable<string> topics) { var result = new Dictionary<string, IList<string>>(); foreach (string topic in topics) { var partList = new List<string>(); var brokers = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic); foreach (var broker in brokers) { var numberOfParts = int.Parse( zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic + "/" + broker), CultureInfo.CurrentCulture); for (int i = 0; i < numberOfParts; i++) { partList.Add(broker + "-" + i); } } partList.Sort(); result.Add(topic, partList); } return result; }
internal static void DeletePath(IZooKeeperClient zkClient, string path) { try { zkClient.Delete(path); } catch (KeeperException.NoNodeException) { Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} deleted during connection loss; this is ok", path); } }
internal static int? GetLeaderForPartition(IZooKeeperClient zkClient, string topic, int partition) { var stateData = zkClient.ReadData<string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true); if (string.IsNullOrWhiteSpace(stateData)) { return (int?)null; } else { int leader; return TryParsePartitionLeader(stateData, out leader) ? (int?)leader : null; } }
internal static IDictionary<string, IList<string>> GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable<string> topics) { var result = new Dictionary<string, IList<string>>(); foreach (string topic in topics) { var partitions = zkClient.GetChildrenParentMayNotExist(GetTopicPartitionsPath(topic)); if (partitions == null) { throw new NoPartitionsForTopicException(topic); } Logger.DebugFormat("children of /brokers/topics/{0} are {1}", topic, string.Join(",", partitions)); result.Add(topic, partitions != null ? partitions.OrderBy(x => x).ToList() : new List<string>()); } return result; }