internal static TopicPartitionState GetPartitionState(IZooKeeperClient zkClient, string topic, int partition)
{
var stateData = zkClient.ReadData <string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true);
if (string.IsNullOrWhiteSpace(stateData))
{
return(null);
}
TopicPartitionState partitionState;
try
{
var ser = new JavaScriptSerializer();
var result = ser.Deserialize <Dictionary <string, object> >(stateData);
partitionState = new TopicPartitionState()
{
Leader = int.Parse(result["leader"].ToString()),
Leader_Epoch = int.Parse(result["leader_epoch"].ToString()),
Controller_Epoch = int.Parse(result["controller_epoch"].ToString()),
Verstion = int.Parse(result["version"].ToString()),
};
var isrArr = result["isr"] as System.Collections.ArrayList;
partitionState.Isr = isrArr != null?isrArr.Cast <int>().ToArray() : null;
}
catch (Exception exc)
{
Logger.WarnFormat("Unexpected error while trying to get topic partition state for topic '{0}' partition '{1}'. Error: {2} ", topic, partition, exc.FormatException());
return(null);
}
return(partitionState);
}
internal static void CreateEphemeralPathExpectConflict(IZooKeeperClient zkClient, string path, string data)
{
try
{
CreateEphemeralPath(zkClient, path, data);
}
catch (KeeperException.NodeExistsException)
{
string storedData;
try
{
storedData = zkClient.ReadData<string>(path);
}
catch (KeeperException.NoNodeException)
{
// the node disappeared; treat as if node existed and let caller handles this
throw;
}
if (storedData == null || storedData != data)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "conflict in {0} data: {1} stored data: {2}", path, data, storedData);
throw;
}
else
{
// otherwise, the creation succeeded, return normally
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} exits with value {1} during connection loss; this is ok", path, data);
}
}
}
internal static IDictionary <string, IList <string> > GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable <string> topics)
{
var result = new Dictionary <string, IList <string> >();
foreach (string topic in topics)
{
var partList = new List <string>();
var brokers =
zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic);
foreach (var broker in brokers)
{
var numberOfParts =
int.Parse(
zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic + "/" +
broker),
CultureInfo.CurrentCulture);
for (int i = 0; i < numberOfParts; i++)
{
partList.Add(broker + "-" + i);
}
}
partList.Sort();
result.Add(topic, partList);
}
return(result);
}
public static IEnumerable <Broker> GetBrokerInfoFromIds(IZooKeeperClient zkClient, IEnumerable <int> brokerIds)
{
return(brokerIds.Select(
brokerId =>
Broker.CreateBroker(brokerId,
zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + brokerId))));
}
public static IEnumerable<Broker> GetBrokerInfoFromIds(IZooKeeperClient zkClient, IEnumerable<int> brokerIds)
{
return brokerIds.Select(
brokerId =>
Broker.CreateBroker(brokerId,
zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + brokerId)));
}
public OffsetManager(KafkaSimpleManager <int, Message> manager)
{
_manager = manager;
zkClient = new ZooKeeperClient(manager.Config.ZookeeperConfig.ZkConnect, manager.Config.ZookeeperConfig.ZkSessionTimeoutMs, ZooKeeperNetBinarySerializer.Serializer);
zkClient.Connect();
}
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.WriteData(path, data);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
CreateParentPath(zkClient, path);
try
{
zkClient.CreatePersistent(path, data);
}
catch (KeeperException e2)
{
if (e2.ErrorCode == KeeperException.Code.NODEEXISTS)
{
zkClient.WriteData(path, data);
}
else
{
throw;
}
}
}
else
{
throw;
}
}
}
private void ConnectZk()
{
Logger.InfoFormat("Connecting to zookeeper instance at {0}", config.ZooKeeper.ZkConnect);
zkClient = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs);
zkClient.Connect();
}
private ServiceBeacon CreateBeacon(IZooKeeperClient zooKeeperClient, BuildContext context)
{
var settings = new ServiceBeaconSettings();
if (registrationDeniedFromNonActiveDatacenters)
{
settings.RegistrationAllowedProvider = LocalDatacenterIsActive(context.Datacenters);
}
settingsCustomization.Customize(settings);
return(new ServiceBeacon(
zooKeeperClient,
s =>
{
s.SetProperty(WellKnownApplicationIdentityProperties.Project, context.ApplicationIdentity.Project);
s.SetProperty(WellKnownApplicationIdentityProperties.Subproject, context.ApplicationIdentity.Subproject);
s.SetProperty(WellKnownApplicationIdentityProperties.Environment, context.ApplicationIdentity.Environment);
s.SetProperty(WellKnownApplicationIdentityProperties.Application, context.ApplicationIdentity.Application);
s.SetProperty(WellKnownApplicationIdentityProperties.Instance, context.ApplicationIdentity.Instance);
replicaInfoCustomization.Customize(s);
},
settings,
context.Log));
}
internal static void CreateEphemeralPathExpectConflict(IZooKeeperClient zkClient, string path, string data)
{
try
{
CreateEphemeralPath(zkClient, path, data);
}
catch (KeeperException e) // KeeperException.NoNodeException)
{
if (e.ErrorCode == KeeperException.Code.NODEEXISTS)
{
string storedData;
try
{
storedData = zkClient.ReadData <string>(path);
}
catch (Exception)
{
// the node disappeared; treat as if node existed and let caller handles this
throw;
}
if (storedData == null || storedData != data)
{
Logger.InfoFormat("conflict in {0} data: {1} stored data: {2}", path, data, storedData);
throw;
}
// otherwise, the creation succeeded, return normally
Logger.InfoFormat("{0} exits with value {1} during connection loss; this is ok", path, data);
}
else
{
throw;
}
}
}
public EnvironmentsStorage(IZooKeeperClient zooKeeperClient, ServiceDiscoveryPathHelper pathHelper, ActionsQueue eventsHandler, ILog log)
{
this.zooKeeperClient = zooKeeperClient;
this.pathHelper = pathHelper;
this.eventsHandler = eventsHandler;
this.log = log;
nodeWatcher = new AdHocNodeWatcher(OnNodeEvent);
}
internal static void CreateParentPath(IZooKeeperClient zkClient, string path)
{
string parentDir = path.Substring(0, path.LastIndexOf('/'));
if (parentDir.Length != 0)
{
zkClient.CreatePersistent(parentDir, true);
}
}
public ServiceBeacon(
[NotNull] IZooKeeperClient zooKeeperClient,
[CanBeNull] ReplicaInfoSetup replicaInfoSetup = null,
[CanBeNull] ServiceBeaconSettings settings = null,
[CanBeNull] ILog log = null)
: this(zooKeeperClient, ReplicaInfoBuilder.Build(replicaInfoSetup, (settings ?? new ServiceBeaconSettings()).UseFQDN), settings, log)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="zkclient">The wrapper above ZooKeeper client.</param>
public ZKBrokerPartitionInfo(IZooKeeperClient zkclient)
{
this.zkclient = zkclient;
this.zkclient.Connect();
this.InitializeBrokers();
this.InitializeTopicBrokerPartitions();
this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, this.topicBrokerPartitions, this.brokers, this.callback);
this.RegisterListeners();
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfig config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos)
{
this.name = name;
this.zkClient = zkClient;
this.config = config;
this.broker = broker;
this.partitionTopicInfos = partitionTopicInfos;
this.simpleConsumer = new Consumer(this.config);
}
internal static void CreateParentPath(IZooKeeperClient zkClient, string path)
{
var parentDir = path.Substring(0, path.LastIndexOf('/'));
if (parentDir.Length != 0)
{
zkClient.CreatePersistent(parentDir, true);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="zkclient">The wrapper above ZooKeeper client.</param>
public ZKBrokerPartitionInfo(IZooKeeperClient zkclient)
{
this.zkclient = zkclient;
this.zkclient.Connect();
this.InitializeBrokers();
this.InitializeTopicBrokerPartitions();
this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, this.topicBrokerPartitions, this.brokers, this.callback);
this.RegisterListeners();
}
/// <summary>
/// Initializes a new instance of the <see cref="ZKBrokerPartitionInfo"/> class.
/// </summary>
/// <param name="zkclient">The wrapper above ZooKeeper client.</param>
/// <param name="callback">The callback invoked when new broker is added..</param>
public ZKBrokerPartitionInfo(IZooKeeperClient zkclient, Action <int, string, int> callback = null)
{
this.zkclient = zkclient;
this.zkclient.Connect();
this.InitializeBrokers();
this.InitializeTopicBrokerPartitions();
this.brokerTopicsListener = new BrokerTopicsListener(this.zkclient, topicBrokerPartitions, brokers, callback);
this.RegisterListeners();
}
internal static void WaitUntillIdle(IZooKeeperClient client, int timeout)
{
Thread.Sleep(timeout);
int rest = timeout - client.IdleTime;
while (rest > 0)
{
Thread.Sleep(rest);
rest = timeout - client.IdleTime;
}
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List <PartitionTopicInfo> partitionTopicInfos)
{
this.name = name;
this.zkClient = zkClient;
this.config = config;
this.broker = broker;
this.partitionTopicInfos = partitionTopicInfos;
this.simpleConsumer = new Consumer(this.config, broker.Host, broker.Port);
}
internal static void WaitUntillIdle(IZooKeeperClient client, int timeout)
{
Thread.Sleep(timeout);
int rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout;
while (rest > 0)
{
Thread.Sleep(rest);
rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout;
}
}
internal static void DeletePath(IZooKeeperClient zkClient, string path)
{
try
{
zkClient.Delete(path);
}
catch (KeeperException.NoNodeException)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} deleted during connection loss; this is ok", path);
}
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List<PartitionTopicInfo> partitionTopicInfos, Action<PartitionTopicInfo> markPartitonWithError)
{
_name = name;
_zkClient = zkClient;
_config = config;
_broker = broker;
_partitionTopicInfos = partitionTopicInfos;
_fetchBufferLength = config.MaxFetchBufferLength;
_markPartitonWithError = markPartitonWithError;
_simpleConsumer = new Consumer(_config, broker.Host, broker.Port);
}
/// <summary>
/// Initializes a new instance of the <see cref="Cluster"/> class.
/// </summary>
/// <param name="zkClient">IZooKeeperClient object</param>
public Cluster(IZooKeeperClient zkClient) : this()
{
var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath);
foreach (var node in nodes)
{
var brokerZkString = zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node);
Broker broker = this.CreateBroker(node, brokerZkString);
brokers[broker.Id] = broker;
}
}
internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List <PartitionTopicInfo> partitionTopicInfos, Action <PartitionTopicInfo> markPartitonWithError)
{
_name = name;
_zkClient = zkClient;
_config = config;
_broker = broker;
_partitionTopicInfos = partitionTopicInfos;
_fetchBufferLength = config.MaxFetchBufferLength;
_markPartitonWithError = markPartitonWithError;
_simpleConsumer = new Consumer(_config, broker.Host, broker.Port);
}
public ServiceDiscoveryManager(
[NotNull] IZooKeeperClient zooKeeperClient,
[CanBeNull] ServiceDiscoveryManagerSettings settings = null,
[CanBeNull] ILog log = null)
{
this.zooKeeperClient = zooKeeperClient ?? throw new ArgumentNullException(nameof(zooKeeperClient));
this.settings = settings ?? new ServiceDiscoveryManagerSettings();
this.log = (log ?? LogProvider.Get()).ForContext <ServiceDiscoveryManager>();
pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper);
}
/// <summary>
/// Initializes a new instance of the <see cref="Cluster"/> class.
/// </summary>
/// <param name="zkClient">IZooKeeperClient object</param>
public Cluster(IZooKeeperClient zkClient)
: this()
{
var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath);
foreach (var node in nodes)
{
var brokerZkString = zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node);
Broker broker = this.CreateBroker(node, brokerZkString);
brokers[broker.Id] = broker;
}
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException.NoNodeException)
{
ZkUtils.CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException.NoNodeException)
{
ZkUtils.CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
}
private void ConnectZk()
{
Logger.Info("Enter connectZk()");
if (UseSharedStaticZookeeperClient)
{
Logger.Info("After check UseSharedStaticZookeeperClient");
if (zkClientStatic == null || zkClientStatic != null &&
zkClientStatic.GetClientState() != KeeperState.SyncConnected)
{
Logger.Info("After check UseSharedStaticZookeeperClient, will lock");
lock (zkClientStaticLock)
{
Logger.Info("got lock ... ");
if (zkClientStatic == null || zkClientStatic != null &&
zkClientStatic.GetClientState() != KeeperState.SyncConnected)
{
Logger.InfoFormat("zkClientStatic: {0} will create one ...",
zkClientStatic == null ? "null" : "not null");
Logger.InfoFormat("Connecting to zookeeper instance at {0} STATIC",
config.ZooKeeper.ZkConnect);
zkClientStatic = new ZooKeeperClient(config.ZooKeeper.ZkConnect,
config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer,
config.ZooKeeper.ZkConnectionTimeoutMs);
zkClientStatic.Connect();
Logger.InfoFormat("Connecting to zookeeper instance at {0} STATIC. Done",
config.ZooKeeper.ZkConnect);
}
Logger.Info("release lock ... ");
}
}
Logger.InfoFormat("zkClientStatic: {0}", zkClientStatic == null ? "null" : "not null");
if (zkClientStatic != null)
{
Logger.InfoFormat("zkClientStatic.ClientState: {0}", zkClientStatic.GetClientState());
}
}
else
{
Logger.InfoFormat("Connecting to zookeeper instance at {0}", config.ZooKeeper.ZkConnect);
if (zkClientInternal != null)
{
zkClientInternal.Dispose();
zkClientInternal = null;
}
zkClientInternal = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs,
ZooKeeperStringSerializer.Serializer, config.ZooKeeper.ZkConnectionTimeoutMs);
zkClientInternal.Connect();
}
}
internal static int?GetLeaderForPartition(IZooKeeperClient zkClient, string topic, int partition)
{
var stateData =
zkClient.ReadData <string>(
GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true);
if (string.IsNullOrWhiteSpace(stateData))
{
return(null);
}
int leader;
return(TryParsePartitionLeader(stateData, out leader) ? (int?)leader : null);
}
public ServiceLocator(
[NotNull] IZooKeeperClient zooKeeperClient,
[CanBeNull] ServiceLocatorSettings settings = null,
[CanBeNull] ILog log = null)
{
this.zooKeeperClient = zooKeeperClient;
this.settings = settings ?? new ServiceLocatorSettings();
this.log = (log ?? LogProvider.Get()).ForContext <ServiceLocator>();
pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper);
eventsQueue = new ActionsQueue(this.log);
environmentsStorage = new EnvironmentsStorage(zooKeeperClient, pathHelper, eventsQueue, log);
applicationsStorage = new ApplicationsStorage(zooKeeperClient, pathHelper, eventsQueue, log);
}
/// <summary>
/// Initializes a new instance of the <see cref="Cluster"/> class.
/// </summary>
/// <param name="zkClient">IZooKeeperClient object</param>
public Cluster(IZooKeeperClient zkClient)
{
var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath);
foreach (var node in nodes)
{
var brokerZkString = zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node);
Broker broker = this.CreateBroker(node, brokerZkString);
if (brokers.ContainsKey(broker.Id))
{
brokers[broker.Id] = broker;
}
else
{
brokers.Add(broker.Id, broker);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Cluster"/> class.
/// </summary>
/// <param name="zkClient">IZooKeeperClient object</param>
public Cluster(IZooKeeperClient zkClient)
{
var nodes = zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath);
foreach (var node in nodes)
{
var brokerZkString = zkClient.ReadData <string>(ZooKeeperClient.DefaultBrokerIdsPath + "/" + node);
Broker broker = this.CreateBroker(node, brokerZkString);
if (brokers.ContainsKey(broker.Id))
{
brokers[broker.Id] = broker;
}
else
{
brokers.Add(broker.Id, broker);
}
}
}
internal static void DeletePath(IZooKeeperClient zkClient, string path)
{
try
{
zkClient.Delete(path);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
Logger.InfoFormat("{0} deleted during connection loss; this is ok", path);
}
else
{
throw;
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="BrokerTopicsListener"/> class.
/// </summary>
/// <param name="zkclient">The wrapper on ZooKeeper client.</param>
/// <param name="actualBrokerTopicsPartitionsMap">The actual broker topics partitions map.</param>
/// <param name="actualBrokerIdMap">The actual broker id map.</param>
/// <param name="callback">The callback invoked after new broker is added.</param>
public BrokerTopicsListener(
IZooKeeperClient zkclient,
IDictionary<string, SortedSet<Partition>> actualBrokerTopicsPartitionsMap,
IDictionary<int, Broker> actualBrokerIdMap,
Action<int, string, int> callback)
{
this.zkclient = zkclient;
this.actualBrokerTopicsPartitionsMap = actualBrokerTopicsPartitionsMap;
this.actualBrokerIdMap = actualBrokerIdMap;
this.callback = callback;
this.oldBrokerIdMap = new Dictionary<int, Broker>(this.actualBrokerIdMap);
this.oldBrokerTopicsPartitionsMap = new Dictionary<string, SortedSet<Partition>>(this.actualBrokerTopicsPartitionsMap);
Logger.Debug("Creating broker topics listener to watch the following paths - \n"
+ "/broker/topics, /broker/topics/topic, /broker/ids");
Logger.Debug("Initialized this broker topics listener with initial mapping of broker id to "
+ "partition id per topic with " + this.oldBrokerTopicsPartitionsMap.ToMultiString(
x => x.Key + " --> " + x.Value.ToMultiString(y => y.ToString(), ","), "; "));
}
/// <summary>
/// Initializes a new instance of the <see cref="BrokerTopicsListener"/> class.
/// </summary>
/// <param name="zkclient">The wrapper on ZooKeeper client.</param>
/// <param name="actualBrokerTopicsPartitionsMap">The actual broker topics partitions map.</param>
/// <param name="actualBrokerIdMap">The actual broker id map.</param>
/// <param name="callback">The callback invoked after new broker is added.</param>
public BrokerTopicsListener(
IZooKeeperClient zkclient,
IDictionary <string, SortedSet <Partition> > actualBrokerTopicsPartitionsMap,
IDictionary <int, Broker> actualBrokerIdMap,
Action <int, string, int> callback)
{
this.zkclient = zkclient;
this.actualBrokerTopicsPartitionsMap = actualBrokerTopicsPartitionsMap;
this.actualBrokerIdMap = actualBrokerIdMap;
this.callback = callback;
this.oldBrokerIdMap = new Dictionary <int, Broker>(this.actualBrokerIdMap);
this.oldBrokerTopicsPartitionsMap = new Dictionary <string, SortedSet <Partition> >(this.actualBrokerTopicsPartitionsMap);
Logger.Debug("Creating broker topics listener to watch the following paths - \n"
+ "/broker/topics, /broker/topics/topic, /broker/ids");
Logger.Debug("Initialized this broker topics listener with initial mapping of broker id to "
+ "partition id per topic with " + this.oldBrokerTopicsPartitionsMap.ToMultiString(
x => x.Key + " --> " + x.Value.ToMultiString(y => y.ToString(), ","), "; "));
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
else
{
throw;
}
}
}
internal static IDictionary <string, IList <string> > GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable <string> topics)
{
var result = new Dictionary <string, IList <string> >();
foreach (string topic in topics)
{
var partitions = zkClient.GetChildrenParentMayNotExist(GetTopicPartitionsPath(topic));
if (partitions == null)
{
throw new NoPartitionsForTopicException(topic);
}
Logger.DebugFormat("children of /brokers/topics/{0} are {1}", topic, string.Join(",", partitions));
result.Add(topic, partitions != null ? partitions.OrderBy(x => x).ToList() : new List <string>());
}
return(result);
}
/// <summary>
/// <para>Trying to update node data with optimistic concurrency strategy according to given <paramref name="request"/>.</para>
/// <para>Check returned <see cref="UpdateDataResult"/> to see if operation was successful.</para>
/// </summary>
public static async Task <UpdateDataResult> UpdateDataAsync(this IZooKeeperClient zooKeeperClient, UpdateDataRequest request)
{
try
{
for (var i = 0; i < request.Attempts; i++)
{
var readResult = await zooKeeperClient.GetDataAsync(new GetDataRequest(request.Path)).ConfigureAwait(false);
if (!readResult.IsSuccessful)
{
return(UpdateDataResult.Unsuccessful(readResult.Status, readResult.Path, readResult.Exception));
}
var newData = request.Update(readResult.Data);
if (ByteArrayKey.Equals(readResult.Data, newData))
{
return(UpdateDataResult.Successful(readResult.Path));
}
var setDataRequest = new SetDataRequest(request.Path, newData)
{
Version = readResult.Stat.Version
};
var updateResult = await zooKeeperClient.SetDataAsync(setDataRequest).ConfigureAwait(false);
if (updateResult.Status == ZooKeeperStatus.VersionsMismatch)
{
continue;
}
return(updateResult.IsSuccessful ? UpdateDataResult.Successful(updateResult.Path) : UpdateDataResult.Unsuccessful(updateResult.Status, updateResult.Path, updateResult.Exception));
}
return(UpdateDataResult.Unsuccessful(ZooKeeperStatus.VersionsMismatch, request.Path, null));
}
catch (Exception e)
{
return(UpdateDataResult.Unsuccessful(ZooKeeperStatus.UnknownError, request.Path, e));
}
}
internal ZKRebalancerListener(
ConsumerConfiguration config,
string consumerIdString,
IDictionary<string, IDictionary<Partition, PartitionTopicInfo>> topicRegistry,
IZooKeeperClient zkClient,
ZookeeperConsumerConnector zkConsumerConnector,
IDictionary<Tuple<string, string>, BlockingCollection<FetchedDataChunk>> queues,
Fetcher fetcher,
object syncLock)
{
this.syncLock = syncLock;
this.consumerIdString = consumerIdString;
this.config = config;
this.topicRegistry = topicRegistry;
this.zkClient = zkClient;
this.dirs = new ZKGroupDirs(config.GroupId);
this.zkConsumerConnector = zkConsumerConnector;
this.queues = queues;
this.fetcher = fetcher;
}
internal ServiceBeacon(
[NotNull] IZooKeeperClient zooKeeperClient,
[NotNull] ReplicaInfo replicaInfo,
[CanBeNull] ServiceBeaconSettings settings,
[CanBeNull] ILog log)
{
this.zooKeeperClient = zooKeeperClient ?? throw new ArgumentNullException(nameof(settings));
this.replicaInfo = replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(settings));
this.settings = settings ?? new ServiceBeaconSettings();
this.log = (log ?? LogProvider.Get()).ForContext <ServiceBeacon>();
var pathHelper = new ServiceDiscoveryPathHelper(this.settings.ZooKeeperNodesPrefix, this.settings.ZooKeeperNodesPathEscaper);
environmentNodePath = pathHelper.BuildEnvironmentPath(replicaInfo.Environment);
applicationNodePath = pathHelper.BuildApplicationPath(replicaInfo.Environment, replicaInfo.Application);
replicaNodePath = pathHelper.BuildReplicaPath(replicaInfo.Environment, replicaInfo.Application, replicaInfo.Replica);
replicaNodeData = ReplicaNodeDataSerializer.Serialize(replicaInfo);
nodeWatcher = new AdHocNodeWatcher(OnNodeEvent);
}
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.WriteData(path, data);
}
catch (KeeperException.NoNodeException)
{
CreateParentPath(zkClient, path);
try
{
zkClient.CreatePersistent(path, data);
}
catch (KeeperException.NodeExistsException)
{
zkClient.WriteData(path, data);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Fetcher"/> class.
/// </summary>
/// <param name="config">
/// The consumer configuration.
/// </param>
/// <param name="zkClient">
/// The wrapper above ZooKeeper client.
/// </param>
public Fetcher(ConsumerConfig config, IZooKeeperClient zkClient)
{
this.config = config;
this.zkClient = zkClient;
}
public static IEnumerable<Broker> GetAllBrokersInCluster(IZooKeeperClient zkClient)
{
var brokerIds = zkClient.GetChildren(ZooKeeperClient.DefaultBrokerIdsPath).OrderBy(x => x).ToList();
return ZkUtils.GetBrokerInfoFromIds(zkClient, brokerIds.Select(x => int.Parse(x)));
}
internal static void UpdatePersistentPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.WriteData(path, data);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
CreateParentPath(zkClient, path);
try
{
zkClient.CreatePersistent(path, data);
}
catch (KeeperException e2)
{
if (e2.ErrorCode == KeeperException.Code.NODEEXISTS)
zkClient.WriteData(path, data);
else
throw;
}
}
else
throw;
}
}
internal static void CreateEphemeralPath(IZooKeeperClient zkClient, string path, string data)
{
try
{
zkClient.CreateEphemeral(path, data);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
{
ZkUtils.CreateParentPath(zkClient, path);
zkClient.CreateEphemeral(path, data);
}
else
throw;
}
}
private void ConnectZk()
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "Connecting to zookeeper instance at {0}", this.config.ZkConnect);
this.zkClient = new ZooKeeperClient(this.config.ZkConnect, this.config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer);
this.zkClient.Connect();
}
internal static TopicPartitionState GetPartitionState(IZooKeeperClient zkClient, string topic, int partition)
{
var stateData = zkClient.ReadData<string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true);
if (string.IsNullOrWhiteSpace(stateData))
{
return null;
}
TopicPartitionState partitionState;
try
{
var ser = new JavaScriptSerializer();
var result = ser.Deserialize<Dictionary<string, object>>(stateData);
partitionState = new TopicPartitionState()
{
Leader = int.Parse(result["leader"].ToString()),
Leader_Epoch = int.Parse(result["leader_epoch"].ToString()),
Controller_Epoch = int.Parse(result["controller_epoch"].ToString()),
Verstion = int.Parse(result["version"].ToString()),
};
var isrArr = result["isr"] as System.Collections.ArrayList;
partitionState.Isr = isrArr != null ? isrArr.Cast<int>().ToArray() : null;
}
catch (Exception exc)
{
Logger.WarnFormat("Unexpected error while trying to get topic partition state for topic '{0}' partition '{1}'. Error: {2} ", topic, partition, exc.FormatException());
return null;
}
return partitionState;
}
internal static void DeletePath(IZooKeeperClient zkClient, string path)
{
try
{
zkClient.Delete(path);
}
catch (KeeperException e)
{
if (e.ErrorCode == KeeperException.Code.NONODE)
Logger.InfoFormat("{0} deleted during connection loss; this is ok", path);
else
throw;
}
}
internal static IDictionary<string, IList<string>> GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable<string> topics)
{
var result = new Dictionary<string, IList<string>>();
foreach (string topic in topics)
{
var partList = new List<string>();
var brokers =
zkClient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic);
foreach (var broker in brokers)
{
var numberOfParts =
int.Parse(
zkClient.ReadData<string>(ZooKeeperClient.DefaultBrokerTopicsPath + "/" + topic + "/" +
broker),
CultureInfo.CurrentCulture);
for (int i = 0; i < numberOfParts; i++)
{
partList.Add(broker + "-" + i);
}
}
partList.Sort();
result.Add(topic, partList);
}
return result;
}
internal static void DeletePath(IZooKeeperClient zkClient, string path)
{
try
{
zkClient.Delete(path);
}
catch (KeeperException.NoNodeException)
{
Logger.InfoFormat(CultureInfo.CurrentCulture, "{0} deleted during connection loss; this is ok", path);
}
}
internal static int? GetLeaderForPartition(IZooKeeperClient zkClient, string topic, int partition)
{
var stateData = zkClient.ReadData<string>(GetTopicPartitionStatePath(topic, partition.ToString(CultureInfo.InvariantCulture)), true);
if (string.IsNullOrWhiteSpace(stateData))
{
return (int?)null;
}
else
{
int leader;
return TryParsePartitionLeader(stateData, out leader) ? (int?)leader : null;
}
}
internal static IDictionary<string, IList<string>> GetPartitionsForTopics(IZooKeeperClient zkClient, IEnumerable<string> topics)
{
var result = new Dictionary<string, IList<string>>();
foreach (string topic in topics)
{
var partitions = zkClient.GetChildrenParentMayNotExist(GetTopicPartitionsPath(topic));
if (partitions == null)
{
throw new NoPartitionsForTopicException(topic);
}
Logger.DebugFormat("children of /brokers/topics/{0} are {1}", topic, string.Join(",", partitions));
result.Add(topic, partitions != null ? partitions.OrderBy(x => x).ToList() : new List<string>());
}
return result;
}