public void ClearFetcherQueues <TData>(IList <PartitionTopicInfo> topicInfos, Cluster.Cluster cluster, IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared, IDictionary <string, IList <KafkaMessageStream <TData> > > kafkaMessageStreams) { if (kafkaMessageStreams != null) { foreach (var kafkaMessageStream in kafkaMessageStreams) { foreach (var stream in kafkaMessageStream.Value) { stream.Clear(); } } } Logger.Info("Cleared the data chunks in all the consumer message iterators"); // Clear all but the currently iterated upon chunk in the consumer thread's queue foreach (var queueToBeCleared in queuesToBeCleared) { while (queueToBeCleared.Count > 0) { queueToBeCleared.Take(); } } Logger.Info("Cleared all relevant queues for this fetcher"); }
public Replicator(ReplicatorSettings settings) { _settings = settings; _cluster = Cluster.Cluster.Get(Context.System); _selfAddress = _cluster.SelfAddress; _selfUniqueAddress = _cluster.SelfUniqueAddress; _log = Context.GetLogger(); if (_cluster.IsTerminated) { throw new ArgumentException("Cluster node must not be terminated"); } if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role)) { throw new ArgumentException($"The cluster node {_selfAddress} does not have the role {_settings.Role}"); } _gossipTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self); _notifyTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.NotifySubscribersInterval, _settings.NotifySubscribersInterval, Self, FlushChanges.Instance, Self); _pruningTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.PruningInterval, _settings.PruningInterval, Self, RemovedNodePruningTick.Instance, Self); _clockTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, ClockTick.Instance, Self); _serializer = Context.System.Serialization.FindSerializerForType(typeof(DataEnvelope)); _maxPruningDisseminationNanos = _settings.MaxPruningDissemination.Ticks * 100; _previousClockTime = DateTime.UtcNow.Ticks * 100; }
public Replicator(ReplicatorSettings settings) { _settings = settings; _cluster = Cluster.Cluster.Get(Context.System); _selfAddress = _cluster.SelfAddress; _selfUniqueAddress = _cluster.SelfUniqueAddress; _log = Context.GetLogger(); if (_cluster.IsTerminated) { throw new ArgumentException("Cluster node must not be terminated"); } if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role)) { throw new ArgumentException($"The cluster node {_selfAddress} does not have the role {_settings.Role}"); } _gossipTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self); _notifyTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.NotifySubscribersInterval, _settings.NotifySubscribersInterval, Self, FlushChanges.Instance, Self); _pruningTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.PruningInterval, _settings.PruningInterval, Self, RemovedNodePruningTick.Instance, Self); _clockTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, ClockTick.Instance, Self); _serializer = Context.System.Serialization.FindSerializerForType(typeof(DataEnvelope)); _maxPruningDisseminationNanos = _settings.MaxPruningDissemination.Ticks * 100; _previousClockTime = DateTime.UtcNow.Ticks * 100; _hasDurableKeys = settings.DurableKeys.Count > 0; var durableKeysBuilder = ImmutableHashSet <string> .Empty.ToBuilder(); var durableWildcardsBuilder = ImmutableHashSet <string> .Empty.ToBuilder(); foreach (var key in settings.DurableKeys) { if (key.EndsWith("*")) { durableWildcardsBuilder.Add(key.Substring(0, key.Length - 1)); } else { durableKeysBuilder.Add(key); } } _durableKeys = durableKeysBuilder.ToImmutable(); _durableWildcards = durableWildcardsBuilder.ToImmutable(); _durableStore = _hasDurableKeys ? Context.Watch(Context.ActorOf(_settings.DurableStoreProps, "durableStore")) : Context.System.DeadLetters; if (_hasDurableKeys) { Load(); } else { NormalReceive(); } }
internal ClusterClient(Configuration configuration, ILogger logger, Cluster.Cluster cluster, IStatistics statistics) { PrepareConfig(configuration); _configuration = configuration; _logger = logger; _cluster = cluster ?? new Cluster.Cluster(configuration, logger, statistics); _cluster.InternalError += e => _logger.LogError("Cluster internal error: " + e); _cluster.ConsumeRouter.MessageReceived += kr => MessageReceived(kr); Messages = Observable.FromEvent <RawKafkaRecord>(a => MessageReceived += a, a => MessageReceived -= a); _cluster.ProduceRouter.MessageExpired += (t, m) => MessageExpired(new RawKafkaRecord { Key = m.Key, Value = m.Value, Topic = t, Partition = Partitions.None, Offset = 0 }); ExpiredMessages = Observable.FromEvent <RawKafkaRecord>(a => MessageExpired += a, a => MessageExpired -= a); _cluster.ProduceRouter.MessageDiscarded += (t, m) => MessageDiscarded(new RawKafkaRecord { Key = m.Key, Value = m.Value, Topic = t, Partition = Partitions.None, Offset = 0 }); DiscardedMessages = Observable.FromEvent <RawKafkaRecord>(a => MessageDiscarded += a, a => MessageDiscarded -= a); _cluster.ProduceRouter.MessagesAcknowledged += (t, n) => ProduceAcknowledged(t, n); _cluster.Start(); }
private void CloseFetchersForQueues(Cluster.Cluster cluster, IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared, IDictionary <string, IList <KafkaMessageStream <TData> > > kafkaMessageStreams, ZookeeperConsumerConnector zkConsumerConnector) { if (fetcher != null) { var allPartitionInfos = new List <PartitionTopicInfo>(); foreach (var item in topicRegistry.Values) { foreach (var partitionTopicInfo in item.Values) { allPartitionInfos.Add(partitionTopicInfo); } } fetcher.Shutdown(); fetcher.ClearFetcherQueues(allPartitionInfos, cluster, queuesToBeCleared, kafkaMessageStreams); Logger.Info("Committing all offsets after clearing the fetcher queues"); if (config.AutoCommit) { zkConsumerConnector.CommitOffsets(); } } }
protected ReplicatorChaosSpec(ReplicatorChaosSpecConfig config) : base(config, typeof(ReplicatorChaosSpec)) { _cluster = Akka.Cluster.Cluster.Get(Sys); _timeout = Dilated(TimeSpan.FromSeconds(3)); _replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys) .WithRole("backend") .WithGossipInterval(TimeSpan.FromSeconds(1))), "replicator"); }
public ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config) { _cluster = Cluster.Cluster.Get(Sys); _timeout = Dilated(TimeSpan.FromSeconds(3)); _replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys) .WithGossipInterval(TimeSpan.FromSeconds(1)) .WithPruning(pruningInterval: TimeSpan.FromSeconds(1), maxPruningDissemination: _maxPruningDissemination)), "replicator"); }
protected DurableDataSpec(DurableDataSpecConfig config, Type type) : base(config, type) { InitialParticipantsValueFactory = Roles.Count; cluster = Akka.Cluster.Cluster.Get(Sys); writeTwo = new WriteTo(2, timeout); readTwo = new ReadFrom(2, timeout); first = config.First; second = config.Second; }
public PartitionLeaderFinder(ConcurrentQueue <PartitionTopicInfo> partitionsNeedingLeaders, Cluster.Cluster brokers, ConsumerConfiguration config, Action <PartitionTopicInfo, Broker> createNewFetcher) { _partitionsNeedingLeader = partitionsNeedingLeaders; _brokers = brokers; _config = config; _createNewFetcher = createNewFetcher; }
protected DurableDataSpecBase(DurableDataSpecConfig config, Type type) : base(config, type) { cluster = Akka.Cluster.Cluster.Get(Sys); var timeout = Dilated(14.Seconds()); // initialization of lmdb can be very slow in CI environment writeTwo = new WriteTo(2, timeout); readTwo = new ReadFrom(2, timeout); first = config.First; second = config.Second; }
private void CloseFetchers(Cluster.Cluster cluster, IDictionary <string, IList <string> > relevantTopicThreadIdsMap, ZookeeperConsumerConnector zkConsumerConnector) { Logger.Info("enter CloseFetchers ..."); var queuesToBeCleared = queues.Where(q => relevantTopicThreadIdsMap.ContainsKey(q.Key.Item1)) .Select(q => q.Value) .ToList(); CloseFetchersForQueues(cluster, queuesToBeCleared, kafkaMessageStreams, zkConsumerConnector); Logger.Info("exit CloseFetchers"); }
protected DurableDataPocoSpecBase(DurableDataPocoSpecConfig config, Type type) : base(config, type) { _cluster = Akka.Cluster.Cluster.Get(Sys); var timeout = Dilated(14.Seconds()); // initialization of lmdb can be very slow in CI environment _writeThree = new WriteTo(3, timeout); _readThree = new ReadFrom(3, timeout); _first = config.First; _second = config.Second; _third = config.Third; }
public JepsenInspiredInsertSpec(JepsenInspiredInsertSpecConfig config) : base(config) { _cluster = Cluster.Cluster.Get(Sys); _replicator = DistributedData.Get(Sys).Replicator; _nodes = Roles.Remove(Controller); _nodeCount = _nodes.Count; _timeout = Dilated(TimeSpan.FromSeconds(3)); _expectedData = Enumerable.Range(0, _totalCount).ToArray(); var nodeindex = _nodes.Zip(Enumerable.Range(0, _nodes.Count - 1), (name, i) => new KeyValuePair <int, RoleName>(i, name)) .ToImmutableDictionary(); _data = Enumerable.Range(0, _totalCount).GroupBy(i => nodeindex[i % _nodeCount]) .ToImmutableDictionary(x => x.Key, x => (IEnumerable <int>)x.ToArray()); }
private void SyncedRebalance(CancellationTokenSource cancellationTokenSource) { Logger.InfoFormat("Consumer {0} has entered rebalance", consumerIdString); lock (syncLock) { while (!cancellationTokenSource.IsCancellationRequested) { // Notify listeners that a rebalance is occurring OnConsumerRebalance(EventArgs.Empty); Logger.InfoFormat("Begin rebalancing consumer {0}", consumerIdString); try { // Query ZooKeeper for current broker metadata var cluster = new Cluster.Cluster(zkClient); // Begin consumer rebalance if (Rebalance(cluster, cancellationTokenSource)) { Logger.InfoFormat("End rebalancing consumer {0}", consumerIdString); break; } Logger.ErrorFormat("Rebalance return false, will retry. "); } catch (ObjectDisposedException objDisposedEx) { // some lower methods have methods like EnsureNotDisposed() implemented and so we should expect to catch here Logger.InfoFormat( "ObjectDisposedException in rebalance. Assume end of SyncedRebalance. Exception occurred during rebalance {1} for consumer {2}", config.ConsumeGroupRebalanceRetryIntervalMs, objDisposedEx.FormatException(), consumerIdString); cancellationTokenSource.Cancel(); } catch (Exception ex) { // Some unknown exception occurred, bail Logger.ErrorFormat( "Exception in rebalance. Will retry after {0} ms. Exception occurred during rebalance {1} for consumer {2}", config.ConsumeGroupRebalanceRetryIntervalMs, ex.FormatException(), consumerIdString); Thread.Sleep(config.ConsumeGroupRebalanceRetryIntervalMs); } } // Clear flag on exit to indicate rebalance has completed isRebalanceRunning = false; } Logger.InfoFormat("Consumer {0} has exited rebalance", consumerIdString); }
protected ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config, typeof(ReplicatorPruningSpec)) { _cluster = Akka.Cluster.Cluster.Get(Sys); _timeout = Dilated(TimeSpan.FromSeconds(3)); _replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys) .WithGossipInterval(TimeSpan.FromSeconds(1)) .WithPruning(pruningInterval: TimeSpan.FromSeconds(1), maxPruningDissemination: _maxPruningDissemination)), "replicator"); First = config.First; Second = config.Second; Third = config.Third; }
private void UpdateFetcher(Cluster.Cluster cluster) { var allPartitionInfos = new List <PartitionTopicInfo>(); foreach (var partitionInfos in topicRegistry.Values) { foreach (var partition in partitionInfos.Values) { allPartitionInfos.Add(partition); } } Logger.InfoFormat("Consumer {0} selected partitions : {1}", consumerIdString, string.Join(",", allPartitionInfos.OrderBy(x => x.PartitionId).Select(y => y.ToString()))); if (fetcher != null) { fetcher.InitConnections(allPartitionInfos, cluster); } }
public ReplicatorSpec(ReplicatorSpecConfig config) : base(config) { _config = config; _cluster = Cluster.Cluster.Get(Sys); var settings = ReplicatorSettings.Create(Sys).WithGossipInterval(TimeSpan.FromSeconds(1.0)).WithMaxDeltaElements(10); var props = Replicator.Props(settings); _replicator = Sys.ActorOf(props, "replicator"); _timeOut = Dilated(TimeSpan.FromSeconds(2.0)); _writeTwo = new WriteTo(2, _timeOut); _writeMajority = new WriteMajority(_timeOut); _writeAll = new WriteAll(_timeOut); _readTwo = new ReadFrom(2, _timeOut); _readMajority = new ReadMajority(_timeOut); _readAll = new ReadAll(_timeOut); }
protected ReplicatorSpec(ReplicatorSpecConfig config) : base(config, typeof(ReplicatorSpec)) { _config = config; _first = config.First; _second = config.Second; _third = config.Third; _cluster = Akka.Cluster.Cluster.Get(Sys); var settings = ReplicatorSettings.Create(Sys) .WithGossipInterval(TimeSpan.FromSeconds(1.0)) .WithMaxDeltaElements(10); var props = Replicator.Props(settings); _replicator = Sys.ActorOf(props, "replicator"); _timeOut = Dilated(TimeSpan.FromSeconds(3.0)); _writeTwo = new WriteTo(2, _timeOut); _writeMajority = new WriteMajority(_timeOut); _writeAll = new WriteAll(_timeOut); _readTwo = new ReadFrom(2, _timeOut); _readMajority = new ReadMajority(_timeOut); _readAll = new ReadAll(_timeOut); }
/// <summary> /// TBD /// </summary> /// <param name="cluster">TBD</param> /// <param name="counter">TBD</param> public LocalPNCounter(Cluster.Cluster cluster, PNCounter counter) : this(cluster.SelfUniqueAddress, counter) { }
/// <summary> /// Increment the counter with the delta specified. /// If the delta is negative then it will decrement instead of increment. /// </summary> public PNCounterDictionary <TKey> Increment(Cluster.Cluster node, TKey key, long delta = 1L) => Increment(node.SelfUniqueAddress, key, delta);
/// <summary> /// Removes an entry from the map. /// Note that if there is a conflicting update on another node the entry will /// not be removed after merge. /// </summary> public PNCounterDictionary <TKey> Remove(Cluster.Cluster node, TKey key) => Remove(node.SelfUniqueAddress, key);
/// <summary> /// Decrement the counter with the delta specified. /// If the delta is negative then it will increment instead of decrement. /// </summary> public PNCounter Decrement(Cluster.Cluster node, long delta = 1) => Decrement(node.SelfUniqueAddress, delta);
/// <summary> /// TBD /// </summary> /// <param name="cluster">TBD</param> /// <param name="crdt">TBD</param> public LocalORSet(Cluster.Cluster cluster, ORSet <T> crdt) : this(cluster.SelfUniqueAddress, crdt) { }
/// <summary> /// Removes an entry from the map. /// Note that if there is a conflicting update on another node the entry will /// not be removed after merge. /// </summary> public LWWDictionary <TKey, TValue> Remove(Cluster.Cluster node, TKey key) => Remove(node.SelfUniqueAddress, key);
/// <summary> /// Adds an entry to the map. /// /// You can provide your <paramref name="clock"/> implementation instead of using timestamps based /// on DateTime.UtcNow.Ticks time. The timestamp can for example be an /// increasing version number from a database record that is used for optimistic /// concurrency control. /// </summary> public LWWDictionary <TKey, TValue> SetItem(Cluster.Cluster node, TKey key, TValue value, Clock <TValue> clock = null) => SetItem(node.SelfUniqueAddress, key, value, clock);
/// <summary> /// TBD /// </summary> /// <param name="cluster">TBD</param> /// <param name="crdt">TBD</param> public LocalORDictionary(Cluster.Cluster cluster, ORDictionary <TKey, TVal> crdt) : this(cluster.SelfUniqueAddress, crdt) { }
/// <summary> /// Replace an element of a set associated with a key with a new one if it is different. This is useful when an element is removed /// and another one is added within the same Update. The order of addition and removal is important in order /// to retain history for replicated data. /// </summary> public ORMultiValueDictionary <TKey, TValue> ReplaceItem(Cluster.Cluster node, TKey key, TValue oldElement, TValue newElement) => ReplaceItem(node.SelfUniqueAddress, key, oldElement, newElement);
protected DurablePruningSpec(DurablePruningSpecConfig config) : base(config, typeof(DurablePruningSpec)) { InitialParticipantsValueFactory = Roles.Count; cluster = Akka.Cluster.Cluster.Get(Sys); timeout = Dilated(TimeSpan.FromSeconds(5)); }
internal ClusterClient(Configuration configuration, ILogger logger, Cluster.Cluster cluster) : this(configuration, logger, cluster, null) { }
/// <summary> /// Remove an element of a set associated with a key. If there are no more elements in the set then the /// entire set will be removed. /// </summary> public ORMultiValueDictionary <TKey, TValue> RemoveItem(Cluster.Cluster node, TKey key, TValue element) => RemoveItem(node.SelfUniqueAddress, key, element);