public void ClearFetcherQueues <TData>(IList <PartitionTopicInfo> topicInfos,
Cluster.Cluster cluster,
IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared,
IDictionary <string, IList <KafkaMessageStream <TData> > > kafkaMessageStreams)
{
if (kafkaMessageStreams != null)
{
foreach (var kafkaMessageStream in kafkaMessageStreams)
{
foreach (var stream in kafkaMessageStream.Value)
{
stream.Clear();
}
}
}
Logger.Info("Cleared the data chunks in all the consumer message iterators");
// Clear all but the currently iterated upon chunk in the consumer thread's queue
foreach (var queueToBeCleared in queuesToBeCleared)
{
while (queueToBeCleared.Count > 0)
{
queueToBeCleared.Take();
}
}
Logger.Info("Cleared all relevant queues for this fetcher");
}
public Replicator(ReplicatorSettings settings)
{
_settings = settings;
_cluster = Cluster.Cluster.Get(Context.System);
_selfAddress = _cluster.SelfAddress;
_selfUniqueAddress = _cluster.SelfUniqueAddress;
_log = Context.GetLogger();
if (_cluster.IsTerminated)
{
throw new ArgumentException("Cluster node must not be terminated");
}
if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role))
{
throw new ArgumentException($"The cluster node {_selfAddress} does not have the role {_settings.Role}");
}
_gossipTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self);
_notifyTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.NotifySubscribersInterval, _settings.NotifySubscribersInterval, Self, FlushChanges.Instance, Self);
_pruningTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.PruningInterval, _settings.PruningInterval, Self, RemovedNodePruningTick.Instance, Self);
_clockTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, ClockTick.Instance, Self);
_serializer = Context.System.Serialization.FindSerializerForType(typeof(DataEnvelope));
_maxPruningDisseminationNanos = _settings.MaxPruningDissemination.Ticks * 100;
_previousClockTime = DateTime.UtcNow.Ticks * 100;
}
public Replicator(ReplicatorSettings settings)
{
_settings = settings;
_cluster = Cluster.Cluster.Get(Context.System);
_selfAddress = _cluster.SelfAddress;
_selfUniqueAddress = _cluster.SelfUniqueAddress;
_log = Context.GetLogger();
if (_cluster.IsTerminated)
{
throw new ArgumentException("Cluster node must not be terminated");
}
if (!string.IsNullOrEmpty(_settings.Role) && !_cluster.SelfRoles.Contains(_settings.Role))
{
throw new ArgumentException($"The cluster node {_selfAddress} does not have the role {_settings.Role}");
}
_gossipTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, GossipTick.Instance, Self);
_notifyTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.NotifySubscribersInterval, _settings.NotifySubscribersInterval, Self, FlushChanges.Instance, Self);
_pruningTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.PruningInterval, _settings.PruningInterval, Self, RemovedNodePruningTick.Instance, Self);
_clockTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(_settings.GossipInterval, _settings.GossipInterval, Self, ClockTick.Instance, Self);
_serializer = Context.System.Serialization.FindSerializerForType(typeof(DataEnvelope));
_maxPruningDisseminationNanos = _settings.MaxPruningDissemination.Ticks * 100;
_previousClockTime = DateTime.UtcNow.Ticks * 100;
_hasDurableKeys = settings.DurableKeys.Count > 0;
var durableKeysBuilder = ImmutableHashSet <string> .Empty.ToBuilder();
var durableWildcardsBuilder = ImmutableHashSet <string> .Empty.ToBuilder();
foreach (var key in settings.DurableKeys)
{
if (key.EndsWith("*"))
{
durableWildcardsBuilder.Add(key.Substring(0, key.Length - 1));
}
else
{
durableKeysBuilder.Add(key);
}
}
_durableKeys = durableKeysBuilder.ToImmutable();
_durableWildcards = durableWildcardsBuilder.ToImmutable();
_durableStore = _hasDurableKeys
? Context.Watch(Context.ActorOf(_settings.DurableStoreProps, "durableStore"))
: Context.System.DeadLetters;
if (_hasDurableKeys)
{
Load();
}
else
{
NormalReceive();
}
}
internal ClusterClient(Configuration configuration, ILogger logger, Cluster.Cluster cluster, IStatistics statistics)
{
PrepareConfig(configuration);
_configuration = configuration;
_logger = logger;
_cluster = cluster ?? new Cluster.Cluster(configuration, logger, statistics);
_cluster.InternalError += e => _logger.LogError("Cluster internal error: " + e);
_cluster.ConsumeRouter.MessageReceived += kr => MessageReceived(kr);
Messages = Observable.FromEvent <RawKafkaRecord>(a => MessageReceived += a, a => MessageReceived -= a);
_cluster.ProduceRouter.MessageExpired +=
(t, m) =>
MessageExpired(new RawKafkaRecord
{
Key = m.Key,
Value = m.Value,
Topic = t,
Partition = Partitions.None,
Offset = 0
});
ExpiredMessages = Observable.FromEvent <RawKafkaRecord>(a => MessageExpired += a, a => MessageExpired -= a);
_cluster.ProduceRouter.MessageDiscarded +=
(t, m) =>
MessageDiscarded(new RawKafkaRecord
{
Key = m.Key,
Value = m.Value,
Topic = t,
Partition = Partitions.None,
Offset = 0
});
DiscardedMessages = Observable.FromEvent <RawKafkaRecord>(a => MessageDiscarded += a, a => MessageDiscarded -= a);
_cluster.ProduceRouter.MessagesAcknowledged += (t, n) => ProduceAcknowledged(t, n);
_cluster.Start();
}
private void CloseFetchersForQueues(Cluster.Cluster cluster,
IEnumerable <BlockingCollection <FetchedDataChunk> > queuesToBeCleared,
IDictionary <string, IList <KafkaMessageStream <TData> > > kafkaMessageStreams,
ZookeeperConsumerConnector zkConsumerConnector)
{
if (fetcher != null)
{
var allPartitionInfos = new List <PartitionTopicInfo>();
foreach (var item in topicRegistry.Values)
{
foreach (var partitionTopicInfo in item.Values)
{
allPartitionInfos.Add(partitionTopicInfo);
}
}
fetcher.Shutdown();
fetcher.ClearFetcherQueues(allPartitionInfos, cluster, queuesToBeCleared, kafkaMessageStreams);
Logger.Info("Committing all offsets after clearing the fetcher queues");
if (config.AutoCommit)
{
zkConsumerConnector.CommitOffsets();
}
}
}
protected ReplicatorChaosSpec(ReplicatorChaosSpecConfig config) : base(config, typeof(ReplicatorChaosSpec))
{
_cluster = Akka.Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithRole("backend")
.WithGossipInterval(TimeSpan.FromSeconds(1))), "replicator");
}
public ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config)
{
_cluster = Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1))
.WithPruning(pruningInterval: TimeSpan.FromSeconds(1), maxPruningDissemination: _maxPruningDissemination)),
"replicator");
}
protected DurableDataSpec(DurableDataSpecConfig config, Type type) : base(config, type)
{
InitialParticipantsValueFactory = Roles.Count;
cluster = Akka.Cluster.Cluster.Get(Sys);
writeTwo = new WriteTo(2, timeout);
readTwo = new ReadFrom(2, timeout);
first = config.First;
second = config.Second;
}
public PartitionLeaderFinder(ConcurrentQueue <PartitionTopicInfo> partitionsNeedingLeaders,
Cluster.Cluster brokers,
ConsumerConfiguration config,
Action <PartitionTopicInfo, Broker> createNewFetcher)
{
_partitionsNeedingLeader = partitionsNeedingLeaders;
_brokers = brokers;
_config = config;
_createNewFetcher = createNewFetcher;
}
protected DurableDataSpecBase(DurableDataSpecConfig config, Type type) : base(config, type)
{
cluster = Akka.Cluster.Cluster.Get(Sys);
var timeout = Dilated(14.Seconds()); // initialization of lmdb can be very slow in CI environment
writeTwo = new WriteTo(2, timeout);
readTwo = new ReadFrom(2, timeout);
first = config.First;
second = config.Second;
}
private void CloseFetchers(Cluster.Cluster cluster,
IDictionary <string, IList <string> > relevantTopicThreadIdsMap,
ZookeeperConsumerConnector zkConsumerConnector)
{
Logger.Info("enter CloseFetchers ...");
var queuesToBeCleared = queues.Where(q => relevantTopicThreadIdsMap.ContainsKey(q.Key.Item1))
.Select(q => q.Value)
.ToList();
CloseFetchersForQueues(cluster, queuesToBeCleared, kafkaMessageStreams, zkConsumerConnector);
Logger.Info("exit CloseFetchers");
}
protected DurableDataPocoSpecBase(DurableDataPocoSpecConfig config, Type type) : base(config, type)
{
_cluster = Akka.Cluster.Cluster.Get(Sys);
var timeout = Dilated(14.Seconds()); // initialization of lmdb can be very slow in CI environment
_writeThree = new WriteTo(3, timeout);
_readThree = new ReadFrom(3, timeout);
_first = config.First;
_second = config.Second;
_third = config.Third;
}
public JepsenInspiredInsertSpec(JepsenInspiredInsertSpecConfig config) : base(config)
{
_cluster = Cluster.Cluster.Get(Sys);
_replicator = DistributedData.Get(Sys).Replicator;
_nodes = Roles.Remove(Controller);
_nodeCount = _nodes.Count;
_timeout = Dilated(TimeSpan.FromSeconds(3));
_expectedData = Enumerable.Range(0, _totalCount).ToArray();
var nodeindex = _nodes.Zip(Enumerable.Range(0, _nodes.Count - 1), (name, i) => new KeyValuePair <int, RoleName>(i, name))
.ToImmutableDictionary();
_data = Enumerable.Range(0, _totalCount).GroupBy(i => nodeindex[i % _nodeCount])
.ToImmutableDictionary(x => x.Key, x => (IEnumerable <int>)x.ToArray());
}
private void SyncedRebalance(CancellationTokenSource cancellationTokenSource)
{
Logger.InfoFormat("Consumer {0} has entered rebalance", consumerIdString);
lock (syncLock)
{
while (!cancellationTokenSource.IsCancellationRequested)
{
// Notify listeners that a rebalance is occurring
OnConsumerRebalance(EventArgs.Empty);
Logger.InfoFormat("Begin rebalancing consumer {0}", consumerIdString);
try
{
// Query ZooKeeper for current broker metadata
var cluster = new Cluster.Cluster(zkClient);
// Begin consumer rebalance
if (Rebalance(cluster, cancellationTokenSource))
{
Logger.InfoFormat("End rebalancing consumer {0}", consumerIdString);
break;
}
Logger.ErrorFormat("Rebalance return false, will retry. ");
}
catch (ObjectDisposedException objDisposedEx)
{
// some lower methods have methods like EnsureNotDisposed() implemented and so we should expect to catch here
Logger.InfoFormat(
"ObjectDisposedException in rebalance. Assume end of SyncedRebalance. Exception occurred during rebalance {1} for consumer {2}",
config.ConsumeGroupRebalanceRetryIntervalMs, objDisposedEx.FormatException(),
consumerIdString);
cancellationTokenSource.Cancel();
}
catch (Exception ex)
{
// Some unknown exception occurred, bail
Logger.ErrorFormat(
"Exception in rebalance. Will retry after {0} ms. Exception occurred during rebalance {1} for consumer {2}",
config.ConsumeGroupRebalanceRetryIntervalMs, ex.FormatException(), consumerIdString);
Thread.Sleep(config.ConsumeGroupRebalanceRetryIntervalMs);
}
}
// Clear flag on exit to indicate rebalance has completed
isRebalanceRunning = false;
}
Logger.InfoFormat("Consumer {0} has exited rebalance", consumerIdString);
}
protected ReplicatorPruningSpec(ReplicatorPruningSpecConfig config) : base(config,
typeof(ReplicatorPruningSpec))
{
_cluster = Akka.Cluster.Cluster.Get(Sys);
_timeout = Dilated(TimeSpan.FromSeconds(3));
_replicator = Sys.ActorOf(Replicator.Props(ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1))
.WithPruning(pruningInterval: TimeSpan.FromSeconds(1),
maxPruningDissemination: _maxPruningDissemination)),
"replicator");
First = config.First;
Second = config.Second;
Third = config.Third;
}
private void UpdateFetcher(Cluster.Cluster cluster)
{
var allPartitionInfos = new List <PartitionTopicInfo>();
foreach (var partitionInfos in topicRegistry.Values)
{
foreach (var partition in partitionInfos.Values)
{
allPartitionInfos.Add(partition);
}
}
Logger.InfoFormat("Consumer {0} selected partitions : {1}", consumerIdString,
string.Join(",", allPartitionInfos.OrderBy(x => x.PartitionId).Select(y => y.ToString())));
if (fetcher != null)
{
fetcher.InitConnections(allPartitionInfos, cluster);
}
}
public ReplicatorSpec(ReplicatorSpecConfig config)
: base(config)
{
_config = config;
_cluster = Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys).WithGossipInterval(TimeSpan.FromSeconds(1.0)).WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(2.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
protected ReplicatorSpec(ReplicatorSpecConfig config)
: base(config, typeof(ReplicatorSpec))
{
_config = config;
_first = config.First;
_second = config.Second;
_third = config.Third;
_cluster = Akka.Cluster.Cluster.Get(Sys);
var settings = ReplicatorSettings.Create(Sys)
.WithGossipInterval(TimeSpan.FromSeconds(1.0))
.WithMaxDeltaElements(10);
var props = Replicator.Props(settings);
_replicator = Sys.ActorOf(props, "replicator");
_timeOut = Dilated(TimeSpan.FromSeconds(3.0));
_writeTwo = new WriteTo(2, _timeOut);
_writeMajority = new WriteMajority(_timeOut);
_writeAll = new WriteAll(_timeOut);
_readTwo = new ReadFrom(2, _timeOut);
_readMajority = new ReadMajority(_timeOut);
_readAll = new ReadAll(_timeOut);
}
/// <summary>
/// TBD
/// </summary>
/// <param name="cluster">TBD</param>
/// <param name="counter">TBD</param>
public LocalPNCounter(Cluster.Cluster cluster, PNCounter counter) : this(cluster.SelfUniqueAddress, counter)
{
}
/// <summary> /// Increment the counter with the delta specified. /// If the delta is negative then it will decrement instead of increment. /// </summary> public PNCounterDictionary <TKey> Increment(Cluster.Cluster node, TKey key, long delta = 1L) => Increment(node.SelfUniqueAddress, key, delta);
/// <summary> /// Removes an entry from the map. /// Note that if there is a conflicting update on another node the entry will /// not be removed after merge. /// </summary> public PNCounterDictionary <TKey> Remove(Cluster.Cluster node, TKey key) => Remove(node.SelfUniqueAddress, key);
/// <summary> /// Decrement the counter with the delta specified. /// If the delta is negative then it will increment instead of decrement. /// </summary> public PNCounter Decrement(Cluster.Cluster node, long delta = 1) => Decrement(node.SelfUniqueAddress, delta);
/// <summary>
/// TBD
/// </summary>
/// <param name="cluster">TBD</param>
/// <param name="crdt">TBD</param>
public LocalORSet(Cluster.Cluster cluster, ORSet <T> crdt) : this(cluster.SelfUniqueAddress, crdt)
{
}
/// <summary> /// Removes an entry from the map. /// Note that if there is a conflicting update on another node the entry will /// not be removed after merge. /// </summary> public LWWDictionary <TKey, TValue> Remove(Cluster.Cluster node, TKey key) => Remove(node.SelfUniqueAddress, key);
/// <summary>
/// Adds an entry to the map.
///
/// You can provide your <paramref name="clock"/> implementation instead of using timestamps based
/// on DateTime.UtcNow.Ticks time. The timestamp can for example be an
/// increasing version number from a database record that is used for optimistic
/// concurrency control.
/// </summary>
public LWWDictionary <TKey, TValue> SetItem(Cluster.Cluster node, TKey key, TValue value,
Clock <TValue> clock = null) => SetItem(node.SelfUniqueAddress, key, value, clock);
/// <summary>
/// TBD
/// </summary>
/// <param name="cluster">TBD</param>
/// <param name="crdt">TBD</param>
public LocalORDictionary(Cluster.Cluster cluster, ORDictionary <TKey, TVal> crdt) : this(cluster.SelfUniqueAddress, crdt)
{
}
/// <summary>
/// Replace an element of a set associated with a key with a new one if it is different. This is useful when an element is removed
/// and another one is added within the same Update. The order of addition and removal is important in order
/// to retain history for replicated data.
/// </summary>
public ORMultiValueDictionary <TKey, TValue> ReplaceItem(Cluster.Cluster node, TKey key, TValue oldElement,
TValue newElement) =>
ReplaceItem(node.SelfUniqueAddress, key, oldElement, newElement);
protected DurablePruningSpec(DurablePruningSpecConfig config) : base(config, typeof(DurablePruningSpec))
{
InitialParticipantsValueFactory = Roles.Count;
cluster = Akka.Cluster.Cluster.Get(Sys);
timeout = Dilated(TimeSpan.FromSeconds(5));
}
internal ClusterClient(Configuration configuration, ILogger logger, Cluster.Cluster cluster)
: this(configuration, logger, cluster, null)
{
}
/// <summary> /// Remove an element of a set associated with a key. If there are no more elements in the set then the /// entire set will be removed. /// </summary> public ORMultiValueDictionary <TKey, TValue> RemoveItem(Cluster.Cluster node, TKey key, TValue element) => RemoveItem(node.SelfUniqueAddress, key, element);
