internal static ImmutableList<IMemberEvent> DiffMemberEvents(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip)) return ImmutableList.Create<IMemberEvent>();
var newMembers = newGossip.Members.Except(oldGossip.Members);
var membersGroupedByAddress = newGossip.Members
.Concat(oldGossip.Members)
.GroupBy(m => m.UniqueAddress);
var changedMembers = membersGroupedByAddress
.Where(g => g.Count() == 2 && g.First().Status != g.Skip(1).First().Status)
.Select(g => g.First());
var memberEvents = CollectMemberEvents(newMembers.Union(changedMembers));
var removedMembers = oldGossip.Members.Except(newGossip.Members);
var removedEvents = removedMembers.Select(m => new MemberRemoved(m.Copy(status: MemberStatus.Removed), m.Status));
return memberEvents.Concat(removedEvents).ToImmutableList();
}
internal static ImmutableHashSet<RoleLeaderChanged> DiffRolesLeader(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
return InternalDiffRolesLeader(oldGossip, newGossip, selfUniqueAddress).ToImmutableHashSet();
}
public Gossip Merge(Gossip that)
{
//TODO: Member ordering import?
// 1. merge vector clocks
var mergedVClock = _version.Merge(that._version);
// 2. merge members by selecting the single Member with highest MemberStatus out of the Member groups
var mergedMembers = EmptyMembers.Union(Member.PickHighestPriority(this._members, that._members));
// 3. merge reachability table by picking records with highest version
var mergedReachability = this._overview.Reachability.Merge(mergedMembers.Select(m => m.UniqueAddress),
that._overview.Reachability);
// 4. Nobody can have seen this new gossip yet
var mergedSeen = ImmutableHashSet.Create<UniqueAddress>();
return new Gossip(mergedMembers, new GossipOverview(mergedSeen, mergedReachability), mergedVClock);
}
internal static ImmutableList<UnreachableMember> DiffUnreachable(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip)) return ImmutableList.Create<UnreachableMember>();
var oldUnreachableNodes = oldGossip.Overview.Reachability.AllUnreachableOrTerminated;
return
newGossip.Overview.Reachability.AllUnreachableOrTerminated
.Where(a => !oldUnreachableNodes.Contains(a))
.Select(a => new UnreachableMember(newGossip.GetMember(a)))
.ToImmutableList();
}
/// <summary>
/// TBD
/// </summary>
/// <param name="oldGossip">TBD</param>
/// <param name="newGossip">TBD</param>
/// <returns>TBD</returns>
internal static ImmutableList <IMemberEvent> DiffMemberEvents(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip))
{
return(ImmutableList <IMemberEvent> .Empty);
}
var newMembers = newGossip.Members.Except(oldGossip.Members);
var membersGroupedByAddress = newGossip.Members
.Concat(oldGossip.Members)
.GroupBy(m => m.UniqueAddress);
var changedMembers = membersGroupedByAddress
.Where(g => g.Count() == 2 && (g.First().Status != g.Skip(1).First().Status || g.First().UpNumber != g.Skip(1).First().UpNumber))
.Select(g => g.First());
var memberEvents = CollectMemberEvents(newMembers.Union(changedMembers));
var removedMembers = oldGossip.Members.Except(newGossip.Members);
var removedEvents = removedMembers.Select(m => new MemberRemoved(m.Copy(status: MemberStatus.Removed), m.Status));
return(memberEvents.Concat(removedEvents).ToImmutableList());
}
internal static ImmutableHashSet<RoleLeaderChanged> DiffRolesLeader(Gossip oldGossip, Gossip newGossip)
{
return InternalDiffRolesLeader(oldGossip, newGossip).ToImmutableHashSet();
}
//Reply from Join request
public void Welcome(Address joinWith, UniqueAddress from, Gossip gossip)
{
if (!_latestGossip.Members.IsEmpty) throw new InvalidOperationException("Welcome can only be done from an empty state");
if (!joinWith.Equals(from.Address))
{
_log.Info("Ignoring welcome from [{0}] when trying to join with [{1}]", from.Address, joinWith);
}
else
{
_log.Info("Welcome from [{0}]", from.Address);
_latestGossip = gossip.Seen(SelfUniqueAddress);
Publish(_latestGossip);
if (!from.Equals(SelfUniqueAddress))
GossipTo(from, Sender);
BecomeInitialized();
}
}
private void PublishDiff(Gossip oldGossip, Gossip newGossip, Action<object> pub)
{
foreach (var @event in ClusterEvent.DiffMemberEvents(oldGossip, newGossip)) pub(@event);
foreach (var @event in ClusterEvent.DiffUnreachable(oldGossip, newGossip)) pub(@event);
foreach (var @event in ClusterEvent.DiffReachable(oldGossip, newGossip)) pub(@event);
foreach (var @event in ClusterEvent.DiffLeader(oldGossip, newGossip, _selfUniqueAddress)) pub(@event);
foreach (var @event in ClusterEvent.DiffRolesLeader(oldGossip, newGossip, _selfUniqueAddress)) pub(@event);
// publish internal SeenState for testing purposes
foreach (var @event in ClusterEvent.DiffSeen(oldGossip, newGossip, _selfUniqueAddress)) pub(@event);
foreach (var @event in ClusterEvent.DiffReachability(oldGossip, newGossip)) pub(@event);
}
internal static ImmutableHashSet <RoleLeaderChanged> DiffRolesLeader(Gossip oldGossip, Gossip newGossip)
{
return(InternalDiffRolesLeader(oldGossip, newGossip).ToImmutableHashSet());
}
public ClusterDomainEventPublisher()
{
_latestGossip = Gossip.Empty;
_eventStream = Context.System.EventStream;
}
internal static ImmutableList <ReachableMember> DiffReachable(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip))
{
return(ImmutableList.Create <ReachableMember>());
}
return
(oldGossip.Overview.Reachability.AllUnreachable
.Where(a => newGossip.HasMember(a) && newGossip.Overview.Reachability.IsReachable(a))
.Select(a => new ReachableMember(newGossip.GetMember(a)))
.ToImmutableList());
}
internal static ImmutableList <UnreachableMember> DiffUnreachable(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip))
{
return(ImmutableList.Create <UnreachableMember>());
}
var oldUnreachableNodes = oldGossip.Overview.Reachability.AllUnreachableOrTerminated;
return
(newGossip.Overview.Reachability.AllUnreachableOrTerminated
.Where(a => !oldUnreachableNodes.Contains(a))
.Select(a => new UnreachableMember(newGossip.GetMember(a)))
.ToImmutableList());
}
private bool Equals(MetricsGossipEnvelope other)
{
return(From.Equals(other.From) && Gossip.Equals(other.Gossip) && Reply.Equals(other.Reply));
}
internal static ImmutableList<SeenChanged> DiffSeen(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddres)
{
if (newGossip.Equals(oldGossip))
return ImmutableList.Create<SeenChanged>();
var newConvergence = newGossip.Convergence(selfUniqueAddres);
var newSeenBy = newGossip.SeenBy;
if (newConvergence != oldGossip.Convergence(selfUniqueAddres) || newSeenBy != oldGossip.SeenBy)
return ImmutableList.Create(new SeenChanged(newConvergence,
newSeenBy.Select(s => s.Address).ToImmutableHashSet()));
return ImmutableList.Create<SeenChanged>();
}
/// <summary>
/// The types of gossip actions that receive gossip has performed.
/// </summary>
public ReceiveGossipType ReceiveGossip(GossipEnvelope envelope)
{
var from = envelope.From;
var remoteGossip = envelope.Gossip;
var localGossip = _latestGossip;
if (remoteGossip.Equals(Gossip.Empty))
{
_log.Debug("Cluster Node [{0}] - Ignoring received gossip from [{1}] to protect against overload",
_cluster.SelfAddress, from);
return ReceiveGossipType.Ignored;
}
if (!envelope.To.Equals(SelfUniqueAddress))
{
_log.Info("Ignoring received gossip intended for someone else, from [{0}] to [{1}]",
from.Address, envelope.To);
return ReceiveGossipType.Ignored;
}
if (!localGossip.Overview.Reachability.IsReachable(SelfUniqueAddress, from))
{
_log.Info("Ignoring received gossip from unreachable [{0}]", from);
return ReceiveGossipType.Ignored;
}
if (localGossip.Members.All(m => !m.UniqueAddress.Equals(from)))
{
_log.Debug("Cluster Node [{0}] - Ignoring received gossip from unknown [{1}]", _cluster.SelfAddress, from);
return ReceiveGossipType.Ignored;
}
if (remoteGossip.Members.All(m => !m.UniqueAddress.Equals(SelfUniqueAddress)))
{
_log.Debug("Ignoring received gossip that does not contain myself, from [{0}]", from);
return ReceiveGossipType.Ignored;
}
var comparison = remoteGossip.Version.CompareTo(localGossip.Version);
Gossip winningGossip;
bool talkback;
ReceiveGossipType gossipType;
switch (comparison)
{
case VectorClock.Ordering.Same:
//same version
winningGossip = remoteGossip.MergeSeen(localGossip);
talkback = !remoteGossip.SeenByNode(SelfUniqueAddress);
gossipType = ReceiveGossipType.Same;
break;
case VectorClock.Ordering.Before:
//local is newer
winningGossip = localGossip;
talkback = true;
gossipType = ReceiveGossipType.Older;
break;
case VectorClock.Ordering.After:
//remote is newer
winningGossip = remoteGossip;
talkback = !remoteGossip.SeenByNode(SelfUniqueAddress);
gossipType = ReceiveGossipType.Newer;
break;
default:
//conflicting versions, merge
winningGossip = remoteGossip.Merge(localGossip);
talkback = true;
gossipType = ReceiveGossipType.Merge;
break;
}
_latestGossip = winningGossip.Seen(SelfUniqueAddress);
// for all new joining nodes we remove them from the failure detector
foreach (var node in _latestGossip.Members)
{
if (node.Status == MemberStatus.Joining && !localGossip.Members.Contains(node))
_cluster.FailureDetector.Remove(node.Address);
}
_log.Debug("Cluster Node [{0}] - Receiving gossip from [{1}]", _cluster.SelfAddress, from);
if (comparison == VectorClock.Ordering.Concurrent)
{
_log.Debug(@"""Couldn't establish a causal relationship between ""remote"" gossip and ""local"" gossip - Remote[{0}] - Local[{1}] - merged them into [{2}]""",
remoteGossip, localGossip, winningGossip);
}
if (_statsEnabled)
{
switch (gossipType)
{
case ReceiveGossipType.Merge:
_gossipStats = _gossipStats.IncrementMergeCount();
break;
case ReceiveGossipType.Same:
_gossipStats = _gossipStats.IncrementSameCount();
break;
case ReceiveGossipType.Newer:
_gossipStats = _gossipStats.IncrementNewerCount();
break;
case ReceiveGossipType.Older:
_gossipStats = _gossipStats.IncrementOlderCount();
break;
}
}
Publish(_latestGossip);
var selfStatus = _latestGossip.GetMember(SelfUniqueAddress).Status;
if (selfStatus == MemberStatus.Exiting || selfStatus == MemberStatus.Down)
Shutdown();
else if (talkback)
{
// send back gossip to sender() when sender() had different view, i.e. merge, or sender() had
// older or sender() had newer
GossipTo(from, Sender);
}
return gossipType;
}
public ClusterDomainEventPublisher()
{
_latestGossip = Gossip.Empty;
_eventStream = Context.System.EventStream;
}
/// <param name="from">the sender node in the cluster, i.e. the node that received the Join command</param>
/// <param name="gossip"></param>
public Welcome(UniqueAddress from, Gossip gossip)
{
_from = from;
_gossip = gossip;
}
public MembershipState(Gossip latestGossip, UniqueAddress selfUniqueAddress)
{
LatestGossip = latestGossip;
SelfUniqueAddress = selfUniqueAddress;
}
public void UpdateLatestGossip(Gossip newGossip)
{
// Updating the vclock version for the changes
var versionedGossip = newGossip.Increment(_vclockNode);
// Nobody else have seen this gossip but us
var seenVersionedGossip = versionedGossip.OnlySeen(SelfUniqueAddress);
// Update the state with the new gossip
_latestGossip = seenVersionedGossip;
}
/// <summary>
/// TBD
/// </summary>
/// <param name="oldGossip">TBD</param>
/// <param name="newGossip">TBD</param>
/// <param name="selfUniqueAddress">TBD</param>
/// <returns>TBD</returns>
internal static ImmutableHashSet <RoleLeaderChanged> DiffRolesLeader(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
return(InternalDiffRolesLeader(oldGossip, newGossip, selfUniqueAddress).ToImmutableHashSet());
}
public void Publish(Gossip newGossip)
{
_publisher.Tell(new InternalClusterAction.PublishChanges(newGossip));
if (_cluster.Settings.PublishStatsInterval == TimeSpan.MinValue) PublishInternalStats();
}
public Gossip MergeSeen(Gossip that)
{
return Copy(overview: _overview.Copy(seen: _overview.Seen.Union(that._overview.Seen)));
}
internal PublishChanges(Gossip newGossip)
{
_newGossip = newGossip;
}
public GossipEnvelope(UniqueAddress from, UniqueAddress to, Gossip gossip, Deadline deadline = null)
{
_from = from;
_to = to;
Gossip = gossip;
Deadline = deadline;
}
public Gossip MergeSeen(Gossip that)
{
return(Copy(overview: _overview.Copy(seen: _overview.Seen.Union(that._overview.Seen))));
}
internal static ImmutableList<ReachableMember> DiffReachable(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Equals(oldGossip)) return ImmutableList.Create<ReachableMember>();
return
oldGossip.Overview.Reachability.AllUnreachable
.Where(a => newGossip.HasMember(a) && newGossip.Overview.Reachability.IsReachable(a))
.Select(a => new ReachableMember(newGossip.GetMember(a)))
.ToImmutableList();
}
private static IEnumerable <RoleLeaderChanged> InternalDiffRolesLeader(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
foreach (var role in oldGossip.AllRoles.Union(newGossip.AllRoles))
{
var newLeader = newGossip.RoleLeader(role, selfUniqueAddress);
if (newLeader == null && oldGossip.RoleLeader(role, selfUniqueAddress) != null)
{
yield return(new RoleLeaderChanged(role, null));
}
if (newLeader != null && !newLeader.Equals(oldGossip.RoleLeader(role, selfUniqueAddress)))
{
yield return(new RoleLeaderChanged(role, newLeader.Address));
}
}
}
internal static ImmutableList<LeaderChanged> DiffLeader(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
var newLeader = newGossip.Leader(selfUniqueAddress);
if ((newLeader == null && oldGossip.Leader(selfUniqueAddress) == null)
|| newLeader != null && newLeader.Equals(oldGossip.Leader(selfUniqueAddress)))
return ImmutableList.Create<LeaderChanged>();
return ImmutableList.Create(newLeader == null
? new LeaderChanged(null)
: new LeaderChanged(newLeader.Address));
}
/// <summary>
/// TBD
/// </summary>
/// <param name="oldGossip">TBD</param>
/// <param name="newGossip">TBD</param>
/// <returns>TBD</returns>
internal static ImmutableList <ReachabilityChanged> DiffReachability(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Overview.Reachability.Equals(oldGossip.Overview.Reachability))
{
return(ImmutableList <ReachabilityChanged> .Empty);
}
return(ImmutableList.Create(new ReachabilityChanged(newGossip.Overview.Reachability)));
}
private static IEnumerable<RoleLeaderChanged> InternalDiffRolesLeader(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
foreach (var role in oldGossip.AllRoles.Union(newGossip.AllRoles))
{
var newLeader = newGossip.RoleLeader(role, selfUniqueAddress);
if(newLeader == null && oldGossip.RoleLeader(role, selfUniqueAddress) != null)
yield return new RoleLeaderChanged(role, null);
if(newLeader != null && !newLeader.Equals(oldGossip.RoleLeader(role, selfUniqueAddress)))
yield return new RoleLeaderChanged(role, newLeader.Address);
}
}
private void ClearState()
{
_latestGossip = Gossip.Empty;
}
internal static ImmutableList<ReachabilityChanged> DiffReachability(Gossip oldGossip, Gossip newGossip)
{
if (newGossip.Overview.Reachability.Equals(oldGossip.Overview.Reachability))
return ImmutableList.Create<ReachabilityChanged>();
return ImmutableList.Create(new ReachabilityChanged(newGossip.Overview.Reachability));
}
/// <summary>
/// TBD
/// </summary>
/// <param name="oldGossip">TBD</param>
/// <param name="newGossip">TBD</param>
/// <param name="selfUniqueAddress">TBD</param>
/// <returns>TBD</returns>
internal static ImmutableList <UnreachableMember> DiffUnreachable(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
if (newGossip.Equals(oldGossip))
{
return(ImmutableList <UnreachableMember> .Empty);
}
var oldUnreachableNodes = oldGossip.Overview.Reachability.AllUnreachableOrTerminated;
return(newGossip.Overview.Reachability.AllUnreachableOrTerminated
.Where(node => !oldUnreachableNodes.Contains(node) && !node.Equals(selfUniqueAddress))
.Select(node => new UnreachableMember(newGossip.GetMember(node)))
.ToImmutableList());
}
private void PublishChanges(Gossip newGossip)
{
var oldGossip = _latestGossip;
_latestGossip = newGossip;
PublishDiff(oldGossip, newGossip, Publish);
}
/// <summary>
/// TBD
/// </summary>
/// <param name="oldGossip">TBD</param>
/// <param name="newGossip">TBD</param>
/// <param name="selfUniqueAddress">TBD</param>
/// <returns>TBD</returns>
internal static ImmutableList <ReachableMember> DiffReachable(Gossip oldGossip, Gossip newGossip, UniqueAddress selfUniqueAddress)
{
if (newGossip.Equals(oldGossip))
{
return(ImmutableList <ReachableMember> .Empty);
}
return(oldGossip.Overview.Reachability.AllUnreachable
.Where(node => newGossip.HasMember(node) && newGossip.Overview.Reachability.IsReachable(node) && !node.Equals(selfUniqueAddress))
.Select(node => new ReachableMember(newGossip.GetMember(node)))
.ToImmutableList());
}
private void ClearState()
{
_latestGossip = Gossip.Empty;
}
public MembershipState Copy(Gossip gossip = null, UniqueAddress selfUniqueAddress = null)
{
return(new MembershipState(gossip ?? LatestGossip, selfUniqueAddress ?? SelfUniqueAddress));
}