public void ShouldDownAllMembersWhenNotEnoughNodes()
{
string refereeAddress = "akka.tcp://system@localhost:33333";
var referee = TestUtils.CreateMember(MemberStatus.Up, Address.Parse(refereeAddress));
var members =
TestUtils.CreateMembers(MemberStatus.Up)
.Take(4)
.Concat(new List <Member> {
referee
})
.ToImmutableSortedSet();
var unreachable =
members.Where(m => m != referee)
.Take(2)
.ToImmutableHashSet();
var state = new CurrentClusterState().Copy(members, unreachable);
var strategy = new KeepRefereeDowningStrategy(refereeAddress, downAllIfLessThanNodes: 4);
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(members)
.Should().BeTrue("We have 3 available members, which is less than downAllIfLessThanNodes 4, so down all members");
}
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
var oldest = clusterState.GetMembers(Role).SortByAge().FirstOrDefault();
var available = clusterState.GetAvailableMembers(Role);
var haveOldest = oldest != null && available.Contains(oldest);
var oldestIsAlone =
(haveOldest && available.Count == 1) ||
(!haveOldest && available.Count == clusterState.GetMembers(Role).Count - 1);
if (oldest == null)
{
return(clusterState.GetUnreachableMembers());
}
if (oldestIsAlone && DownIfAlone)
{
return new List <Member> {
oldest
}
}
;
return(haveOldest
? clusterState.GetUnreachableMembers()
: clusterState.GetMembers());
}
}
public void ShouldDownMembersInPartitionsWithoutReferee()
{
string refereeAddress = "akka.tcp://system@localhost:33333";
var referee = TestUtils.CreateMember(MemberStatus.Up, Address.Parse(refereeAddress));
var members =
TestUtils.CreateMembers(MemberStatus.Up)
.Take(4)
.Concat(new List <Member> {
referee
})
.ToImmutableSortedSet();
var unreachable =
members.Where(m => m != referee)
.Take(2)
.Concat(new List <Member> {
referee
})
.ToImmutableHashSet();
var state = new CurrentClusterState().Copy(members, unreachable);
var strategy = new KeepRefereeDowningStrategy(refereeAddress, downAllIfLessThanNodes: 0);
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(members)
.Should().BeTrue("When we don't have the referee, we down all members");
}
public static ImmutableHashSet <Member> GetMembers(this CurrentClusterState state, string role = null)
{
bool ShouldConsider(Member member) => CurrentClusterStateExtensions.ShouldConsider(member, role);
return(state.Members.Where(ShouldConsider)
.ToImmutableHashSet());
}
/// <summary>
/// The strategy named keep-majority will down the unreachable nodes if the
/// current node is in the majority part based on the last known membership information.
/// Otherwise down the reachable nodes, i.e. the own part. If the parts are of equal size
/// the part containing the node with the lowest address is kept.
/// </summary>
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
var members = clusterState.GetMembers(Role);
var unreachableMembers = clusterState.GetUnreachableMembers(Role);
var availableMembers = clusterState.GetAvailableMembers(Role);
int unreachableCount = unreachableMembers.Count;
int availableCount = availableMembers.Count;
if (availableCount == unreachableCount)
{
var oldest = clusterState.GetMembers(Role).SortByAge().FirstOrDefault();
if (oldest != null && availableMembers.Contains(oldest))
{
return(clusterState.GetUnreachableMembers());
}
else
{
return(clusterState.GetMembers());
}
}
return(availableCount < unreachableCount
//too few available, down our partition (entire members)
? clusterState.GetMembers()
//enough available, down unreachable
: clusterState.GetUnreachableMembers());;
}
public void ShouldNotDownPartitionWithOldestWhenEqualMembers()
{
var oldest = TestUtils.CreateMembers(MemberStatus.Up).First();
//Oldest is in the reachable members
var members = TestUtils.CreateMembers(MemberStatus.Up)
.Take(5)
.ToImmutableSortedSet()
.Add(oldest);
var unreachableMembers = members
.Where(m => !m.Equals(oldest))
.Take(3)
.ToImmutableHashSet();
var clusterState =
new CurrentClusterState().Copy(members: members, unreachable: unreachableMembers);
var strategy = new KeepMajorityDowningStrategy();
var victims = strategy.GetVictims(clusterState).ToList();
victims.ToImmutableSortedSet().SetEquals(unreachableMembers)
.Should().BeTrue("Partition with oldest should not be marked for downing");
}
public void ShouldDownUnreachableMembersInMajorityPartitionsConsideringRole()
{
string roleName = "SomeRole";
var roles = new string[] { roleName }.ToImmutableHashSet();
var membersInRole = TestUtils.CreateMembers(MemberStatus.Up, roles).Take(3).ToList();
var unreachableMembersInRole = membersInRole.Take(1).ToImmutableHashSet();
var members = TestUtils.CreateMembers(MemberStatus.Up)
.Take(4)
.Concat(membersInRole)
.ToImmutableSortedSet();
var clusterState =
new CurrentClusterState().Copy(members: members, unreachable: unreachableMembersInRole);
var strategy = new KeepMajorityDowningStrategy(roleName);
var victims = strategy.GetVictims(clusterState).ToList();
victims.ToImmutableHashSet().SetEquals(unreachableMembersInRole)
.Should().BeTrue("Unreachable members in majority should be marked for downing");
victims.All(v => v.HasRole(roleName))
.Should().BeTrue("We should only down members in the specified role");
}
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
bool haveReferee = clusterState.HasAvailableMember(Address);
return(!haveReferee || clusterState.GetAvailableMembers().Count < DownAllIfLessThanNodes
? clusterState.GetMembers()
: clusterState.GetUnreachableMembers());
}
public static bool HasAvailableMember(this CurrentClusterState state, Address address)
{
var member = state.Members.FirstOrDefault(m => m.Address.Equals(address)); //cannot use ==
var unavilable = state.Unreachable.FirstOrDefault(m => m.Address.Equals(address));
return(member != null && unavilable == null
? member.Status == MemberStatus.Up
: false);
}
public void NotifyNewCurrentState(CurrentClusterState newState)
{
currentState = newState;
var message = new ClusterStateMessage {
ClusterState = currentState
};
var json = JsonConvert.SerializeObject(message);
currentStateSubject.OnNext(json);
}
public void NotifyMemberRemoved(CurrentClusterState newState, Member memberRemoved)
{
currentState = newState;
var message = new MemberRemovedMessage
{
ClusterState = currentState,
MemberRemoved = memberRemoved
};
var json = JsonConvert.SerializeObject(message);
currentStateSubject.OnNext(json);
}
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
var members = clusterState.GetMembers(Role);
var unreachable = clusterState.GetUnreachableMembers(Role);
int availableCount = members.Count - unreachable.Count;
return(availableCount < QuorumSize
//too few available, down our partition
? clusterState.GetMembers()
//enough available, down unreachable
: clusterState.GetUnreachableMembers());
}
public void ShouldNotDownPartitionsWithEnoughMembers()
{
var members = TestUtils.CreateMembers(MemberStatus.Up).Take(3).ToImmutableSortedSet();
var clusterState =
new CurrentClusterState()
.Copy(members);
var strategy = new StaticQuorumDowningStrategy(quorumSize: 3);
var victims = strategy.GetVictims(clusterState);
victims.Count().Should().Be(0, "Partitions with quorum size or greater should not be marked for downing");
}
public void ShouldDownUnreachablesWhenEnoughAvailableMembers()
{
var members = TestUtils.CreateMembers(MemberStatus.Up).Take(5).ToImmutableSortedSet();
var unreachableMembers = members.Take(2).ToImmutableHashSet();
var clusterState =
new CurrentClusterState()
.Copy(members: members, unreachable: unreachableMembers);
var strategy = new StaticQuorumDowningStrategy(quorumSize: 3);
var victims = strategy.GetVictims(clusterState).ToList();
victims.ToImmutableHashSet().SetEquals(unreachableMembers)
.Should().BeTrue("Partitions with available members >= quorum size should down unreachable members");
}
public void ShouldDownPartitionsWithTooFewAvailableMembers()
{
var members = TestUtils.CreateMembers(MemberStatus.Up).Take(3).ToImmutableSortedSet();
var unreachableMembers = members.Take(1).ToImmutableHashSet();
var clusterState =
new CurrentClusterState()
.Copy(members, unreachableMembers);
var strategy = new StaticQuorumDowningStrategy(quorumSize: 3);
var victims = strategy.GetVictims(clusterState);
victims.ToImmutableHashSet().SetEquals(members)
.Should().BeTrue("Partitions with fewer available members than the quorum size should be marked for downing");
}
public void ShouldDownMinorityPartitions()
{
var strategy = new KeepMajorityDowningStrategy();
var members = TestUtils.CreateMembers(MemberStatus.Up).Take(5).ToImmutableSortedSet();
var unreachableMembers = members.Take(3).ToImmutableHashSet();
var clusterState =
new CurrentClusterState()
.Copy(members: members, unreachable: unreachableMembers);
var victims = strategy.GetVictims(clusterState).ToList();
victims.ToImmutableHashSet().SetEquals(members)
.Should().BeTrue("Minority partitions should be marked for downing");
}
public void ShouldNotDownPartitionsWithEnoughMembersInRole()
{
string roleName = "SomeRole";
var roles = new string[] { roleName }.ToImmutableHashSet();
var membersInRole = TestUtils.CreateMembers(MemberStatus.Up, roles).Take(3).ToList();
var members =
TestUtils.CreateMembers(MemberStatus.Up)
.Take(2)
.Concat(membersInRole)
.ToImmutableSortedSet();
var clusterState = new CurrentClusterState().Copy(members);
var strategy = new StaticQuorumDowningStrategy(quorumSize: 3, role: roleName);
var victims = strategy.GetVictims(clusterState);
victims.Count().Should().Be(0, "Partitions with fewer than the quorum size should be marked for downing");
}
public void ShouldDownRemoteOldestWhenOldestAndAlone()
{
var oldest = TestUtils.CreateMembers(MemberStatus.Up).First();
var rest = TestUtils.CreateMembers(MemberStatus.Up)
.Take(4)
.ToImmutableSortedSet();
var members = rest.Add(oldest);
var unreachable = ImmutableHashSet <Member> .Empty.Add(oldest);
var state = new CurrentClusterState().Copy(members, unreachable);
var strategy = new KeepOldestDowningStrategy(downIfAlone: true);
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(ImmutableHashSet <Member> .Empty.Add(oldest))
.Should().BeTrue("When downIfAlone=true, we should down the oldest member if its clustered by itself");
}
public void ShouldDownSelfWhenNotInOldestPartition()
{
var oldest = TestUtils.CreateMembers(MemberStatus.Up).First();
var members = TestUtils.CreateMembers(MemberStatus.Up)
.Take(4)
.ToImmutableSortedSet()
.Add(oldest);
var unreachable = ImmutableHashSet <Member> .Empty.Add(oldest);
var state = new CurrentClusterState().Copy(members, unreachable);
var strategy = new KeepOldestDowningStrategy();
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(members)
.Should().BeTrue("When we don't have the oldest node, down ourselves");
}
private void Stable(CurrentClusterState clusterState)
{
Receive <IClusterDomainEvent>(msg =>
{
Stash.Stash();
Become(() => WaitingForStability(clusterState));
Stash.UnstashAll();
});
if (clusterState.Leader != null && clusterState.Leader.Equals(cluster.SelfAddress))
{
log.Info($"Checking downing strategy {downingStrategy.GetType().Name} for leader {clusterState.Leader} on node {cluster.SelfAddress}");
foreach (var victim in downingStrategy.GetVictims(clusterState))
{
log.Warning($"Downing victim {victim}");
cluster.Down(victim.Address);
}
}
}
public void ShouldDownUnreachablesInOldestPartition()
{
var oldest = TestUtils.CreateMembers(MemberStatus.Up)
.Take(3)
.ToImmutableSortedSet();
var newest = TestUtils.CreateMembers(MemberStatus.Up)
.Take(2)
.ToImmutableSortedSet();
var members = oldest.Union(newest);
var unreachable = newest.ToImmutableHashSet();
var state = new CurrentClusterState().Copy(members, unreachable);
var strategy = new KeepOldestDowningStrategy();
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(unreachable)
.Should().BeTrue("When we have the oldest node, the unreachable nodes should be downed");
}
public void ShouldDownUnreachableWhenNoOldest()
{
var oldest = TestUtils.CreateMembers(MemberStatus.Up)
.Take(3)
.ToImmutableSortedSet();
var members = TestUtils.CreateMembers(MemberStatus.Up)
.Take(5)
.ToImmutableSortedSet();
var unreachable = members.Take(2).ToImmutableHashSet();
var state = new CurrentClusterState().Copy(members, unreachable);
//There will be no oldest when constrained to this role
var strategy = new KeepOldestDowningStrategy(role: "SomeRole");
var victims = strategy.GetVictims(state);
victims.ToImmutableHashSet().SetEquals(unreachable)
.Should().BeTrue("When there isn't an oldest node, the unreachable nodes should be downed");
}
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
return(Enumerable.Empty <Member>());
}
private void WaitingForStability(CurrentClusterState clusterState)
{
log.Debug($"Waiting {stableAfter.TotalSeconds} seconds for cluster stability");
var timeoutHandle =
Context.System.Scheduler.ScheduleTellOnceCancelable(
stableAfter,
Self,
new Timeout(),
Self);
Receive <Timeout>(msg =>
{
Become(() => Stable(clusterState));
});
Receive <IClusterDomainEvent>(msg =>
{
timeoutHandle.CancelIfNotNull();
CurrentClusterState newState;
switch (msg)
{
case LeaderChanged changed:
newState = clusterState.Copy(leader: changed.Leader);
break;
case RoleLeaderChanged changed:
var roleLeaders =
clusterState.AllRoles
.Select(role => (Role: role, Leader: clusterState.RoleLeader(role)))
.Where(t => t.Leader != null)
.ToImmutableDictionary(t => t.Role, t => t.Leader);
newState = clusterState.Copy(roleLeaderMap: roleLeaders.SetItem(changed.Role, changed.Leader));
break;
case MemberRemoved member:
newState = clusterState.Copy(
members: clusterState.Members.Remove(member.Member),
unreachable: clusterState.Unreachable.Remove(member.Member));
break;
case MemberUp member:
newState = clusterState.Copy(members: clusterState.Members.Add(member.Member));
break;
case UnreachableMember member:
newState = clusterState.Copy(unreachable: clusterState.Unreachable.Add(member.Member));
break;
case ReachableMember member:
newState = clusterState.Copy(unreachable: clusterState.Unreachable.Remove(member.Member));
break;
default:
newState = clusterState;
break;
}
Become(() => WaitingForStability(newState));
});
}
public IEnumerable <Member> GetVictims(CurrentClusterState clusterState)
{
_system.EventStream.Publish(new Execution());
return(Enumerable.Empty <Member>());
}
