/// <summary>
/// TBD
/// </summary>
/// <param name="typeName">TBD</param>
/// <param name="settings">TBD</param>
/// <param name="allocationStrategy">TBD</param>
public PersistentShardCoordinator(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy)
{
TypeName = typeName;
Settings = settings;
_currentState = State.Empty.WithRememberEntities(settings.RememberEntities);
AllocationStrategy = allocationStrategy;
RemovalMargin = Cluster.DowningProvider.DownRemovalMargin;
if (string.IsNullOrEmpty(settings.Role))
{
MinMembers = Cluster.Settings.MinNrOfMembers;
}
else
{
MinMembers = Cluster.Settings.MinNrOfMembersOfRole.GetValueOrDefault(settings.Role, Cluster.Settings.MinNrOfMembers);
}
JournalPluginId = Settings.JournalPluginId;
SnapshotPluginId = Settings.SnapshotPluginId;
_rebalanceTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(Settings.TunningParameters.RebalanceInterval, Settings.TunningParameters.RebalanceInterval, Self, RebalanceTick.Instance, Self);
Cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, new[] { typeof(ClusterEvent.ClusterShuttingDown) });
}
static ImmutableList <int> AllocationCountsAfterRebalance(
IShardAllocationStrategy allocationStrategy,
IImmutableDictionary <IActorRef, IImmutableList <string> > allocations,
IImmutableSet <string> rebalance)
{
return(CountShardsPerRegion(AfterRebalance(allocationStrategy, allocations, rebalance)));
}
private void TestRebalance(
IShardAllocationStrategy allocationStrategy,
IImmutableDictionary <IActorRef, IImmutableList <string> > allocations,
ImmutableList <IImmutableDictionary <IActorRef, IImmutableList <string> > > steps,
int maxSteps)
{
var round = steps.Count;
var rebalanceResult = allocationStrategy.Rebalance(allocations, ImmutableHashSet <string> .Empty).Result;
var newAllocations = LeastShardAllocationStrategySpec.AfterRebalance(allocationStrategy, allocations, rebalanceResult);
LeastShardAllocationStrategySpec.CountShards(newAllocations).Should().Be(LeastShardAllocationStrategySpec.CountShards(allocations), $"test {allocationStrategy}[{ string.Join(", ", LeastShardAllocationStrategySpec.CountShardsPerRegion(allocations))}]: ");
var min = LeastShardAllocationStrategySpec.CountShardsPerRegion(newAllocations).Min();
var max = LeastShardAllocationStrategySpec.CountShardsPerRegion(newAllocations).Max();
var diff = max - min;
var newSteps = steps.Add(newAllocations);
if (diff <= 1)
{
if (round >= 3 && maxSteps <= 10)
{
// Should be very rare (I have not seen it)
Sys.Log.Info($"rebalance solved in round {round}, [{string.Join(" => ", newSteps.Select(step => string.Join(", ", LeastShardAllocationStrategySpec.CountShardsPerRegion(step))))}]");
}
}
else if (round == maxSteps)
{
throw new AssertionFailedException($"Couldn't solve rebalance in $round rounds, [{string.Join(" => ", newSteps.Select(step => string.Join(", ", LeastShardAllocationStrategySpec.CountShardsPerRegion(step))))}]");
}
else
{
TestRebalance(allocationStrategy, newAllocations, newSteps, maxSteps);
}
}
private void TestRebalance(
IShardAllocationStrategy allocationStrategy,
int maxRegions,
int maxShardsPerRegion,
int expectedMaxSteps)
{
foreach (var i in Enumerable.Range(1, iterationsPerTest))
{
iteration += 1;
var numberOfRegions = rnd.Next(maxRegions) + 1;
var memberArray = Enumerable.Range(1, numberOfRegions).Select(n => LeastShardAllocationStrategySpec.NewUpMember("127.0.0.1", port: n)).ToArray();
clusterMembers = ImmutableSortedSet.Create(memberArray);//.toIndexedSeq: _ *);
var regions = Enumerable.Range(1, numberOfRegions).Select(n => LeastShardAllocationStrategySpec.NewFakeRegion($"{iteration}-R{n}", memberArray[n - 1]));
//var regions = Enumerable.Range(1, numberOfRegions).Select(n => Sys.ActorOf(Props.Empty, $"{iteration}-R{n}")).ToImmutableList();
var countPerRegion = regions.ToImmutableDictionary(region => region, region => rnd.Next(maxShardsPerRegion));
var allocations = CreateAllocations(countPerRegion);
TestRebalance(allocationStrategy, allocations, ImmutableList.Create(allocations), expectedMaxSteps);
foreach (var region in regions)
{
Sys.Stop(region);
}
}
}
public LeastShardAllocationStrategyRandomizedSpec()
{
rndSeed = DateTime.UtcNow.Millisecond;
rnd = new Random(rndSeed);
Log.Info($"Random seed: {rndSeed}");
strategyWithoutLimits = StrategyWithFakeCluster();
}
/// <summary>
/// Register a named entity type by defining the <see cref="Actor.Props"/> of the entity actor and
/// functions to extract entity and shard identifier from messages. The <see cref="Sharding.ShardRegion"/>
/// actor for this type can later be retrieved with the <see cref="ShardRegion"/> method.
/// </summary>
/// <param name="typeName">The name of the entity type</param>
/// <param name="entityProps">
/// The <see cref="Actor.Props"/> of the entity actors that will be created by the <see cref="Sharding.ShardRegion"/>
/// </param>
/// <param name="settings">Configuration settings, see <see cref="ClusterShardingSettings"/></param>
/// <param name="messageExtractor">
/// Functions to extract the entity id, shard id, and the message to send to the entity from the incoming message.
/// </param>
/// <param name="allocationStrategy">Possibility to use a custom shard allocation and rebalancing logic</param>
/// <param name="handOffMessage">
/// The message that will be sent to entities when they are to be stopped for a rebalance or
/// graceful shutdown of a <see cref="Sharding.ShardRegion"/>, e.g. <see cref="PoisonPill"/>.
/// </param>
/// <returns>The actor ref of the <see cref="Sharding.ShardRegion"/> that is to be responsible for the shard.</returns>
public Task <IActorRef> StartAsync(string typeName, Props entityProps, ClusterShardingSettings settings,
IMessageExtractor messageExtractor, IShardAllocationStrategy allocationStrategy, object handOffMessage)
{
IdExtractor idExtractor = messageExtractor.ToIdExtractor();
ShardResolver shardResolver = messageExtractor.ShardId;
return(StartAsync(typeName, entityProps, settings, idExtractor, shardResolver, allocationStrategy, handOffMessage));
}
public LeastShardAllocationStrategySpec()
{
_regionA = Sys.ActorOf(Props.Empty, "regionA");
_regionB = Sys.ActorOf(Props.Empty, "regionB");
_regionC = Sys.ActorOf(Props.Empty, "regionC");
_allocationStrategy = new LeastShardAllocationStrategy(3, 2);
}
public LeastShardAllocationStrategySpec() : base(new XunitAssertions(), "LeastShardAllocationStrategySpec")
{
_regionA = Sys.ActorOf(Props.Empty, "regionA");
_regionB = Sys.ActorOf(Props.Empty, "regionB");
_regionC = Sys.ActorOf(Props.Empty, "regionC");
_allocationStrategy = new LeastShardAllocationStrategy(3, 2);
}
/// <summary>
/// Register a named entity type by defining the <see cref="Actor.Props"/> of the entity actor and
/// functions to extract entity and shard identifier from messages. The <see cref="Sharding.ShardRegion"/>
/// actor for this type can later be retrieved with the <see cref="ShardRegion"/> method.
/// </summary>
/// <param name="typeName">The name of the entity type</param>
/// <param name="entityProps">
/// The <see cref="Actor.Props"/> of the entity actors that will be created by the <see cref="Sharding.ShardRegion"/>
/// </param>
/// <param name="settings">Configuration settings, see <see cref="ClusterShardingSettings"/></param>
/// <param name="messageExtractor">
/// Functions to extract the entity id, shard id, and the message to send to the entity from the incoming message.
/// </param>
/// <param name="allocationStrategy">Possibility to use a custom shard allocation and rebalancing logic</param>
/// <param name="handOffMessage">
/// The message that will be sent to entities when they are to be stopped for a rebalance or
/// graceful shutdown of a <see cref="Sharding.ShardRegion"/>, e.g. <see cref="PoisonPill"/>.
/// </param>
/// <returns>The actor ref of the <see cref="Sharding.ShardRegion"/> that is to be responsible for the shard.</returns>
public Task <IActorRef> StartAsync(string typeName, Props entityProps, ClusterShardingSettings settings,
IMessageExtractor messageExtractor, IShardAllocationStrategy allocationStrategy, object handOffMessage)
{
ExtractEntityId extractEntityId = messageExtractor.ToExtractEntityId();
ExtractShardId extractShardId = messageExtractor.ShardId;
return(StartAsync(typeName, entityProps, settings, extractEntityId, extractShardId, allocationStrategy, handOffMessage));
}
public LeastShardAllocationStrategySpec() : base(new XunitAssertions(), "LeastShardAllocationStrategySpec")
{
_regionA = Sys.ActorOf(Props.Empty, "regionA");
_regionB = Sys.ActorOf(Props.Empty, "regionB");
_regionC = Sys.ActorOf(Props.Empty, "regionC");
_allocationStrategy = new LeastShardAllocationStrategy(3, 2);
}
public LeastShardAllocationStrategySpec()
{
memberA = NewUpMember("127.0.0.1");
memberB = NewUpMember("127.0.0.2");
memberC = NewUpMember("127.0.0.3");
regionA = NewFakeRegion("regionA", memberA);
regionB = NewFakeRegion("regionB", memberB);
regionC = NewFakeRegion("regionC", memberC);
strategyWithoutLimits = StrategyWithFakeCluster(absoluteLimit: 1000, relativeLimit: 1.0);
}
public PersistentShardCoordinator(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy)
{
TypeName = typeName;
Settings = settings;
AllocationStrategy = allocationStrategy;
DownRemovalMargin = Cluster.Settings.DownRemovalMargin;
JournalPluginId = Settings.JournalPluginId;
SnapshotPluginId = Settings.SnapshotPluginId;
_rebalanceTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(Settings.TunningParameters.RebalanceInterval, Settings.TunningParameters.RebalanceInterval, Self, RebalanceTick.Instance, Self);
Cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, new[] { typeof(ClusterEvent.ClusterShuttingDown) });
}
public Start(string typeName, Props entityProps, ClusterShardingSettings settings,
IdExtractor idIdExtractor, ShardResolver shardResolver, IShardAllocationStrategy allocationStrategy, object handOffStopMessage)
{
if (string.IsNullOrEmpty(typeName)) throw new ArgumentNullException("typeName", "ClusterSharding start requires type name to be provided");
if (entityProps == null) throw new ArgumentNullException("entityProps", string.Format("ClusterSharding start requires Props for [{0}] to be provided", typeName));
TypeName = typeName;
EntityProps = entityProps;
Settings = settings;
IdExtractor = idIdExtractor;
ShardResolver = shardResolver;
AllocationStrategy = allocationStrategy;
HandOffStopMessage = handOffStopMessage;
}
internal static IImmutableDictionary <IActorRef, IImmutableList <string> > AfterRebalance(
IShardAllocationStrategy allocationStrategy,
IImmutableDictionary <IActorRef, IImmutableList <string> > allocations,
IImmutableSet <string> rebalance)
{
var allocationsAfterRemoval = allocations.SetItems(allocations.Select(i => new KeyValuePair <IActorRef, IImmutableList <string> >(i.Key, i.Value.ToImmutableHashSet().Except(rebalance).OrderBy(j => j).ToImmutableList())));
IImmutableDictionary <IActorRef, IImmutableList <string> > acc = allocationsAfterRemoval;
foreach (var shard in rebalance.OrderBy(i => i))
{
var region = allocationStrategy.AllocateShard(new DummyActorRef(), shard, acc).Result;
acc = acc.SetItem(region, acc[region].Add(shard));
}
return(acc);
}
/// <summary>
/// Register a named entity type by defining the <see cref="Actor.Props"/> of the entity actor and
/// functions to extract entity and shard identifier from messages. The <see cref="Sharding.ShardRegion"/>
/// actor for this type can later be retrieved with the <see cref="ShardRegion"/> method.
/// </summary>
/// <param name="typeName">The name of the entity type</param>
/// <param name="entityProps">
/// The <see cref="Actor.Props"/> of the entity actors that will be created by the <see cref="Sharding.ShardRegion"/>
/// </param>
/// <param name="settings">Configuration settings, see <see cref="ClusterShardingSettings"/></param>
/// <param name="extractEntityId">
/// Partial function to extract the entity id and the message to send to the entity from the incoming message,
/// if the partial function does not match the message will be `unhandled`,
/// i.e.posted as `Unhandled` messages on the event stream
/// </param>
/// <param name="extractShardId">
/// Function to determine the shard id for an incoming message, only messages that passed the `extractEntityId` will be used
/// </param>
/// <param name="allocationStrategy">Possibility to use a custom shard allocation and rebalancing logic</param>
/// <param name="handOffStopMessage">
/// The message that will be sent to entities when they are to be stopped for a rebalance or
/// graceful shutdown of a <see cref="Sharding.ShardRegion"/>, e.g. <see cref="PoisonPill"/>.
/// </param>
/// <exception cref="IllegalStateException">
/// This exception is thrown when the cluster member doesn't have the role specified in <paramref name="settings"/>.
/// </exception>
/// <returns>The actor ref of the <see cref="Sharding.ShardRegion"/> that is to be responsible for the shard.</returns>
public IActorRef Start(
string typeName,
Props entityProps,
ClusterShardingSettings settings,
ExtractEntityId extractEntityId,
ExtractShardId extractShardId,
IShardAllocationStrategy allocationStrategy,
object handOffStopMessage)
{
RequireClusterRole(settings.Role);
var timeout = _system.Settings.CreationTimeout;
var startMsg = new ClusterShardingGuardian.Start(typeName, entityProps, settings, extractEntityId, extractShardId, allocationStrategy, handOffStopMessage);
var started = _guardian.Value.Ask <ClusterShardingGuardian.Started>(startMsg, timeout).Result;
var shardRegion = started.ShardRegion;
_regions.TryAdd(typeName, shardRegion);
return(shardRegion);
}
/// <summary>
/// TBD
/// </summary>
/// <param name="typeName">TBD</param>
/// <param name="entityProps">TBD</param>
/// <param name="settings">TBD</param>
/// <param name="extractEntityId">TBD</param>
/// <param name="extractShardId">TBD</param>
/// <param name="allocationStrategy">TBD</param>
/// <param name="handOffStopMessage">TBD</param>
/// <exception cref="ArgumentNullException">
/// This exception is thrown when the specified <paramref name="typeName"/> or <paramref name="entityProps"/> is undefined.
/// </exception>
public Start(string typeName, Func <string, Props> entityProps, ClusterShardingSettings settings,
ExtractEntityId extractEntityId, ExtractShardId extractShardId, IShardAllocationStrategy allocationStrategy, object handOffStopMessage)
{
if (string.IsNullOrEmpty(typeName))
{
throw new ArgumentNullException(nameof(typeName), "ClusterSharding start requires type name to be provided");
}
if (entityProps == null)
{
throw new ArgumentNullException(nameof(entityProps), $"ClusterSharding start requires Props for [{typeName}] to be provided");
}
TypeName = typeName;
EntityProps = entityProps;
Settings = settings;
ExtractEntityId = extractEntityId;
ExtractShardId = extractShardId;
AllocationStrategy = allocationStrategy;
HandOffStopMessage = handOffStopMessage;
}
/// <summary>
/// TBD
/// </summary>
/// <param name="typeName">TBD</param>
/// <param name="entityProps">TBD</param>
/// <param name="settings">TBD</param>
/// <param name="idIdExtractor">TBD</param>
/// <param name="shardResolver">TBD</param>
/// <param name="allocationStrategy">TBD</param>
/// <param name="handOffStopMessage">TBD</param>
/// <exception cref="ArgumentNullException">
/// This exception is thrown when the specified <paramref name="typeName"/> or <paramref name="entityProps"/> is undefined.
/// </exception>
public Start(string typeName, Props entityProps, ClusterShardingSettings settings,
IdExtractor idIdExtractor, ShardResolver shardResolver, IShardAllocationStrategy allocationStrategy, object handOffStopMessage)
{
if (string.IsNullOrEmpty(typeName))
{
throw new ArgumentNullException(nameof(typeName), "ClusterSharding start requires type name to be provided");
}
if (entityProps == null)
{
throw new ArgumentNullException(nameof(entityProps), $"ClusterSharding start requires Props for [{typeName}] to be provided");
}
TypeName = typeName;
EntityProps = entityProps;
Settings = settings;
IdExtractor = idIdExtractor;
ShardResolver = shardResolver;
AllocationStrategy = allocationStrategy;
HandOffStopMessage = handOffStopMessage;
}
/// <summary>
/// Register a named entity type by defining the <see cref="Actor.Props"/> of the entity actor and
/// functions to extract entity and shard identifier from messages. The <see cref="Sharding.ShardRegion"/>
/// actor for this type can later be retrieved with the <see cref="ShardRegion"/> method.
/// </summary>
/// <param name="typeName">The name of the entity type</param>
/// <param name="entityProps">
/// The <see cref="Actor.Props"/> of the entity actors that will be created by the <see cref="Sharding.ShardRegion"/>
/// </param>
/// <param name="settings">Configuration settings, see <see cref="ClusterShardingSettings"/></param>
/// <param name="idExtractor">
/// Partial function to extract the entity id and the message to send to the entity from the incoming message,
/// if the partial function does not match the message will be `unhandled`,
/// i.e.posted as `Unhandled` messages on the event stream
/// </param>
/// <param name="shardResolver">
/// Function to determine the shard id for an incoming message, only messages that passed the `extractEntityId` will be used
/// </param>
/// <param name="allocationStrategy">Possibility to use a custom shard allocation and rebalancing logic</param>
/// <param name="handOffStopMessage">
/// The message that will be sent to entities when they are to be stopped for a rebalance or
/// graceful shutdown of a <see cref="Sharding.ShardRegion"/>, e.g. <see cref="PoisonPill"/>.
/// </param>
/// <returns>The actor ref of the <see cref="Sharding.ShardRegion"/> that is to be responsible for the shard.</returns>
public IActorRef Start(
string typeName, //TODO: change type name to type instance?
Props entityProps,
ClusterShardingSettings settings,
IdExtractor idExtractor,
ShardResolver shardResolver,
IShardAllocationStrategy allocationStrategy,
object handOffStopMessage)
{
RequireClusterRole(settings.Role);
var timeout = _system.Settings.CreationTimeout;
var startMsg = new ClusterShardingGuardian.Start(typeName, entityProps, settings, idExtractor, shardResolver, allocationStrategy, handOffStopMessage);
var started = _guardian.Value.Ask <ClusterShardingGuardian.Started>(startMsg, timeout).Result;
var shardRegion = started.ShardRegion;
_regions.TryAdd(typeName, shardRegion);
return(shardRegion);
}
public DDataShardCoordinator(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy, IActorRef replicator, int majorityMinCap, bool rememberEntities)
{
_replicator = replicator;
_rememberEntities = rememberEntities;
Settings = settings;
AllocationStrategy = allocationStrategy;
Log = Context.GetLogger();
Cluster = Cluster.Get(Context.System);
CurrentState = PersistentShardCoordinator.State.Empty.WithRememberEntities(settings.RememberEntities);
RemovalMargin = Cluster.DowningProvider.DownRemovalMargin;
MinMembers = string.IsNullOrEmpty(settings.Role)
? Cluster.Settings.MinNrOfMembers
: (Cluster.Settings.MinNrOfMembersOfRole.TryGetValue(settings.Role, out var min) ? min : Cluster.Settings.MinNrOfMembers);
RebalanceTask = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(Settings.TunningParameters.RebalanceInterval, Settings.TunningParameters.RebalanceInterval, Self, RebalanceTick.Instance, Self);
_readConsistency = new ReadMajority(settings.TunningParameters.WaitingForStateTimeout, majorityMinCap);
_writeConsistency = new WriteMajority(settings.TunningParameters.UpdatingStateTimeout, majorityMinCap);
_coordinatorStateKey = new LWWRegisterKey <PersistentShardCoordinator.State>(typeName + "CoordinatorState");
_allShardsKey = new GSetKey <string>($"shard-{typeName}-all");
_allKeys = rememberEntities
? ImmutableHashSet.CreateRange(new IKey <IReplicatedData>[] { _coordinatorStateKey, _allShardsKey })
: ImmutableHashSet.Create <IKey <IReplicatedData> >(_coordinatorStateKey);
if (rememberEntities)
{
replicator.Tell(Dsl.Subscribe(_allShardsKey, Self));
}
Cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsEvents, typeof(ClusterEvent.ClusterShuttingDown));
// get state from ddata replicator, repeat until GetSuccess
GetCoordinatorState();
GetAllShards();
Context.Become(WaitingForState(_allKeys));
}
internal static Props Props(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy, IActorRef replicator, int majorityMinCap, bool rememberEntities) => Actor.Props.Create(() => new DDataShardCoordinator(typeName, settings, allocationStrategy, replicator, majorityMinCap, rememberEntities)).WithDeploy(Deploy.Local);
/// <summary>
/// Factory method for the <see cref="Actor.Props"/> of the <see cref="PersistentShardCoordinator"/> actor.
/// </summary>
internal static Props Props(string typeName, ClusterShardingSettings settings, IShardAllocationStrategy allocationStrategy)
{
return(Actor.Props.Create(() => new PersistentShardCoordinator(typeName, settings, allocationStrategy)).WithDeploy(Deploy.Local));
}
