private void TakeOwnership(TakeOwnership msg, IContext context)
{
//Check again if I'm the owner
var address = MemberList.GetPartition(msg.Name, _kind);
if (!string.IsNullOrEmpty(address) && address != ProcessRegistry.Instance.Address)
{
//if not, forward to the correct owner
var owner = Partition.PartitionForKind(address, _kind);
owner.Tell(msg);
}
else
{
_logger.LogDebug($"Kind {_kind} Take Ownership name: {msg.Name}, pid: {msg.Pid}");
_partition[msg.Name] = msg.Pid;
_reversePartition[msg.Pid] = msg.Name;
context.Watch(msg.Pid);
}
}
public static void StartWithConfig(ClusterConfig config)
{
Config = config;
Remote.Remote.Start(Config.Address, Config.Port, Config.RemoteConfig);
Serialization.RegisterFileDescriptor(ProtosReflection.Descriptor);
Logger.LogInformation("Starting Proto.Actor cluster");
var(host, port) = ParseAddress(ProcessRegistry.Instance.Address);
var kinds = Remote.Remote.GetKnownKinds();
Partition.Setup(kinds);
PidCache.Setup();
MemberList.Setup();
Config.ClusterProvider.RegisterMemberAsync(Config.Name, host, port, kinds, config.InitialMemberStatusValue, config.MemberStatusValueSerializer).Wait();
Config.ClusterProvider.MonitorMemberStatusChanges();
Logger.LogInformation("Started Cluster");
}
private async Task Spawning(ActorPidRequest req, string activator, int retryLeft, SpawningProcess spawning)
{
if (string.IsNullOrEmpty(activator))
{
activator = MemberList.GetActivator(req.Kind);
if (string.IsNullOrEmpty(activator))
{
//No activator currently available, return unavailable
_logger.LogDebug("No activator currently available");
spawning.TrySetResult(ActorPidResponse.Unavailable);
return;
}
}
ActorPidResponse pidResp;
try
{
pidResp = await Remote.Remote.SpawnNamedAsync(activator, req.Name, req.Kind, Cluster.Config.TimeoutTimespan)
.ConfigureAwait(false);
}
catch (TimeoutException)
{
spawning.TrySetResult(ActorPidResponse.TimeOut);
return;
}
catch
{
spawning.TrySetResult(ActorPidResponse.Err);
return;
}
if ((ResponseStatusCode)pidResp.StatusCode == ResponseStatusCode.Unavailable && retryLeft != 0)
{
await Spawning(req, null, --retryLeft, spawning).ConfigureAwait(false);
return;
}
spawning.TrySetResult(pidResp);
}
public static void StartWithConfig(ClusterConfig config)
{
cfg = config;
Remote.Remote.Start(cfg.Address, cfg.Port, cfg.RemoteConfig);
Serialization.RegisterFileDescriptor(ProtosReflection.Descriptor);
Logger.LogInformation("Starting Proto.Actor cluster");
var(h, p) = ParseAddress(ProcessRegistry.Instance.Address);
var kinds = Remote.Remote.GetKnownKinds();
Partition.SpawnPartitionActors(kinds);
Partition.SubscribeToEventStream();
PidCache.Spawn();
PidCache.SubscribeToEventStream();
MemberList.SubscribeToEventStream();
cfg.ClusterProvider.RegisterMemberAsync(cfg.Name, h, p, kinds, config.InitialMemberStatusValue, config.MemberStatusValueSerializer).Wait();
cfg.ClusterProvider.MonitorMemberStatusChanges();
Logger.LogInformation("Started Cluster");
}
public static void Start(string clusterName, string address, int port, IClusterProvider provider)
{
Remote.Remote.Start(address, port);
cp = provider;
Serialization.RegisterFileDescriptor(ProtosReflection.Descriptor);
Logger.LogInformation("Starting Proto.Actor cluster");
var(h, p) = ParseAddress(ProcessRegistry.Instance.Address);
var kinds = Remote.Remote.GetKnownKinds();
Partition.SpawnPartitionActors(kinds);
Partition.SubscribeToEventStream();
PidCache.Spawn();
PidCache.SubscribeToEventStream();
MemberList.Spawn();
MemberList.SubscribeToEventStream();
cp.RegisterMemberAsync(clusterName, h, p, kinds).Wait();
cp.MonitorMemberStatusChanges();
Logger.LogInformation("Started Cluster");
}
private void MemberLeft(MemberLeftEvent msg, IContext context)
{
_logger.LogInformation($"Kind {_kind} Member Left {msg.Address}");
//If the left member is self, transfer remaining pids to others
if (msg.Address == ProcessRegistry.Instance.Address)
{
//TODO: right now we transfer ownership on a per actor basis.
//this could be done in a batch
//ownership is also racy, new nodes should maybe forward requests to neighbours (?)
foreach (var(actorId, _) in _partition.ToArray())
{
var address = MemberList.GetPartition(actorId, _kind);
if (!string.IsNullOrEmpty(address))
{
TransferOwnership(actorId, address, context);
}
}
}
foreach (var(actorId, pid) in _partition.ToArray())
{
if (pid.Address == msg.Address)
{
_partition.Remove(actorId);
_reversePartition.Remove(pid);
}
}
//Process Spawning Process
foreach (var(_, sp) in _spawningProcs)
{
if (sp.SpawningAddress == msg.Address)
{
sp.TrySetResult(ActorPidResponse.Unavailable);
}
}
}
private void GetPid(IContext context, PidCacheRequest msg)
{
if (_cache.TryGetValue(msg.Name, out var pid))
{
context.Respond(new ActorPidResponse
{
Pid = pid
});
return; //found the pid, replied, exit
}
var name = msg.Name;
var kind = msg.Kind;
context.ReenterAfter(MemberList.GetMemberAsync(name, kind), address =>
{
var remotePid = Partition.PartitionForKind(address.Result, kind);
var req = new ActorPidRequest
{
Kind = kind,
Name = name
};
var resp = remotePid.RequestAsync <ActorPidResponse>(req);
context.ReenterAfter(resp, t =>
{
var res = t.Result;
var respid = res.Pid;
var key = respid.ToShortString();
_cache[name] = respid;
_reverseCache[key] = name;
context.Watch(respid);
context.Respond(res);
return(Actor.Done);
});
return(Actor.Done);
});
}
private void Spawn(ActorPidRequest msg, IContext context)
{
//Check if exist in current partition dictionary
if (_partition.TryGetValue(msg.Name, out var pid))
{
context.Respond(new ActorPidResponse {
Pid = pid
});
return;
}
//Check if is spawning, if so just await spawning finish.
SpawningProcess spawning;
if (_spawningProcs.TryGetValue(msg.Name, out spawning))
{
context.ReenterAfter(spawning.Task, rst =>
{
context.Respond(rst.IsFaulted ? ActorPidResponse.Err : rst.Result);
return(Actor.Done);
});
return;
}
//Get activator
var activator = MemberList.GetActivator(msg.Kind);
if (string.IsNullOrEmpty(activator))
{
//No activator currently available, return unavailable
_logger.LogDebug("No members currently available");
context.Respond(ActorPidResponse.Unavailable);
return;
}
//Create SpawningProcess and cache it in spawning dictionary.
spawning = new SpawningProcess(activator);
_spawningProcs[msg.Name] = spawning;
//Await SpawningProcess
context.ReenterAfter(spawning.Task, rst =>
{
_spawningProcs.Remove(msg.Name);
//Check if exist in current partition dictionary
//This is necessary to avoid race condition during partition map transfering.
if (_partition.TryGetValue(msg.Name, out pid))
{
context.Respond(new ActorPidResponse {
Pid = pid
});
return(Actor.Done);
}
//Check if process is faulted
if (rst.IsFaulted)
{
context.Respond(ActorPidResponse.Err);
return(Actor.Done);
}
var pidResp = rst.Result;
if ((ResponseStatusCode)pidResp.StatusCode == ResponseStatusCode.OK)
{
pid = pidResp.Pid;
_partition[msg.Name] = pid;
_reversePartition[pid] = msg.Name;
context.Watch(pid);
}
context.Respond(pidResp);
return(Actor.Done);
});
//Perform Spawning
Task.Factory.StartNew(() => Spawning(msg, activator, 3, spawning));
}
private async Task Spawn(ActorPidRequest msg, IContext context)
{
if (_partition.TryGetValue(msg.Name, out var pid))
{
context.Respond(new ActorPidResponse {
Pid = pid
});
return;
}
var activator = MemberList.GetActivator(msg.Kind);
if (string.IsNullOrEmpty(activator))
{
//No activator currently available, return unavailable
_logger.LogDebug("No members currently available");
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Unavailable
});
return;
}
for (var retry = 3; retry >= 0; retry--)
{
if (string.IsNullOrEmpty(activator))
{
activator = MemberList.GetActivator(msg.Kind);
if (string.IsNullOrEmpty(activator))
{
//No activator currently available, return unavailable
_logger.LogDebug("No activator currently available");
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Unavailable
});
return;
}
}
ActorPidResponse pidResp;
try
{
pidResp = await Remote.Remote.SpawnNamedAsync(activator, msg.Name, msg.Kind, Cluster.cfg.TimeoutTimespan);
}
catch (TimeoutException)
{
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Timeout
});
throw;
}
catch
{
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Error
});
throw;
}
switch ((ResponseStatusCode)pidResp.StatusCode)
{
case ResponseStatusCode.OK:
pid = pidResp.Pid;
_partition[msg.Name] = pid;
_reversePartition[pid] = msg.Name;
context.Watch(pid);
context.Respond(pidResp);
return;
case ResponseStatusCode.Unavailable:
//Get next activator to spawn
if (retry != 0)
{
activator = null;
continue;
}
context.Respond(pidResp);
return;
default:
//Return to requester to wait
context.Respond(pidResp);
return;
}
}
}
private async Task Spawn(ActorPidRequest msg, IContext context)
{
if (_partition.TryGetValue(msg.Name, out var pid))
{
context.Respond(new ActorPidResponse {
Pid = pid
});
return;
}
var members = await MemberList.GetMembersAsync(msg.Kind);
if (members == null || members.Length == 0)
{
//No members currently available, return unavailable
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Unavailable
});
return;
}
var retrys = members.Length - 1;
for (int retry = retrys; retry >= 0; retry--)
{
members = members ?? await MemberList.GetMembersAsync(msg.Kind);
if (members == null || members.Length == 0)
{
//No members currently available, return unavailable
context.Respond(new ActorPidResponse {
StatusCode = (int)ResponseStatusCode.Unavailable
});
return;
}
var activator = members[_counter.Next() % members.Length];
members = null;
var pidResp = await Remote.Remote.SpawnNamedAsync(activator, msg.Name, msg.Kind, TimeSpan.FromSeconds(5));
switch ((ResponseStatusCode)pidResp.StatusCode)
{
case ResponseStatusCode.OK:
pid = pidResp.Pid;
_partition[msg.Name] = pid;
context.Watch(pid);
context.Respond(pidResp);
return;
case ResponseStatusCode.Unavailable:
//Get next activator to spawn
if (retry != 0)
{
continue;
}
context.Respond(pidResp);
break;
default:
//Return to requester to wait
context.Respond(pidResp);
return;
}
}
}