public virtual AddressAndTag Register(ActivationAddress address, bool singleActivation)
{
if (singleActivation)
{
var result = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return result;
}
else
{
var tag = DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
return new AddressAndTag() { Address = address, VersionTag = tag };
}
}
public AddressAndTag Register(ActivationAddress address, bool singleActivation)
{
if (singleActivation)
{
var result = directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.ClusterLocal);
return result;
}
else
{
var tag = directoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
return new AddressAndTag() { Address = address, VersionTag = tag };
}
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <param name="retries"></param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task<Tuple<ActivationAddress, int>> RegisterSingleActivation(ActivationAddress address, int retries)
{
router.RegistrationsSingleActRemoteReceived.Increment();
if (logger.IsVerbose2) logger.Verbose2("Trying to register activation for grain. GrainId: {0}. ActivationId: {1}.", address.Grain, address.Activation);
// validate that this grain should be stored in our partition
SiloAddress owner = router.CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (router.MyAddress.Equals(owner))
{
router.RegistrationsSingleActLocal.Increment();
return partition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
}
if (retries <= 0)
throw new OrleansException("Silo " + router.MyAddress + " is not the owner of the grain " +
address.Grain + " Owner=" + owner);
if (logger.IsVerbose2) logger.Verbose2("Retry {0} RemoteGrainDirectory.RegisterSingleActivation for address={1} at Owner={2}", retries, address, owner);
PrepareForRetry(retries);
await Task.Delay(RETRY_DELAY);
SiloAddress o = router.CalculateTargetSilo(address.Grain);
if (o == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (o.Equals(router.MyAddress))
{
router.RegistrationsSingleActLocal.Increment();
return partition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
}
router.RegistrationsSingleActRemoteSent.Increment();
return await GetDirectoryReference(o).RegisterSingleActivation(address, retries - 1);
}
public void Unregister(ActivationAddress address, UnregistrationCause cause)
{
throw new InvalidOperationException();
}
public void InvalidateCacheEntry(ActivationAddress activationAddress)
{
int version;
IReadOnlyList<Tuple<SiloAddress, ActivationId>> list;
var grainId = activationAddress.Grain;
var activationId = activationAddress.Activation;
// look up grain activations
if (DirectoryCache.LookUp(grainId, out list, out version))
{
RemoveActivations(DirectoryCache, grainId, list, version, t => t.Item2.Equals(activationId));
}
}
public void CacheOrUpdateRemoteClusterRegistration(GrainId grain, ActivationId oldActivation, ActivationAddress otherClusterAddress)
{
lock (lockable)
{
if (partitionData.ContainsKey(grain))
{
partitionData[grain].CacheOrUpdateRemoteClusterRegistration(grain, oldActivation,
otherClusterAddress.Activation, otherClusterAddress.Silo);
}
else
{
AddSingleActivation(grain, otherClusterAddress.Activation, otherClusterAddress.Silo,
GrainDirectoryEntryStatus.Cached);
}
}
}
public Task <AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation)
{
throw new InvalidOperationException();
}
public virtual Task UnregisterAsync(ActivationAddress address, bool force)
{
throw new InvalidOperationException();
}
public Task UnregisterConditionallyAsync(ActivationAddress addr)
{
// This is a no-op if the lazy registration delay is zero or negative
return Silo.CurrentSilo.OrleansConfig.Globals.DirectoryLazyDeregistrationDelay <= TimeSpan.Zero ?
TaskDone.Done : UnregisterAsyncImpl(addr, false);
}
public async Task<AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation, int hopCount)
{
(singleActivation ? router.RegistrationsSingleActRemoteReceived : router.RegistrationsRemoteReceived).Increment();
return await router.RegisterAsync(address, singleActivation, hopCount);
}
public void Unregister(ActivationAddress address, UnregistrationCause cause)
{
throw new InvalidOperationException();
}
public async Task <List <ActivationAddress> > FullLookup(GrainId grain)
{
fullLookups.Increment();
SiloAddress silo = CalculateTargetSilo(grain, false);
// No need to check that silo != null since we're passing excludeThisSiloIfStopping = false
if (log.IsVerbose)
{
log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
// We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()).
// We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called.
if (silo.Equals(MyAddress))
{
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpGrain(grain);
if (localResult == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}=none", grain);
}
return(new List <ActivationAddress>());
}
var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings());
}
LocalDirectorySuccesses.Increment();
return(a);
}
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(silo))
{
throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo));
}
RemoteLookupsSent.Increment();
Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES);
// update the cache
List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList();
List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings());
}
if (entries.Count > 0)
{
DirectoryCache.AddOrUpdate(grain, entries, result.Item2);
}
return(addresses);
}
// used for testing to (carefully!) allow two clients in the same process
private async Task StartInternal(Func <Exception, Task <bool> > retryFilter)
{
// Initialize the gateway list provider, since information from the cluster is required to successfully
// initialize subsequent services.
var initializedGatewayProvider = new[] { false };
await ExecuteWithRetries(async() =>
{
if (!initializedGatewayProvider[0])
{
await this.gatewayListProvider.InitializeGatewayListProvider();
initializedGatewayProvider[0] = true;
}
var gateways = await this.gatewayListProvider.GetGateways();
if (gateways.Count == 0)
{
var gatewayProviderType = this.gatewayListProvider.GetType().GetParseableName();
var err = $"Could not find any gateway in {gatewayProviderType}. Orleans client cannot initialize.";
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new SiloUnavailableException(err);
}
},
retryFilter);
var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative
transport = ActivatorUtilities.CreateInstance <ClientMessageCenter>(this.ServiceProvider, localAddress, generation, clientId);
transport.RegisterLocalMessageHandler(this.HandleMessage);
transport.Start();
CurrentActivationAddress = ActivationAddress.NewActivationAddress(transport.MyAddress, clientId);
await ExecuteWithRetries(
async() => this.GrainTypeResolver = await transport.GetGrainTypeResolver(this.InternalGrainFactory),
retryFilter);
this.typeMapRefreshTimer = new AsyncTaskSafeTimer(
this.logger,
RefreshGrainTypeResolver,
null,
this.typeMapRefreshInterval,
this.typeMapRefreshInterval);
ClientStatistics.Start(transport, clientId);
await ExecuteWithRetries(StreamingInitialize, retryFilter);
async Task ExecuteWithRetries(Func <Task> task, Func <Exception, Task <bool> > shouldRetry)
{
while (true)
{
try
{
await task();
return;
}
catch (Exception exception) when(shouldRetry != null)
{
var retry = await shouldRetry(exception);
if (!retry)
{
throw;
}
}
}
}
}
public Task UnregisterConditionallyAsync(ActivationAddress addr)
{
// This is a no-op if the lazy registration delay is zero or negative
return(Silo.CurrentSilo.OrleansConfig.Globals.DirectoryLazyDeregistrationDelay <= TimeSpan.Zero ?
TaskDone.Done : UnregisterAsyncImpl(addr, false));
}
public Task UnregisterAsync(ActivationAddress addr)
{
return(UnregisterAsyncImpl(addr, true));
}
public void InvalidateCache(ActivationAddress address)
{
// no need to do anything since the directory does not cache activations in remote clusters
}
private void ProtocolError(ActivationAddress address, string msg)
{
logger.Error((int)ErrorCode.GlobalSingleInstance_ProtocolError, string.Format("GSIP:Req {0} PROTOCOL ERROR {1}", address.Grain.ToString(), msg));
}
public Task UnregisterAsync(ActivationAddress address, UnregistrationCause cause, int hopCount)
{
return router.UnregisterAsync(address, cause, hopCount);
}
public Task<GlobalSingleInstanceResponseOutcome> SendRequestRound(ActivationAddress address, List<string> remoteClusters)
{
// array that holds the responses
var responses = new Task<RemoteClusterActivationResponse>[remoteClusters.Count];
// send all requests
for (int i = 0; i < responses.Length; i++)
responses[i] = SendRequest(address.Grain, remoteClusters[i]);
// response processor
return GlobalSingleInstanceResponseTracker.GetOutcomeAsync(responses, address.Grain, logger);
}
/// <summary>
/// Looks up the cached value by the given key.
/// </summary>
/// <param name="cache">grain directory cache to look up results from</param>
/// <param name="key">key for the lookup</param>
/// <param name="result">value if the key is found, undefined otherwise</param>
/// <returns>true if the given key is in the cache</returns>
public static bool LookUp(this IGrainDirectoryCache cache, GrainId key, out ActivationAddress result)
{
return(cache.LookUp(key, out result, out _));
}
public Task <ActivationAddress> Register(ActivationAddress address) => throw new InvalidOperationException($"Cannot register client grain explicitly");
public bool RemoveActivation(ActivationAddress addr)
{
return(RemoveActivation(addr.Activation, true));
}
public async Task RegisterAsync(ActivationAddress address)
{
registrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsLocal.Increment();
// if I am the owner, store the new activation locally
DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
}
else
{
RegistrationsRemoteSent.Increment();
// otherwise, notify the owner
int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES);
if (IsValidSilo(address.Silo))
{
// Caching optimization:
// cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
List<Tuple<SiloAddress, ActivationId>> cached;
if (!DirectoryCache.LookUp(address.Grain, out cached))
{
cached = new List<Tuple<SiloAddress, ActivationId>>(1);
}
cached.Add(Tuple.Create(address.Silo, address.Activation));
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, eTag);
}
}
}
public Task<AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation)
{
throw new InvalidOperationException();
}
public void InvalidateCacheEntry(ActivationAddress activationAddress)
{
var grainId = activationAddress.Grain;
var activationId = activationAddress.Activation;
List<Tuple<SiloAddress, ActivationId>> list;
if (!DirectoryCache.LookUp(grainId, out list)) return;
list.RemoveAll(tuple => tuple.Item2.Equals(activationId));
// if list empty, need to remove from cache
if (list.Count == 0)
DirectoryCache.Remove(grainId);
}
private void ProtocolError(ActivationAddress address, string msg)
{
logger.Error((int)ErrorCode.GlobalSingleInstance_ProtocolError, string.Format("GSIP:Req {0} PROTOCOL ERROR {1}", address.Grain.ToString(), msg));
}
public void CacheOrUpdateRemoteClusterRegistration(GrainId grain, ActivationId oldActivation, ActivationAddress otherClusterAddress)
{
lock (lockable)
{
if (partitionData.TryGetValue(grain, out var graininfo))
{
graininfo.CacheOrUpdateRemoteClusterRegistration(grain, oldActivation,
otherClusterAddress.Activation, otherClusterAddress.Silo);
}
else
{
AddSingleActivation(grain, otherClusterAddress.Activation, otherClusterAddress.Silo,
GrainDirectoryEntryStatus.Cached);
}
}
}
public Task Unregister(ActivationAddress address, UnregistrationCause cause) => throw new InvalidOperationException($"Cannot unregister client grain explicitly");
private async Task <List <SiloHandle> > getSilosToFail(Fail fail, int numOfFailures)
{
List <SiloHandle> failures = new List <SiloHandle>();
int count = 0, index = 0;
// Figure out the primary directory partition and the silo hosting the ReminderTableGrain.
bool usingReminderGrain = this.ClusterConfiguration.Globals.ReminderServiceType.Equals(GlobalConfiguration.ReminderServiceProviderType.ReminderTableGrain);
IReminderTable tableGrain = this.GrainFactory.GetGrain <IReminderTableGrain>(Constants.ReminderTableGrainId);
var tableGrainId = ((GrainReference)tableGrain).GrainId;
SiloAddress reminderTableGrainPrimaryDirectoryAddress = (await TestUtils.GetDetailedGrainReport(this.HostedCluster.InternalGrainFactory, tableGrainId, this.HostedCluster.Primary)).PrimaryForGrain;
// ask a detailed report from the directory partition owner, and get the actionvation addresses
var addresses = (await TestUtils.GetDetailedGrainReport(this.HostedCluster.InternalGrainFactory, tableGrainId, this.HostedCluster.GetSiloForAddress(reminderTableGrainPrimaryDirectoryAddress))).LocalDirectoryActivationAddresses;
ActivationAddress reminderGrainActivation = addresses.FirstOrDefault();
SortedList <int, SiloHandle> ids = new SortedList <int, SiloHandle>();
foreach (var siloHandle in this.HostedCluster.GetActiveSilos())
{
SiloAddress siloAddress = siloHandle.SiloAddress;
if (siloAddress.Equals(this.HostedCluster.Primary.SiloAddress))
{
continue;
}
// Don't fail primary directory partition and the silo hosting the ReminderTableGrain.
if (usingReminderGrain)
{
if (siloAddress.Equals(reminderTableGrainPrimaryDirectoryAddress) || siloAddress.Equals(reminderGrainActivation.Silo))
{
continue;
}
}
ids.Add(siloHandle.SiloAddress.GetConsistentHashCode(), siloHandle);
}
// we should not fail the primary!
// we can't guarantee semantics of 'Fail' if it evalutes to the primary's address
switch (fail)
{
case Fail.First:
index = 0;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index++;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Last:
index = ids.Count - 1;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index--;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Random:
default:
while (count++ < numOfFailures)
{
SiloHandle r = ids.Values[random.Next(ids.Count)];
while (failures.Contains(r))
{
r = ids.Values[random.Next(ids.Count)];
}
failures.Add(r);
}
break;
}
return(failures);
}
// used for testing to (carefully!) allow two clients in the same process
private async Task StartInternal(Func <Exception, Task <bool> > retryFilter)
{
// Initialize the gateway list provider, since information from the cluster is required to successfully
// initialize subsequent services.
var initializedGatewayProvider = new[] { false };
await ExecuteWithRetries(async() =>
{
if (!initializedGatewayProvider[0])
{
await this.gatewayListProvider.InitializeGatewayListProvider();
initializedGatewayProvider[0] = true;
}
var gateways = await this.gatewayListProvider.GetGateways();
if (gateways.Count == 0)
{
var gatewayProviderType = this.gatewayListProvider.GetType().GetParseableName();
var err = $"Could not find any gateway in {gatewayProviderType}. Orleans client cannot initialize.";
logger.Error(ErrorCode.GatewayManager_NoGateways, err);
throw new SiloUnavailableException(err);
}
},
retryFilter);
var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative
transport = ActivatorUtilities.CreateInstance <ClientMessageCenter>(this.ServiceProvider, localAddress, generation, handshakeClientId);
transport.Start();
CurrentActivationAddress = ActivationAddress.NewActivationAddress(transport.MyAddress, handshakeClientId);
listeningCts = new CancellationTokenSource();
var ct = listeningCts.Token;
listenForMessages = true;
// Keeping this thread handling it very simple for now. Just queue task on thread pool.
Task.Run(
() =>
{
while (listenForMessages && !ct.IsCancellationRequested)
{
try
{
RunClientMessagePump(ct);
}
catch (Exception exc)
{
logger.Error(ErrorCode.Runtime_Error_100326, "RunClientMessagePump has thrown exception", exc);
}
}
},
ct).Ignore();
await ExecuteWithRetries(
async() => this.GrainTypeResolver = await transport.GetGrainTypeResolver(this.InternalGrainFactory),
retryFilter);
this.typeMapRefreshTimer = new AsyncTaskSafeTimer(
this.logger,
RefreshGrainTypeResolver,
null,
this.typeMapRefreshInterval,
this.typeMapRefreshInterval);
ClientStatistics.Start(transport, clientId);
await ExecuteWithRetries(StreamingInitialize, retryFilter);
async Task ExecuteWithRetries(Func <Task> task, Func <Exception, Task <bool> > shouldRetry)
{
while (true)
{
try
{
await task();
return;
}
catch (Exception exception) when(shouldRetry != null)
{
var retry = await shouldRetry(exception);
if (!retry)
{
throw;
}
}
}
}
}
public AddressAndTag Register(ActivationAddress address, bool singleActivation)
{
throw new InvalidOperationException();
}
public void CacheOrUpdateRemoteClusterRegistration(GrainId grain, ActivationId oldActivation, ActivationAddress otherClusterAddress)
{
lock (lockable)
{
if (partitionData.ContainsKey(grain))
{
partitionData[grain].CacheOrUpdateRemoteClusterRegistration(grain, oldActivation,
otherClusterAddress.Activation, otherClusterAddress.Silo);
}
else
{
AddSingleActivation(grain, otherClusterAddress.Activation, otherClusterAddress.Silo,
GrainDirectoryEntryStatus.Cached);
}
}
}
public async Task <AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation)
{
if (!singleActivation)
{
throw new OrleansException("global single instance protocol is incompatible with using multiple activations");
}
if (!this.hasMultiClusterNetwork)
{
// no multicluster network. Go to owned state directly.
return(directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.Owned));
}
// examine the multicluster configuration
var config = this.multiClusterOracle.GetMultiClusterConfiguration();
if (config == null || !config.Clusters.Contains(this.clusterId))
{
// we are not joined to the cluster yet/anymore. Go to doubtful state directly.
gsiActivationMaintainer.TrackDoubtfulGrain(address.Grain);
return(directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.Doubtful));
}
var remoteClusters = config.Clusters.Where(id => id != this.clusterId).ToList();
// Try to go into REQUESTED_OWNERSHIP state
var myActivation = directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.RequestedOwnership);
if (!myActivation.Address.Equals(address))
{
//This implies that the registration already existed in some state? return the existing activation.
return(myActivation);
}
// Do request rounds until successful or we run out of retries
int retries = numRetries;
while (retries-- > 0)
{
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("GSIP:Req {0} Round={1} Act={2}", address.Grain.ToString(), numRetries - retries, myActivation.Address.ToString());
}
var outcome = await SendRequestRound(address, remoteClusters);
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("GSIP:End {0} Round={1} Outcome={2}", address.Grain.ToString(), numRetries - retries, outcome);
}
switch (outcome.State)
{
case OutcomeState.RemoteOwner:
case OutcomeState.RemoteOwnerLikely:
{
directoryPartition.CacheOrUpdateRemoteClusterRegistration(address.Grain, address.Activation, outcome.RemoteOwnerAddress.Address);
return(outcome.RemoteOwnerAddress);
}
case OutcomeState.Succeed:
{
if (directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Owned, GrainDirectoryEntryStatus.RequestedOwnership))
{
return(myActivation);
}
else
{
break; // concurrently moved to RACE_LOSER
}
}
case OutcomeState.Inconclusive:
{
break;
}
}
// we were not successful, reread state to determine what is going on
int version;
var mcstatus = directoryPartition.TryGetActivation(address.Grain, out address, out version);
if (mcstatus == GrainDirectoryEntryStatus.RequestedOwnership)
{
// we failed because of inconclusive answers. Stay in this state for retry.
}
else if (mcstatus == GrainDirectoryEntryStatus.RaceLoser)
{
// we failed because an external request moved us to RACE_LOSER. Go back to REQUESTED_OWNERSHIP for retry
var success = directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.RequestedOwnership, GrainDirectoryEntryStatus.RaceLoser);
if (!success)
{
ProtocolError(address, "unable to transition from RACE_LOSER to REQUESTED_OWNERSHIP");
}
// do not wait before retrying because there is a dominant remote request active so we can probably complete quickly
}
else
{
ProtocolError(address, "unhandled protocol state");
}
}
// we are done with the quick retries. Now we go into doubtful state, which means slower retries.
var ok = directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RequestedOwnership);
if (!ok)
{
ProtocolError(address, "unable to transition into doubtful");
}
this.gsiActivationMaintainer.TrackDoubtfulGrain(address.Grain);
return(myActivation);
}
internal List <ActivationAddress> ToListOfActivations(bool singleActivation)
{
var result = new List <ActivationAddress>();
lock (lockable)
{
foreach (var pair in partitionData)
{
var grain = pair.Key;
if (pair.Value.SingleInstance == singleActivation)
{
result.AddRange(pair.Value.Instances.Select(activationPair => ActivationAddress.GetAddress(activationPair.Value.SiloAddress, grain, activationPair.Key))
.Where(addr => IsValidSilo(addr.Silo)));
}
}
}
return(result);
}
public async Task Unregister(ActivationAddress address, bool force, int retries)
{
router.UnregistrationsRemoteReceived.Increment();
// validate that this grain should be stored in our partition
SiloAddress owner = router.CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(router.MyAddress))
{
router.UnregistrationsLocal.Increment();
partition.RemoveActivation(address.Grain, address.Activation, force);
return;
}
if (retries > 0)
{
if (logger.IsVerbose2) logger.Verbose2("Retry {0} RemoteGrainDirectory.Unregister for address={1} at Owner={2}", retries, address, owner);
PrepareForRetry(retries);
await Task.Delay(RETRY_DELAY);
SiloAddress o = router.CalculateTargetSilo(address.Grain);
if (o == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (o.Equals(router.MyAddress))
{
router.UnregistrationsLocal.Increment();
partition.RemoveActivation(address.Grain, address.Activation, force);
return;
}
router.UnregistrationsRemoteSent.Increment();
await GetDirectoryReference(o).Unregister(address, force, retries - 1);
}
else
{
throw new OrleansException("Silo " + router.MyAddress + " is not the owner of the grain " + address.Grain + " Owner=" + owner);
}
}
public void Unregister(ActivationAddress address, UnregistrationCause cause)
{
directoryPartition.RemoveActivation(address.Grain, address.Activation, cause);
}
public void InvalidateCache(ActivationAddress address)
{
IActivationInfo existingAct;
bool wasRemoved;
directoryPartition.RemoveActivation(address.Grain, address.Activation, UnregistrationCause.CacheInvalidation, out existingAct, out wasRemoved);
if (!wasRemoved)
{
logger.Verbose2("GSIP:Inv {0} ignored", address);
}
else
{
logger.Verbose2("GSIP:Inv {0} removed ({1})", address, existingAct.RegistrationStatus);
}
}
public async Task <ActivationAddress> Register(ActivationAddress address) => (await this.localGrainDirectory.RegisterAsync(address, singleActivation: true)).Address;
public AddressAndTag Register(ActivationAddress address, bool singleActivation)
{
throw new InvalidOperationException();
}
public Task Unregister(ActivationAddress address, UnregistrationCause cause) => this.localGrainDirectory.UnregisterAsync(address, cause);
public async Task<AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation)
{
if (!singleActivation)
throw new OrleansException("global single instance protocol is incompatible with using multiple activations");
var myClusterId = Silo.CurrentSilo.ClusterId;
if (myClusterId == null)
{
// no multicluster network. Go to owned state directly.
return directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.Owned);
}
// examine the multicluster configuration
var config = Silo.CurrentSilo.LocalMultiClusterOracle.GetMultiClusterConfiguration();
if (config == null || !config.Clusters.Contains(myClusterId))
{
// we are not joined to the cluster yet/anymore. Go to doubtful state directly.
gsiActivationMaintainer.TrackDoubtfulGrain(address.Grain);
return directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.Doubtful);
}
var remoteClusters = config.Clusters.Where(id => id != myClusterId).ToList();
// Try to go into REQUESTED_OWNERSHIP state
var myActivation = directoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo, GrainDirectoryEntryStatus.RequestedOwnership);
if (!myActivation.Address.Equals(address))
{
//This implies that the registration already existed in some state? return the existing activation.
return myActivation;
}
// Do request rounds until successful or we run out of retries
int retries = numRetries;
while (retries-- > 0)
{
if (logger.IsVerbose)
logger.Verbose("GSIP:Req {0} Round={1} Act={2}", address.Grain.ToString(), numRetries - retries, myActivation.Address.ToString());
var outcome = await SendRequestRound(address, remoteClusters);
if (logger.IsVerbose)
logger.Verbose("GSIP:End {0} Round={1} Outcome={2}", address.Grain.ToString(), numRetries - retries, outcome);
switch (outcome.State)
{
case OutcomeState.RemoteOwner:
case OutcomeState.RemoteOwnerLikely:
{
directoryPartition.CacheOrUpdateRemoteClusterRegistration(address.Grain, address.Activation, outcome.RemoteOwnerAddress.Address);
return outcome.RemoteOwnerAddress;
}
case OutcomeState.Succeed:
{
if (directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Owned, GrainDirectoryEntryStatus.RequestedOwnership))
return myActivation;
else
break; // concurrently moved to RACE_LOSER
}
case OutcomeState.Inconclusive:
{
break;
}
}
// we were not successful, reread state to determine what is going on
int version;
var mcstatus = directoryPartition.TryGetActivation(address.Grain, out address, out version);
if (mcstatus == GrainDirectoryEntryStatus.RequestedOwnership)
{
// we failed because of inconclusive answers. Stay in this state for retry.
}
else if (mcstatus == GrainDirectoryEntryStatus.RaceLoser)
{
// we failed because an external request moved us to RACE_LOSER. Go back to REQUESTED_OWNERSHIP for retry
var success = directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.RequestedOwnership, GrainDirectoryEntryStatus.RaceLoser);
if (!success) ProtocolError(address, "unable to transition from RACE_LOSER to REQUESTED_OWNERSHIP");
// do not wait before retrying because there is a dominant remote request active so we can probably complete quickly
}
else
{
ProtocolError(address, "unhandled protocol state");
}
}
// we are done with the quick retries. Now we go into doubtful state, which means slower retries.
var ok = directoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RequestedOwnership);
if (!ok) ProtocolError(address, "unable to transition into doubtful");
this.gsiActivationMaintainer.TrackDoubtfulGrain(address.Grain);
return myActivation;
}
private ActivationAddress SelectAddress(List <ActivationAddress> results, GrainId grainId)
{
ActivationAddress unadjustedResult = null;
if (results is { Count : > 0 })
public virtual void Unregister(ActivationAddress address, bool force)
{
DirectoryPartition.RemoveActivation(address.Grain, address.Activation, force);
}
private void RunClientMessagePump(CancellationToken ct)
{
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
incomingMessagesThreadTimeTracking.OnStartExecution();
}
while (listenForMessages)
{
var message = transport.WaitMessage(Message.Categories.Application, ct);
if (message == null) // if wait was cancelled
{
break;
}
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
incomingMessagesThreadTimeTracking.OnStartProcessing();
}
#endif
// when we receive the first message, we update the
// clientId for this client because it may have been modified to
// include the cluster name
if (!firstMessageReceived)
{
firstMessageReceived = true;
if (!handshakeClientId.Equals(message.TargetGrain))
{
clientId = message.TargetGrain;
transport.UpdateClientId(clientId);
CurrentActivationAddress = ActivationAddress.GetAddress(transport.MyAddress, clientId, CurrentActivationAddress.Activation);
}
else
{
clientId = handshakeClientId;
}
}
switch (message.Direction)
{
case Message.Directions.Response:
{
ReceiveResponse(message);
break;
}
case Message.Directions.OneWay:
case Message.Directions.Request:
{
this.DispatchToLocalObject(message);
break;
}
default:
logger.Error(ErrorCode.Runtime_Error_100327, String.Format("Message not supported: {0}.", message));
break;
}
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
incomingMessagesThreadTimeTracking.OnStopProcessing();
incomingMessagesThreadTimeTracking.IncrementNumberOfProcessed();
}
#endif
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
incomingMessagesThreadTimeTracking.OnStopExecution();
}
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task<ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address)
{
registrationsSingleActIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsSingleActLocal.Increment();
// if I am the owner, store the new activation locally
Tuple<ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return returnedAddress == null ? null : returnedAddress.Item1;
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
Tuple<ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (returnedAddress == null || returnedAddress.Item1 == null) return null;
if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo)) return returnedAddress.Item1;
var cached = new List<Tuple<SiloAddress, ActivationId>>( new [] { Tuple.Create(address.Silo, address.Activation) });
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2);
return returnedAddress.Item1;
}
}
public Task UnregisterAsync(ActivationAddress address, UnregistrationCause cause, int hopCount)
{
return(router.UnregisterAsync(address, cause, hopCount));
}
public Task UnregisterAsync(ActivationAddress addr)
{
return UnregisterAsyncImpl(addr, true);
}
private List <SiloHandle> getSilosToFail(Fail fail, int numOfFailures)
{
List <SiloHandle> failures = new List <SiloHandle>();
int count = 0, index = 0;
// Figure out the primary directory partition and the silo hosting the ReminderTableGrain.
bool usingReminderGrain = this.HostedCluster.Primary.Silo.GlobalConfig.ReminderServiceType.Equals(GlobalConfiguration.ReminderServiceProviderType.ReminderTableGrain);
IReminderTable tableGrain = GrainClient.GrainFactory.GetGrain <IReminderTableGrain>(Constants.ReminderTableGrainId);
SiloAddress reminderTableGrainPrimaryDirectoryAddress = this.HostedCluster.Primary.Silo.LocalGrainDirectory.GetPrimaryForGrain(((GrainReference)tableGrain).GrainId);
SiloHandle reminderTableGrainPrimaryDirectory = this.HostedCluster.GetActiveSilos().Where(sh => sh.Silo.SiloAddress.Equals(reminderTableGrainPrimaryDirectoryAddress)).FirstOrDefault();
AddressesAndTag addresses;
bool res = reminderTableGrainPrimaryDirectory.Silo.LocalGrainDirectory.LocalLookup(((GrainReference)tableGrain).GrainId, out addresses);
ActivationAddress reminderGrainActivation = addresses.Addresses.FirstOrDefault();
SortedList <int, SiloHandle> ids = new SortedList <int, SiloHandle>();
foreach (var siloHandle in this.HostedCluster.GetActiveSilos())
{
SiloAddress siloAddress = siloHandle.Silo.SiloAddress;
if (siloAddress.Equals(this.HostedCluster.Primary.Silo.SiloAddress))
{
continue;
}
// Don't fail primary directory partition and the silo hosting the ReminderTableGrain.
if (usingReminderGrain)
{
if (siloAddress.Equals(reminderTableGrainPrimaryDirectoryAddress) || siloAddress.Equals(reminderGrainActivation.Silo))
{
continue;
}
}
ids.Add(siloHandle.Silo.SiloAddress.GetConsistentHashCode(), siloHandle);
}
// we should not fail the primary!
// we can't guarantee semantics of 'Fail' if it evalutes to the primary's address
switch (fail)
{
case Fail.First:
index = 0;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index++;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Last:
index = ids.Count - 1;
while (count++ < numOfFailures)
{
while (failures.Contains(ids.Values[index]))
{
index--;
}
failures.Add(ids.Values[index]);
}
break;
case Fail.Random:
default:
while (count++ < numOfFailures)
{
SiloHandle r = ids.Values[random.Next(ids.Count)];
while (failures.Contains(r))
{
r = ids.Values[random.Next(ids.Count)];
}
failures.Add(r);
}
break;
}
return(failures);
}
private Task UnregisterAsyncImpl(ActivationAddress addr, bool force)
{
unregistrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(addr.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (log.IsVerbose) log.Verbose("Silo {0} is going to unregister grain {1}-->{2} ({3}-->{4})", MyAddress, addr.Grain, owner, addr.Grain.GetUniformHashCode(), owner.GetConsistentHashCode());
InvalidateCacheEntry(addr);
if (owner.Equals(MyAddress))
{
UnregistrationsLocal.Increment();
// if I am the owner, remove the old activation locally
DirectoryPartition.RemoveActivation(addr.Grain, addr.Activation, force);
return TaskDone.Done;
}
UnregistrationsRemoteSent.Increment();
// otherwise, notify the owner
return GetDirectoryReference(owner).Unregister(addr, force, NUM_RETRIES);
}
public async Task <AddressAndTag> RegisterAsync(ActivationAddress address, bool singleActivation, int hopCount)
{
(singleActivation ? router.RegistrationsSingleActRemoteReceived : router.RegistrationsRemoteReceived).Increment();
return(await router.RegisterAsync(address, singleActivation, hopCount));
}
/// <summary>
/// Returns the activation of a single-activation grain, if present.
/// </summary>
internal GrainDirectoryEntryStatus TryGetActivation(GrainId grain, out ActivationAddress address, out int version)
{
lock (lockable)
{
IGrainInfo graininfo;
if (partitionData.TryGetValue(grain, out graininfo))
{
var first = graininfo.Instances.FirstOrDefault();
if (first.Value != null)
{
address = ActivationAddress.GetAddress(first.Value.SiloAddress, grain, first.Key);
version = graininfo.VersionTag;
return first.Value.RegistrationStatus;
}
}
}
address = null;
version = 0;
return GrainDirectoryEntryStatus.Invalid;
}
public Task UnregisterAsync(ActivationAddress address, bool force, int hopCount)
{
return(router.UnregisterAsync(address, force, hopCount));
}
public bool RemoveActivation(ActivationAddress addr)
{
return RemoveActivation(addr.Activation, true);
}
public void CachePlacementDecision(ActivationAddress address) => cache.AddOrUpdate(address, 0);
public void Unregister(ActivationAddress address, UnregistrationCause cause)
{
directoryPartition.RemoveActivation(address.Grain, address.Activation, cause);
}
public void InvalidateCache(ActivationAddress address) => cache.Remove(address);
public void InvalidateCache(ActivationAddress address)
{
// no need to do anything since the directory does not cache activations in remote clusters
}
public Task UnregisterAsync(ActivationAddress address, bool force, int hopCount)
{
return router.UnregisterAsync(address, force, hopCount);
}
