/// Adjust local cache following the removal of a silo by droping:
/// 1) entries that point to activations located on the removed silo
/// 2) entries for grains that are now owned by this silo (me)
/// 3) entries for grains that were owned by this removed silo - we currently do NOT do that.
/// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership).
/// We don't do that since first cache refresh handles that.
/// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo.
/// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner).
protected void AdjustLocalCache(SiloAddress removedSilo)
{
// remove all records of activations located on the removed silo
foreach (Tuple <GrainId, List <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues)
{
// 2) remove entries owned by me (they should be retrieved from my directory partition)
if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1)))
{
DirectoryCache.Remove(tuple.Item1);
}
// 1) remove entries that point to activations located on the removed silo
if (tuple.Item2.RemoveAll(tuple2 => tuple2.Item1.Equals(removedSilo)) <= 0)
{
continue;
}
if (tuple.Item2.Count > 0)
{
DirectoryCache.AddOrUpdate(tuple.Item1, tuple.Item2, tuple.Item3);
}
else
{
DirectoryCache.Remove(tuple.Item1);
}
}
}
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount)
{
var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued;
counterStatistic.Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)address.GrainId.GetUniformHashCode());
this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
RegistrationsSingleActLocal.Increment();
var result = DirectoryPartition.AddSingleActivation(address);
return(result);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (result.Address == null)
{
return(result);
}
if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress))
{
return(result);
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address, result.VersionTag);
return(result);
}
}
public async Task RegisterAsync(ActivationAddress address)
{
registrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsLocal.Increment();
// if I am the owner, store the new activation locally
DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
}
else
{
RegistrationsRemoteSent.Increment();
// otherwise, notify the owner
int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES);
if (IsValidSilo(address.Silo))
{
// Caching optimization:
// cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached;
if (!DirectoryCache.LookUp(address.Grain, out cached))
{
cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
}
else
{
var newcached = new List <Tuple <SiloAddress, ActivationId> >(cached.Count + 1);
newcached.AddRange(cached);
newcached.Add(Tuple.Create(address.Silo, address.Activation));
cached = newcached;
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, eTag);
}
}
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address)
{
registrationsSingleActIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsSingleActLocal.Increment();
// if I am the owner, store the new activation locally
Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return(returnedAddress == null ? null : returnedAddress.Item1);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (returnedAddress == null || returnedAddress.Item1 == null)
{
return(null);
}
if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo))
{
return(returnedAddress.Item1);
}
var cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2);
return(returnedAddress.Item1);
}
}
public async Task <List <ActivationAddress> > FullLookup(GrainId grain)
{
fullLookups.Increment();
SiloAddress silo = CalculateTargetSilo(grain, false);
// No need to check that silo != null since we're passing excludeThisSiloIfStopping = false
if (log.IsVerbose)
{
log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
// We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()).
// We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called.
if (silo.Equals(MyAddress))
{
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpGrain(grain);
if (localResult == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}=none", grain);
}
return(new List <ActivationAddress>());
}
var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings());
}
LocalDirectorySuccesses.Increment();
return(a);
}
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(silo))
{
throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo));
}
RemoteLookupsSent.Increment();
Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES);
// update the cache
List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList();
List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings());
}
if (entries.Count > 0)
{
DirectoryCache.AddOrUpdate(grain, entries, result.Item2);
}
return(addresses);
}
public async Task <AddressAndTag> LookupAsync(GrainId grainId, int hopCount = 0)
{
(hopCount > 0 ? RemoteLookupsReceived : fullLookups).Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)grainId.GetUniformHashCode());
this.log.LogWarning($"LookupAsync - It seems we are not the owner of grain {grainId} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
// we are the owner
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpActivation(grainId);
if (localResult.Address == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup mine {0}=none", grainId);
}
localResult.Address = default;
localResult.VersionTag = GrainInfo.NO_ETAG;
return(localResult);
}
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup mine {0}={1}", grainId, localResult.Address);
}
LocalDirectorySuccesses.Increment();
return(localResult);
}
else
{
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(forwardAddress))
{
throw new OrleansException($"Current directory at {MyAddress} is not stable to perform the lookup for grainId {grainId} (it maps to {forwardAddress}, which is not a valid silo). Retry later.");
}
RemoteLookupsSent.Increment();
var result = await GetDirectoryReference(forwardAddress).LookupAsync(grainId, hopCount + 1);
// update the cache
if (result.Address is { } address&& IsValidSilo(address.SiloAddress))
{
DirectoryCache.AddOrUpdate(address, result.VersionTag);
}
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup remote {0}={1}", grainId, result.Address);
}
return(result);
}
}