public Task DeleteGrain(GrainId grain)
{
SiloAddress silo = CalculateTargetSilo(grain);
if (silo == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (log.IsVerbose)
{
log.Verbose("Silo {0} tries to lookup for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
if (silo.Equals(MyAddress))
{
// remove from our partition
DirectoryPartition.RemoveGrain(grain);
return(TaskDone.Done);
}
// remove from the cache
DirectoryCache.Remove(grain);
// send request to the owner
return(GetDirectoryReference(silo).DeleteGrain(grain, NUM_RETRIES));
}
public List <ActivationAddress> GetLocalDirectoryData(GrainId grain)
{
var result = DirectoryPartition.LookUpGrain(grain);
return(result == null ? null :
result.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList());
}
private Task UnregisterAsyncImpl(ActivationAddress addr, bool force)
{
unregistrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(addr.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (log.IsVerbose)
{
log.Verbose("Silo {0} is going to unregister grain {1}-->{2} ({3}-->{4})", MyAddress, addr.Grain, owner, addr.Grain.GetUniformHashCode(), owner.GetConsistentHashCode());
}
InvalidateCacheEntry(addr);
if (owner.Equals(MyAddress))
{
UnregistrationsLocal.Increment();
// if I am the owner, remove the old activation locally
DirectoryPartition.RemoveActivation(addr.Grain, addr.Activation, force);
return(TaskDone.Done);
}
UnregistrationsRemoteSent.Increment();
// otherwise, notify the owner
return(GetDirectoryReference(owner).Unregister(addr, force, NUM_RETRIES));
}
public Task UnregisterManyAsync(List <ActivationAddress> addresses)
{
unregistrationsManyIssued.Increment();
return(Task.WhenAll(
addresses.GroupBy(a => CalculateTargetSilo(a.Grain))
.Select(g =>
{
if (g.Key == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
foreach (var addr in g)
{
InvalidateCacheEntry(addr);
}
if (MyAddress.Equals(g.Key))
{
// if I am the owner, remove the old activation locally
foreach (var addr in g)
{
UnregistrationsLocal.Increment();
DirectoryPartition.RemoveActivation(addr.Grain, addr.Activation, true);
}
return TaskDone.Done;
}
UnregistrationsManyRemoteSent.Increment();
// otherwise, notify the owner
return GetDirectoryReference(g.Key).UnregisterMany(g.ToList(), NUM_RETRIES);
})));
}
public async Task DeleteGrainAsync(GrainId grainId, int hopCount)
{
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync");
this.log.LogWarning($"DeleteGrainAsync - It seems we are not the owner of grain {grainId}, trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
if (forwardAddress == null)
{
// we are the owner
DirectoryPartition.RemoveGrain(grainId);
}
else
{
// otherwise, notify the owner
DirectoryCache.Remove(grainId);
await GetDirectoryReference(forwardAddress).DeleteGrainAsync(grainId, hopCount + 1);
}
}
public async Task UnregisterAsync(GrainAddress address, UnregistrationCause cause, int hopCount)
{
(hopCount > 0 ? UnregistrationsRemoteReceived : unregistrationsIssued).Increment();
if (hopCount == 0)
{
InvalidateCacheEntry(address);
}
// see if the owner is somewhere else (returns null if we are owner)
var forwardaddress = this.CheckIfShouldForward(address.GrainId, hopCount, "UnregisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardaddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardaddress = this.CheckIfShouldForward(address.GrainId, hopCount, "UnregisterAsync");
this.log.LogWarning($"UnregisterAsync - It seems we are not the owner of activation {address}, trying to forward it to {forwardaddress} (hopCount={hopCount})");
}
if (forwardaddress == null)
{
// we are the owner
UnregistrationsLocal.Increment();
DirectoryPartition.RemoveActivation(address.GrainId, address.ActivationId, cause);
}
else
{
UnregistrationsRemoteSent.Increment();
// otherwise, notify the owner
await GetDirectoryReference(forwardaddress).UnregisterAsync(address, cause, hopCount + 1);
}
}
/// <summary>
/// Adjust local directory following the addition/removal of a silo
/// </summary>
protected void AdjustLocalDirectory(SiloAddress silo, bool dead)
{
// Determine which activations to remove.
var activationsToRemove = new List <(GrainId, ActivationId)>();
foreach (var entry in this.DirectoryPartition.GetItems())
{
var(grain, grainInfo) = (entry.Key, entry.Value);
if (grainInfo.Activation is { } address)
{
// Include any activations from dead silos and from predecessors.
var siloIsDead = dead && address.SiloAddress.Equals(silo);
var siloIsPredecessor = address.SiloAddress.IsPredecessorOf(silo);
if (siloIsDead || siloIsPredecessor)
{
activationsToRemove.Add((grain, address.ActivationId));
}
}
}
// Remove all defunct activations.
foreach (var activation in activationsToRemove)
{
DirectoryPartition.RemoveActivation(activation.Item1, activation.Item2);
}
}
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount)
{
var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued;
counterStatistic.Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)address.GrainId.GetUniformHashCode());
this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
RegistrationsSingleActLocal.Increment();
var result = DirectoryPartition.AddSingleActivation(address);
return(result);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (result.Address == null)
{
return(result);
}
if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress))
{
return(result);
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address, result.VersionTag);
return(result);
}
}
/// <summary>
/// Adjust local directory following the removal of a silo by droping all activations located on the removed silo
/// </summary>
/// <param name="removedSilo"></param>
protected void AdjustLocalDirectory(SiloAddress removedSilo)
{
var activationsToRemove = (from pair in DirectoryPartition.GetItems()
from pair2 in pair.Value.Instances.Where(pair3 => pair3.Value.SiloAddress.Equals(removedSilo))
select new Tuple <GrainId, ActivationId>(pair.Key, pair2.Key)).ToList();
// drop all records of activations located on the removed silo
foreach (var activation in activationsToRemove)
{
DirectoryPartition.RemoveActivation(activation.Item1, activation.Item2, true);
}
}
public async Task RegisterAsync(ActivationAddress address)
{
registrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsLocal.Increment();
// if I am the owner, store the new activation locally
DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
}
else
{
RegistrationsRemoteSent.Increment();
// otherwise, notify the owner
int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES);
if (IsValidSilo(address.Silo))
{
// Caching optimization:
// cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached;
if (!DirectoryCache.LookUp(address.Grain, out cached))
{
cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
}
else
{
var newcached = new List <Tuple <SiloAddress, ActivationId> >(cached.Count + 1);
newcached.AddRange(cached);
newcached.Add(Tuple.Create(address.Silo, address.Activation));
cached = newcached;
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, eTag);
}
}
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address)
{
registrationsSingleActIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsSingleActLocal.Increment();
// if I am the owner, store the new activation locally
Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return(returnedAddress == null ? null : returnedAddress.Item1);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (returnedAddress == null || returnedAddress.Item1 == null)
{
return(null);
}
if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo))
{
return(returnedAddress.Item1);
}
var cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2);
return(returnedAddress.Item1);
}
}
// Note that this implementation stops processing directory change requests (Register, Unregister, etc.) when the Stop event is raised.
// This means that there may be a short period during which no silo believes that it is the owner of directory information for a set of
// grains (for update purposes), which could cause application requests that require a new activation to be created to time out.
// The alternative would be to allow the silo to process requests after it has handed off its partition, in which case those changes
// would receive successful responses but would not be reflected in the eventual state of the directory.
// It's easy to change this, if we think the trade-off is better the other way.
public void Stop()
{
// This will cause remote write requests to be forwarded to the silo that will become the new owner.
// Requests might bounce back and forth for a while as membership stabilizes, but they will either be served by the
// new owner of the grain, or will wind up failing. In either case, we avoid requests succeeding at this silo after we've
// begun stopping, which could cause them to not get handed off to the new owner.
//mark Running as false will exclude myself from CalculateGrainDirectoryPartition(grainId)
Running = false;
if (maintainer != null)
{
maintainer.Stop();
}
DirectoryPartition.Clear();
DirectoryCache.Clear();
}
// helper method to avoid code duplication inside UnregisterManyAsync
private void UnregisterOrPutInForwardList(IEnumerable <GrainAddress> addresses, UnregistrationCause cause, int hopCount,
ref Dictionary <SiloAddress, List <GrainAddress> > forward, List <Task> tasks, string context)
{
foreach (var address in addresses)
{
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, context);
if (forwardAddress != null)
{
AddToDictionary(ref forward, forwardAddress, address);
}
else
{
// we are the owner
UnregistrationsLocal.Increment();
DirectoryPartition.RemoveActivation(address.GrainId, address.ActivationId, cause);
}
}
}
public async Task <List <ActivationAddress> > FullLookup(GrainId grain)
{
fullLookups.Increment();
SiloAddress silo = CalculateTargetSilo(grain, false);
// No need to check that silo != null since we're passing excludeThisSiloIfStopping = false
if (log.IsVerbose)
{
log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
// We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()).
// We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called.
if (silo.Equals(MyAddress))
{
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpGrain(grain);
if (localResult == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}=none", grain);
}
return(new List <ActivationAddress>());
}
var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings());
}
LocalDirectorySuccesses.Increment();
return(a);
}
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(silo))
{
throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo));
}
RemoteLookupsSent.Increment();
Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES);
// update the cache
List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList();
List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings());
}
if (entries.Count > 0)
{
DirectoryCache.AddOrUpdate(grain, entries, result.Item2);
}
return(addresses);
}
public async Task <AddressAndTag> LookupAsync(GrainId grainId, int hopCount = 0)
{
(hopCount > 0 ? RemoteLookupsReceived : fullLookups).Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)grainId.GetUniformHashCode());
this.log.LogWarning($"LookupAsync - It seems we are not the owner of grain {grainId} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
// we are the owner
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpActivation(grainId);
if (localResult.Address == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup mine {0}=none", grainId);
}
localResult.Address = default;
localResult.VersionTag = GrainInfo.NO_ETAG;
return(localResult);
}
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup mine {0}={1}", grainId, localResult.Address);
}
LocalDirectorySuccesses.Increment();
return(localResult);
}
else
{
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(forwardAddress))
{
throw new OrleansException($"Current directory at {MyAddress} is not stable to perform the lookup for grainId {grainId} (it maps to {forwardAddress}, which is not a valid silo). Retry later.");
}
RemoteLookupsSent.Increment();
var result = await GetDirectoryReference(forwardAddress).LookupAsync(grainId, hopCount + 1);
// update the cache
if (result.Address is { } address&& IsValidSilo(address.SiloAddress))
{
DirectoryCache.AddOrUpdate(address, result.VersionTag);
}
if (log.IsEnabled(LogLevel.Trace))
{
log.Trace("FullLookup remote {0}={1}", grainId, result.Address);
}
return(result);
}
}
public AddressAndTag GetLocalDirectoryData(GrainId grain)
{
return(DirectoryPartition.LookUpActivation(grain));
}