public async Task UnregisterAsync(GrainAddress address, UnregistrationCause cause, int hopCount)
{
(hopCount > 0 ? UnregistrationsRemoteReceived : unregistrationsIssued).Increment();
if (hopCount == 0)
{
InvalidateCacheEntry(address);
}
// see if the owner is somewhere else (returns null if we are owner)
var forwardaddress = this.CheckIfShouldForward(address.GrainId, hopCount, "UnregisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardaddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardaddress = this.CheckIfShouldForward(address.GrainId, hopCount, "UnregisterAsync");
this.log.LogWarning($"UnregisterAsync - It seems we are not the owner of activation {address}, trying to forward it to {forwardaddress} (hopCount={hopCount})");
}
if (forwardaddress == null)
{
// we are the owner
UnregistrationsLocal.Increment();
DirectoryPartition.RemoveActivation(address.GrainId, address.ActivationId, cause);
}
else
{
UnregistrationsRemoteSent.Increment();
// otherwise, notify the owner
await GetDirectoryReference(forwardaddress).UnregisterAsync(address, cause, hopCount + 1);
}
}
// used for testing to (carefully!) allow two clients in the same process
private async Task StartInternal(CancellationToken cancellationToken)
{
var retryFilter = ServiceProvider.GetService <IClientConnectionRetryFilter>();
var gatewayManager = this.ServiceProvider.GetRequiredService <GatewayManager>();
await ExecuteWithRetries(
async() => await gatewayManager.StartAsync(cancellationToken),
retryFilter,
cancellationToken);
var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative
MessageCenter = ActivatorUtilities.CreateInstance <ClientMessageCenter>(this.ServiceProvider, localAddress, generation, clientId);
MessageCenter.RegisterLocalMessageHandler(this.HandleMessage);
await ExecuteWithRetries(
async() => await MessageCenter.StartAsync(cancellationToken),
retryFilter,
cancellationToken);
CurrentActivationAddress = GrainAddress.NewActivationAddress(MessageCenter.MyAddress, clientId.GrainId);
this.gatewayObserver = new ClientGatewayObserver(gatewayManager);
this.InternalGrainFactory.CreateObjectReference <IClientGatewayObserver>(this.gatewayObserver);
await ExecuteWithRetries(
async() => await this.ServiceProvider.GetRequiredService <ClientClusterManifestProvider>().StartAsync(),
retryFilter,
cancellationToken);
ClientStatistics.Start(MessageCenter, clientId.GrainId);
private GrainAddress GenerateActivationAddress()
{
var grainId = LegacyGrainId.GetGrainIdForTesting(Guid.NewGuid());
var siloAddr = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, 5000), ++generation);
return(GrainAddress.NewActivationAddress(siloAddr, grainId));
}
// used for testing to (carefully!) allow two clients in the same process
private async Task StartInternal(CancellationToken cancellationToken)
{
var retryFilterInterface = ServiceProvider.GetService <IClientConnectionRetryFilter>();
Func <Exception, CancellationToken, Task <bool> > retryFilter = RetryFilter;
var gatewayManager = this.ServiceProvider.GetRequiredService <GatewayManager>();
await ExecuteWithRetries(async() => await gatewayManager.StartAsync(cancellationToken), retryFilter);
var generation = -SiloAddress.AllocateNewGeneration(); // Client generations are negative
MessageCenter = ActivatorUtilities.CreateInstance <ClientMessageCenter>(this.ServiceProvider, localAddress, generation, clientId);
MessageCenter.RegisterLocalMessageHandler(this.HandleMessage);
MessageCenter.Start();
CurrentActivationAddress = GrainAddress.NewActivationAddress(MessageCenter.MyAddress, clientId.GrainId);
this.gatewayObserver = new ClientGatewayObserver(gatewayManager);
this.InternalGrainFactory.CreateObjectReference <IClientGatewayObserver>(this.gatewayObserver);
await ExecuteWithRetries(
async() => await this.ServiceProvider.GetRequiredService <ClientClusterManifestProvider>().StartAsync(),
retryFilter);
ClientStatistics.Start(MessageCenter, clientId.GrainId);
async Task ExecuteWithRetries(Func <Task> task, Func <Exception, CancellationToken, Task <bool> > shouldRetry)
{
while (true)
{
try
{
await task();
return;
}
catch (Exception exception) when(shouldRetry != null)
{
var retry = await shouldRetry(exception, cancellationToken);
if (!retry)
{
throw;
}
}
}
}
async Task <bool> RetryFilter(Exception exception, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
if (retryFilterInterface is { })
{
var result = await retryFilterInterface.ShouldRetryConnectionAttempt(exception, cancellationToken);
return(result && !cancellationToken.IsCancellationRequested);
}
public async Task DoNotOverrideEntry()
{
var expected = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = GrainId.Parse("user/someraondomuser_" + Guid.NewGuid().ToString("N")),
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
var differentActivation = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = expected.GrainId,
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
var differentSilo = new GrainAddress
{
ActivationId = expected.ActivationId,
GrainId = expected.GrainId,
SiloAddress = SiloAddress.FromParsableString("10.0.23.14:1000@4583")
};
Assert.Equal(expected, await this.grainDirectory.Register(expected));
Assert.Equal(expected, await this.grainDirectory.Register(differentActivation));
Assert.Equal(expected, await this.grainDirectory.Register(differentSilo));
Assert.Equal(expected, await this.grainDirectory.Lookup(expected.GrainId));
}
public bool TryLookupInCache(GrainId grainId, out GrainAddress address)
{
var grainType = grainId.Type;
if (grainType.IsClient() || grainType.IsSystemTarget())
{
ThrowUnsupportedGrainType(grainId);
}
if (this.cache.LookUp(grainId, out address, out var version))
{
// If the silo is dead, remove the entry
if (IsKnownDeadSilo(address.SiloAddress, new MembershipVersion(version)))
{
address = default;
this.cache.Remove(grainId);
}
else
{
// Entry found and valid -> return it
return(true);
}
}
return(false);
}
public async Task <GrainAddress> Register(GrainAddress address)
{
var grainType = address.GrainId.Type;
if (grainType.IsClient() || grainType.IsSystemTarget())
{
ThrowUnsupportedGrainType(address.GrainId);
}
address.MembershipVersion = this.clusterMembershipService.CurrentSnapshot.Version;
var result = await GetGrainDirectory(grainType).Register(address);
// Check if the entry point to a dead silo
if (IsKnownDeadSilo(result))
{
// Remove outdated entry and retry to register
await GetGrainDirectory(grainType).Unregister(result);
result = await GetGrainDirectory(grainType).Register(address);
}
// Cache update
this.cache.AddOrUpdate(result, (int)result.MembershipVersion.Value);
return(result);
}
public static ActivationAddress ToActivationAddress(this GrainAddress addr)
{
return(ActivationAddress.GetAddress(
addr.SiloAddress,
addr.GrainId,
ActivationId.GetActivationId(UniqueKey.Parse(addr.ActivationId.AsSpan()))));
}
public static ActivationAddress ToActivationAddress(this GrainAddress addr)
{
return(ActivationAddress.GetAddress(
SiloAddress.FromParsableString(addr.SiloAddress),
GrainId.FromParsableString(addr.GrainId),
ActivationId.GetActivationId(UniqueKey.Parse(addr.ActivationId.AsSpan()))));
}
public void AddOrUpdate(GrainAddress value, int version)
{
var entry = new GrainDirectoryCacheEntry(value, version, initialExpirationTimer);
// Notice that LRU should know how to throw the oldest entry if the cache is full
cache.Add(value.GrainId, entry);
}
public async Task <GrainAddress> Register(GrainAddress address)
{
var entry = GrainDirectoryEntity.FromGrainAddress(this.clusterId, address);
var result = await this.tableDataManager.InsertTableEntryAsync(entry);
// Possible race condition?
return(result.isSuccess ? address : await Lookup(address.GrainId));
}
internal GrainDirectoryCacheEntry(GrainAddress value, int etag, TimeSpan expirationTimer)
{
Address = value;
ETag = etag;
ExpirationTimer = expirationTimer;
LastRefreshed = CoarseStopwatch.StartNew();
NumAccesses = 0;
}
public static GrainDirectoryEntity FromGrainAddress(string clusterId, GrainAddress address)
{
return(new GrainDirectoryEntity
{
PartitionKey = clusterId,
RowKey = GrainIdToRowKey(address.GrainId),
SiloAddress = address.SiloAddress.ToParsableString(),
ActivationId = address.ActivationId.ToParsableString(),
});
}
public static GrainDirectoryEntity FromGrainAddress(string clusterId, GrainAddress address)
{
return(new GrainDirectoryEntity
{
PartitionKey = clusterId,
RowKey = HttpUtility.UrlEncode(address.GrainId, Encoding.UTF8),
SiloAddress = address.SiloAddress,
ActivationId = address.ActivationId,
});
}
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount)
{
var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued;
counterStatistic.Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)address.GrainId.GetUniformHashCode());
this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
RegistrationsSingleActLocal.Increment();
var result = DirectoryPartition.AddSingleActivation(address);
return(result);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (result.Address == null)
{
return(result);
}
if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress))
{
return(result);
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address, result.VersionTag);
return(result);
}
}
public async Task Unregister(GrainAddress address, UnregistrationCause cause)
{
try
{
await GetGrainDirectory(address.GrainId.Type).Unregister(address);
}
finally
{
this.cache.Remove(address);
}
}
public Task UnregisterAfterNonexistingActivation(GrainAddress addr, SiloAddress origin)
{
log.Trace("UnregisterAfterNonexistingActivation addr={0} origin={1}", addr, origin);
if (origin == null || this.directoryMembership.MembershipCache.Contains(origin))
{
// the request originated in this cluster, call unregister here
return(UnregisterAsync(addr, UnregistrationCause.NonexistentActivation, 0));
}
else
{
// the request originated in another cluster, call unregister there
var remoteDirectory = GetDirectoryReference(origin);
return(remoteDirectory.UnregisterAsync(addr, UnregistrationCause.NonexistentActivation));
}
}
public async Task RegisterLookupUnregisterLookup()
{
var expected = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = GrainId.Parse("user/someraondomuser_" + Guid.NewGuid().ToString("N")),
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
Assert.Equal(expected, await this.grainDirectory.Register(expected));
Assert.Equal(expected, await this.grainDirectory.Lookup(expected.GrainId));
await this.grainDirectory.Unregister(expected);
Assert.Null(await this.grainDirectory.Lookup(expected.GrainId));
}
public async Task Unregister(GrainAddress address)
{
var result = await this.tableDataManager.ReadSingleTableEntryAsync(this.clusterId, HttpUtility.UrlEncode(address.GrainId, Encoding.UTF8));
// No entry found
if (result == null)
{
return;
}
// Check if the entry in storage match the one we were asked to delete
var entity = result.Item1;
if (entity.ActivationId == address.ActivationId)
{
await this.tableDataManager.DeleteTableEntryAsync(GrainDirectoryEntity.FromGrainAddress(this.clusterId, address), entity.ETag);
}
}
public async Task Unregister(GrainAddress address)
{
var result = await this.tableDataManager.ReadSingleTableEntryAsync(this.clusterId, GrainDirectoryEntity.GrainIdToRowKey(address.GrainId));
// No entry found
if (result.Entity is null)
{
return;
}
// Check if the entry in storage match the one we were asked to delete
var entity = result.Item1;
if (entity.ActivationId == address.ActivationId.ToParsableString())
{
await this.tableDataManager.DeleteTableEntryAsync(GrainDirectoryEntity.FromGrainAddress(this.clusterId, address), entity.ETag.ToString());
}
}
public async Task UnregisterMany()
{
const int N = 250;
const int R = 40;
// Create and insert N entries
var addresses = new List <GrainAddress>();
for (var i = 0; i < N; i++)
{
var addr = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = GrainId.Parse("user/someraondomuser_" + Guid.NewGuid().ToString("N")),
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
addresses.Add(addr);
await this.grainDirectory.Register(addr);
}
// Modify the Rth entry locally, to simulate another activation tentative by another silo
var oldActivation = addresses[R].ActivationId;
addresses[R].ActivationId = ActivationId.NewId();
// Batch unregister
await this.grainDirectory.UnregisterMany(addresses);
// Now we should only find the old Rth entry
for (int i = 0; i < N; i++)
{
if (i == R)
{
var addr = await this.grainDirectory.Lookup(addresses[i].GrainId);
Assert.NotNull(addr);
Assert.Equal(oldActivation, addr.ActivationId);
}
else
{
Assert.Null(await this.grainDirectory.Lookup(addresses[i].GrainId));
}
}
}
public bool LookUp(GrainId key, out GrainAddress result, out int version)
{
NumAccesses++; // for stats
// Here we do not check whether the found entry is expired.
// It will be done by the thread managing the cache.
// This is to avoid situation where the entry was just expired, but the manager still have not run and have not refereshed it.
if (!cache.TryGetValue(key, out var tmp))
{
result = default;
version = default;
return(false);
}
NumHits++; // for stats
tmp.NumAccesses++;
result = tmp.Address;
version = tmp.ETag;
return(true);
}
public async Task DoNotDeleteDifferentActivationIdEntry()
{
var expected = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = GrainId.Parse("user/someraondomuser_" + Guid.NewGuid().ToString("N")),
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
var otherEntry = new GrainAddress
{
ActivationId = ActivationId.NewId(),
GrainId = expected.GrainId,
SiloAddress = SiloAddress.FromParsableString("10.0.23.12:1000@5678")
};
Assert.Equal(expected, await this.grainDirectory.Register(expected));
await this.grainDirectory.Unregister(otherEntry);
Assert.Equal(expected, await this.grainDirectory.Lookup(expected.GrainId));
}
/// <summary>
/// Looks up the cached value by the given key.
/// </summary>
/// <param name="cache">grain directory cache to look up results from</param>
/// <param name="key">key for the lookup</param>
/// <param name="result">value if the key is found, undefined otherwise</param>
/// <returns>true if the given key is in the cache</returns>
public static bool LookUp(this IGrainDirectoryCache cache, GrainId key, out GrainAddress result)
{
return(cache.LookUp(key, out result, out _));
}
public bool TryCachedLookup(GrainId grainId, out GrainAddress address) => (address = GetLocalCacheData(grainId)) is not null;
public GrainAddress TryAddSingleActivation(GrainAddress address)
{
if (Activation is { } existing)
{
return(existing);
}
public void InvalidateCacheEntry(GrainAddress activationAddress)
{
DirectoryCache.Remove(activationAddress);
}
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount)
{
router.RegistrationsSingleActRemoteReceived.Increment();
return(await router.RegisterAsync(address, hopCount));
}
public Task UnregisterAsync(GrainAddress address, UnregistrationCause cause, int hopCount)
{
return(router.UnregisterAsync(address, cause, hopCount));
}
private GrainAddress SelectAddress(List <GrainAddress> results, GrainId grainId)
{
GrainAddress unadjustedResult = null;
if (results is { Count : > 0 })