private void ProcessTableUpdate(MembershipTableData table, string caller)
{
if (table is null)
{
throw new ArgumentNullException(nameof(table));
}
if (this.log.IsEnabled(LogLevel.Debug))
{
this.log.LogDebug(nameof(ProcessTableUpdate) + " (called from {Caller}) membership table {Table}", caller, table.ToString());
}
// Update the current membership snapshot.
var(localSiloEntry, _) = this.GetOrCreateLocalSiloEntry(table, this.CurrentStatus);
var updated = MembershipTableSnapshot.Create(localSiloEntry, table);
if (this.updates.TryPublish(updated))
{
this.LogMissedIAmAlives(table);
this.log.LogInformation(
(int)ErrorCode.MembershipReadAll_2,
nameof(ProcessTableUpdate) + " (called from {Caller}) membership table: {Table}",
caller,
table.WithoutDuplicateDeads().ToString());
}
}
private void DetectNodeMigration(MembershipTableSnapshot snapshot, string myHostname)
{
string mySiloName = this.localSiloDetails.Name;
MembershipEntry mostRecentPreviousEntry = null;
// look for silo instances that are same as me, find most recent with Generation before me.
foreach (var entry in snapshot.Entries.Select(entry => entry.Value).Where(data => mySiloName.Equals(data.SiloName)))
{
bool iAmLater = myAddress.Generation.CompareTo(entry.SiloAddress.Generation) > 0;
// more recent
if (iAmLater && (mostRecentPreviousEntry == null || entry.SiloAddress.Generation.CompareTo(mostRecentPreviousEntry.SiloAddress.Generation) > 0))
{
mostRecentPreviousEntry = entry;
}
}
if (mostRecentPreviousEntry != null)
{
bool physicalHostChanged = !myHostname.Equals(mostRecentPreviousEntry.HostName) || !myAddress.Endpoint.Equals(mostRecentPreviousEntry.SiloAddress.Endpoint);
if (physicalHostChanged)
{
string error = string.Format("Silo {0} migrated from host {1} silo address {2} to host {3} silo address {4}.",
mySiloName, myHostname, myAddress, mostRecentPreviousEntry.HostName, mostRecentPreviousEntry.SiloAddress);
log.Warn(ErrorCode.MembershipNodeMigrated, error);
}
else
{
string error = string.Format("Silo {0} restarted on same host {1} New silo address = {2} Previous silo address = {3}",
mySiloName, myHostname, myAddress, mostRecentPreviousEntry.SiloAddress);
log.Warn(ErrorCode.MembershipNodeRestarted, error);
}
}
}
public MembershipTableManager(
ILocalSiloDetails localSiloDetails,
IOptions <ClusterMembershipOptions> clusterMembershipOptions,
IMembershipTable membershipTable,
IFatalErrorHandler fatalErrorHandler,
IMembershipGossiper gossiper,
ILogger <MembershipTableManager> log,
IAsyncTimerFactory timerFactory,
ISiloLifecycle siloLifecycle)
{
this.localSiloDetails = localSiloDetails;
this.membershipTableProvider = membershipTable;
this.fatalErrorHandler = fatalErrorHandler;
this.gossiper = gossiper;
this.clusterMembershipOptions = clusterMembershipOptions.Value;
this.myAddress = this.localSiloDetails.SiloAddress;
this.log = log;
this.siloLifecycle = siloLifecycle;
this.snapshot = new MembershipTableSnapshot(
this.CreateLocalSiloEntry(this.CurrentStatus),
MembershipVersion.MinValue,
ImmutableDictionary <SiloAddress, MembershipEntry> .Empty);
this.updates = new AsyncEnumerable <MembershipTableSnapshot>(
(previous, proposed) => proposed.Version > previous.Version,
this.snapshot)
{
OnPublished = update => Interlocked.Exchange(ref this.snapshot, update)
};
this.membershipUpdateTimer = timerFactory.Create(
this.clusterMembershipOptions.TableRefreshTimeout,
nameof(PeriodicallyRefreshMembershipTable));
}
private async Task GossipToRemoteSilo(
SiloAddress silo,
MembershipTableSnapshot snapshot,
SiloAddress updatedSilo,
SiloStatus updatedStatus)
{
if (this.log.IsEnabled(LogLevel.Trace))
{
this.log.LogTrace(
"-Sending status update GOSSIP notification about silo {UpdatedSilo}, status {UpdatedStatus}, to silo {RemoteSilo}",
updatedSilo,
updatedStatus,
silo);
}
try
{
var remoteOracle = this.grainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipOracleId, silo);
try
{
await remoteOracle.MembershipChangeNotification(snapshot);
}
catch (NotImplementedException)
{
// Fallback to "old" gossip
await remoteOracle.SiloStatusChangeNotification(updatedSilo, updatedStatus);
}
}
catch (Exception exception)
{
this.log.LogError(
(int)ErrorCode.MembershipGossipSendFailure,
"Exception while sending gossip notification to remote silo {Silo}: {Exception}", silo, exception);
}
}
public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot)
{
// Check if a refresh is underway
var pending = this.pendingRefresh;
if (pending != null && !pending.IsCompleted)
{
await pending;
}
this.log.LogInformation("Received cluster membership snapshot via gossip: {Snapshot}", snapshot);
if (snapshot.Entries.TryGetValue(this.myAddress, out var localSiloEntry))
{
if (localSiloEntry.Status == SiloStatus.Dead && this.CurrentStatus != SiloStatus.Dead)
{
var msg = $"I should be Dead according to membership table (in RefreshFromSnapshot). Local entry: {(localSiloEntry.ToFullString(full: true))}.";
this.log.Warn(ErrorCode.MembershipFoundMyselfDead1, msg);
this.KillMyselfLocally(msg);
}
snapshot = MembershipTableSnapshot.Create(localSiloEntry.WithStatus(this.CurrentStatus), snapshot);
}
else
{
snapshot = MembershipTableSnapshot.Create(this.CreateLocalSiloEntry(this.CurrentStatus), snapshot);
}
// If we are behind, let's take directly the snapshot in param
this.updates.TryPublish(snapshot);
}
internal static ClusterMembershipSnapshot CreateClusterMembershipSnapshot(this MembershipTableSnapshot membership)
{
var memberBuilder = ImmutableDictionary.CreateBuilder <SiloAddress, ClusterMember>();
foreach (var member in membership.Entries)
{
var entry = member.Value;
memberBuilder[entry.SiloAddress] = new ClusterMember(entry.SiloAddress, entry.Status, entry.SiloName);
}
return(new ClusterMembershipSnapshot(memberBuilder.ToImmutable(), membership.Version));
}
public async Task RefreshFromSnapshot(MembershipTableSnapshot snapshot)
{
// Check if a refresh is underway
var pending = this.pendingRefresh;
if (pending != null && !pending.IsCompleted)
{
await pending;
}
// If we are behind, let's take directly the snapshot in param
this.updates.TryPublish(snapshot);
}
public async Task MembershipChangeNotification(MembershipTableSnapshot snapshot)
{
if (snapshot.Version != MembershipVersion.MinValue)
{
await this.membershipTableManager.RefreshFromSnapshot(snapshot);
}
else
{
if (this.log.IsEnabled(LogLevel.Trace))
{
this.log.LogTrace("-Received GOSSIP MembershipChangeNotification with MembershipVersion.MinValue. Going to read the table");
}
await ReadTable();
}
}
public Task GossipToRemoteSilos(
List <SiloAddress> gossipPartners,
MembershipTableSnapshot snapshot,
SiloAddress updatedSilo,
SiloStatus updatedStatus)
{
async Task Gossip()
{
var tasks = new List <Task>(gossipPartners.Count);
foreach (var silo in gossipPartners)
{
tasks.Add(this.GossipToRemoteSilo(silo, snapshot, updatedSilo, updatedStatus));
}
await Task.WhenAll(tasks);
}
return(this.RunOrQueueTask(Gossip));
}
public Task GossipToRemoteSilos(
List <SiloAddress> gossipPartners,
MembershipTableSnapshot snapshot,
SiloAddress updatedSilo,
SiloStatus updatedStatus)
{
if (gossipPartners.Count == 0)
{
return(Task.CompletedTask);
}
this.log.LogInformation(
"Gossiping {Silo} status change to {Status} to {NumPartners} partners",
updatedSilo,
updatedStatus,
gossipPartners.Count);
var systemTarget = this.serviceProvider.GetRequiredService <MembershipSystemTarget>();
return(systemTarget.GossipToRemoteSilos(gossipPartners, snapshot, updatedSilo, updatedStatus));
}
public MembershipTableManager(
ILocalSiloDetails localSiloDetails,
IOptions <ClusterMembershipOptions> clusterMembershipOptions,
IMembershipTable membershipTable,
IFatalErrorHandler fatalErrorHandler,
IMembershipGossiper gossiper,
ILogger <MembershipTableManager> log,
IAsyncTimerFactory timerFactory)
{
this.localSiloDetails = localSiloDetails;
this.membershipTableProvider = membershipTable;
this.fatalErrorHandler = fatalErrorHandler;
this.gossiper = gossiper;
this.clusterMembershipOptions = clusterMembershipOptions.Value;
this.myAddress = this.localSiloDetails.SiloAddress;
this.log = log;
var backOffMax = StandardExtensions.Max(EXP_BACKOFF_STEP.Multiply(this.clusterMembershipOptions.ExpectedClusterSize), SiloMessageSender.CONNECTION_RETRY_DELAY.Multiply(2));
this.EXP_BACKOFF_CONTENTION_MAX = backOffMax;
this.EXP_BACKOFF_ERROR_MAX = backOffMax;
this.snapshot = new MembershipTableSnapshot(
this.CreateLocalSiloEntry(this.CurrentStatus),
MembershipVersion.MinValue,
ImmutableDictionary <SiloAddress, MembershipEntry> .Empty);
this.updates = new AsyncEnumerable <MembershipTableSnapshot>(
(previous, proposed) => proposed.Version > previous.Version,
this.snapshot)
{
OnPublished = update => Interlocked.Exchange(ref this.snapshot, update)
};
this.membershipUpdateTimer = timerFactory.Create(
this.clusterMembershipOptions.TableRefreshTimeout,
nameof(PeriodicallyRefreshMembershipTable));
}
public async Task MembershipChangeNotification(MembershipTableSnapshot snapshot)
{
await this.membershipTableManager.RefreshFromSnapshot(snapshot);
}