public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
WorkItemGroup.QueueAction(() =>
{
Utils.SafeExecute(() => this.OnSiloStatusChange(updatedSilo, status), this.logger);
});
}
internal void UpdateMyStatusLocal(SiloStatus status)
{
if (CurrentStatus == status)
{
return;
}
// make copies
var tmpLocalTableCopy = GetSiloStatuses(st => true, true); // all the silos including me.
var tmpLocalTableCopyOnlyActive = GetSiloStatuses(st => st == SiloStatus.Active, true); // only active silos including me.
var tmpLocalTableNamesCopy = localTable.ToDictionary(pair => pair.Key, pair => pair.Value.SiloName); // all the silos excluding me.
CurrentStatus = status;
tmpLocalTableCopy[MyAddress] = status;
if (status == SiloStatus.Active)
{
tmpLocalTableCopyOnlyActive[MyAddress] = status;
}
else if (tmpLocalTableCopyOnlyActive.ContainsKey(MyAddress))
{
tmpLocalTableCopyOnlyActive.Remove(MyAddress);
}
localTableCopy = tmpLocalTableCopy;
localTableCopyOnlyActive = tmpLocalTableCopyOnlyActive;
localNamesTableCopy = tmpLocalTableNamesCopy;
if (this.multiClusterActive)
{
localMultiClusterGatewaysCopy = DetermineMultiClusterGateways();
}
NotifyLocalSubscribers(MyAddress, CurrentStatus);
}
private async Task GossipToRemoteSilo(
SiloAddress silo,
MembershipTableSnapshot snapshot,
SiloAddress updatedSilo,
SiloStatus updatedStatus)
{
if (this.log.IsEnabled(LogLevel.Trace))
{
this.log.LogTrace(
"-Sending status update GOSSIP notification about silo {UpdatedSilo}, status {UpdatedStatus}, to silo {RemoteSilo}",
updatedSilo,
updatedStatus,
silo);
}
try
{
var remoteOracle = this.grainFactory.GetSystemTarget <IMembershipService>(Constants.MembershipOracleId, silo);
try
{
await remoteOracle.MembershipChangeNotification(snapshot);
}
catch (NotImplementedException)
{
// Fallback to "old" gossip
await remoteOracle.SiloStatusChangeNotification(updatedSilo, updatedStatus);
}
}
catch (Exception exception)
{
this.log.LogError(
(int)ErrorCode.MembershipGossipSendFailure,
"Exception while sending gossip notification to remote silo {Silo}: {Exception}", silo, exception);
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status == SiloStatus.Dead)
{
_ = Task.Run(() => this.AbortConnectionAsync(updatedSilo));
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (Equals(updatedSilo, MyAddress))
{
if (status == SiloStatus.Stopping || status.Equals(SiloStatus.ShuttingDown))
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(true), CacheValidator.SchedulingContext).Ignore();
}
else if (status == SiloStatus.Dead)
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(false), CacheValidator.SchedulingContext).Ignore();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => RemoveServer(updatedSilo, status), CacheValidator.SchedulingContext).Ignore();
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Starting -- wait until it actually becomes active
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => AddServer(updatedSilo), CacheValidator.SchedulingContext).Ignore();
}
}
}
private void NotifyLocalSubscribers(SiloAddress siloAddress, SiloStatus newStatus)
{
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("-NotifyLocalSubscribers about {0} status {1}", siloAddress.ToLongString(), newStatus);
}
List <ISiloStatusListener> copy;
lock (statusListeners)
{
copy = statusListeners.ToList();
}
foreach (ISiloStatusListener listener in copy)
{
try
{
listener.SiloStatusChangeNotification(siloAddress, newStatus);
}
catch (Exception exc)
{
logger.Error(ErrorCode.MembershipLocalSubscriberException,
String.Format("Local ISiloStatusListener {0} has thrown an exception when was notified about SiloStatusChangeNotification about silo {1} new status {2}",
listener.GetType().FullName, siloAddress.ToLongString(), newStatus), exc);
}
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status == SiloStatus.Dead && updatedSilo != siloStatusOracle.SiloAddress)
{
_ = Task.Run(() => this.CloseConnectionAsync(updatedSilo));
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status == SiloStatus.Dead)
{
this.log.LogInformation("Aborting connections to defunct silo {SiloAddress}", updatedSilo);
this.connectionManager.Abort(updatedSilo);
}
}
private void KillMyselfLocally(string reason)
{
var msg = "I have been told I am dead, so this silo will stop! " + reason;
log.Error(ErrorCode.MembershipKillMyselfLocally, msg);
this.CurrentStatus = SiloStatus.Dead;
this.fatalErrorHandler.OnFatalException(this, msg, null);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status != SiloStatus.Dead)
{
return;
}
scheduler.QueueTask(() => OnClientRefreshTimer(null), SchedulingContext).Ignore();
}
public async Task SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (this.log.IsEnabled(LogLevel.Trace))
{
this.log.LogTrace("-Received GOSSIP SiloStatusChangeNotification about {Silo} status {Status}. Going to read the table.", updatedSilo, status);
}
await ReadTable();
}
public Task GossipToRemoteSilos(
SiloAddress updatedSilo,
SiloStatus updatedStatus,
List <SiloAddress> gossipPartners)
{
var systemTarget = this.serviceProvider.GetRequiredService <MembershipSystemTarget>();
return(systemTarget.GossipToRemoteSilos(updatedSilo, updatedStatus, gossipPartners));
}
// read the table
// find all currently active nodes and test pings to all of them
// try to ping all
// if all pings succeeded
// try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags
// if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes
private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.LogDebug(
"TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}",
(newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"),
table.ToString());
}
LogMissedIAmAlives(table);
var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus);
if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus)
{
this.log.LogWarning(
(int)ErrorCode.MembershipFoundMyselfDead1,
"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.",
myEntry.ToFullString(full: true));
this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}.");
return(true);
}
var now = GetDateTimeUtcNow();
if (newStatus == SiloStatus.Dead)
{
myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on.
}
myEntry.Status = newStatus;
myEntry.IAmAliveTime = now;
bool ok;
TableVersion next = table.Version.Next();
if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update.
{
ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next);
}
else
{
ok = await membershipTableProvider.InsertRow(myEntry, next);
}
if (ok)
{
this.CurrentStatus = newStatus;
var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e);
entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag);
var updatedTable = new MembershipTableData(entries.Values.ToList(), next);
this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce));
}
return(ok);
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status.Equals(SiloStatus.Dead))
{
// just clean up garbage from immatureSilos.
bool ignore;
immatureSilos.TryRemove(updatedSilo, out ignore);
}
SiloStatusChangeNotification().Ignore();
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
this.Notifications.Add(Tuple.Create(updatedSilo, status));
this.Silos[updatedSilo] = status;
this.Version++;
if (this.Version >= this.WaitForVersion)
{
this.VersionReached.Set();
}
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status == SiloStatus.Dead)
{
// just clean up garbage from immatureSilos.
bool ignore;
immatureSilos.TryRemove(updatedSilo, out ignore);
}
SiloStatusChangeNotification().Ignore();
}
private void AssertStatus(SiloAddress address, SiloStatus expected)
{
var localStatus = this.oracle.GetApproximateSiloStatus(address);
Assert.Equal(expected, localStatus);
if (address.Equals(this.siloDetails.SiloAddress))
{
Assert.Equal(localStatus, this.oracle.CurrentStatus);
}
Assert.Equal(!address.Equals(this.siloDetails.SiloAddress) && expected == SiloStatus.Dead, this.oracle.IsDeadSilo(address));
Assert.Equal(address.Equals(this.siloDetails.SiloAddress) || !expected.IsTerminating(), this.oracle.IsFunctionalDirectory(address));
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (!status.IsTerminating())
{
return;
}
SiloRuntimeStatistics ignore;
periodicStats.TryRemove(updatedSilo, out ignore);
NotifyAllStatisticsChangeEventsSubscribers(updatedSilo, null);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (!status.Equals(SiloStatus.Stopping) && !status.Equals(SiloStatus.ShuttingDown) &&
!status.Equals(SiloStatus.Dead))
{
return;
}
SiloRuntimeStatistics ignore;
periodicStats.TryRemove(updatedSilo, out ignore);
NotifyAllStatisticsChangeEventsSubscribers(updatedSilo, null);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status.IsTerminating() && updatedSilo.Equals(this.Silo))
{
refreshTimer?.Dispose();
}
else if (status != SiloStatus.Dead)
{
return;
}
scheduler.QueueTask(() => OnClientRefreshTimer(null), SchedulingContext).Ignore();
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (this.Silos.TryGetValue(updatedSilo, out var existingStatus))
{
if (existingStatus == status)
{
throw new InvalidOperationException($"Silo {updatedSilo} already has status {status}.");
}
}
this.Silos[updatedSilo] = status;
this.Notifications.Add(Tuple.Create(updatedSilo, status));
this.versionUpdated.Set();
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
hasToRefreshClusterGrainInterfaceMap = true;
if (status == SiloStatus.Active && !updatedSilo.Equals(this.Silo))
{
if (this.logger.IsEnabled(LogLevel.Information))
{
this.logger.LogInformation("Expediting cluster type map refresh due to new silo, {SiloAddress}", updatedSilo);
}
this.scheduler.QueueTask(() => this.OnRefreshClusterMapTimer(null), this);
}
}
private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status)
{
if (!status.IsTerminating())
{
return;
}
if (Equals(updatedSilo, this.Silo))
{
this.publishTimer.Dispose();
}
periodicStats.TryRemove(updatedSilo, out _);
NotifyAllStatisticsChangeEventsSubscribers(updatedSilo, null);
}
public Task GossipToRemoteSilos(SiloAddress updatedSilo, SiloStatus updatedStatus, List <SiloAddress> gossipPartners)
{
async Task Gossip()
{
var tasks = new List <Task>(gossipPartners.Count);
foreach (var silo in gossipPartners)
{
tasks.Add(this.GossipToRemoteSilo(silo, updatedSilo, updatedStatus));
}
await Task.WhenAll(tasks);
}
return(this.ScheduleTask(Gossip));
}
private void NotifySubscribers(SiloAddress address, SiloStatus newStatus, List <ISiloStatusListener> listeners)
{
foreach (var subscriber in listeners)
{
try
{
subscriber.SiloStatusChangeNotification(address, newStatus);
}
catch (Exception exception)
{
this.log.Warn(
(int)ErrorCode.ServiceFabric_MembershipOracle_ExceptionNotifyingSubscribers,
"Exception notifying subscriber.",
exception);
}
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (!Equals(updatedSilo, MyAddress)) // Status change for some other silo
{
if (status.IsTerminating())
{
// QueueAction up the "Remove" to run on a system turn
CacheValidator.WorkItemGroup.QueueAction(() => RemoveServer(updatedSilo, status));
}
else if (status == SiloStatus.Active) // do not do anything with SiloStatus.Starting -- wait until it actually becomes active
{
// QueueAction up the "Remove" to run on a system turn
CacheValidator.WorkItemGroup.QueueAction(() => AddServer(updatedSilo));
}
}
}
private async Task GossipToOthers(SiloAddress updatedSilo, SiloStatus updatedStatus)
{
if (!this.clusterMembershipOptions.UseLivenessGossip)
{
return;
}
var now = DateTime.UtcNow;
var gossipPartners = new List <SiloAddress>();
foreach (var item in this.MembershipTableSnapshot.Entries)
{
var entry = item.Value;
if (entry.SiloAddress.IsSameLogicalSilo(this.myAddress))
{
continue;
}
if (!IsFunctionalForMembership(entry.Status))
{
continue;
}
if (entry.HasMissedIAmAlivesSince(this.clusterMembershipOptions, now) != default)
{
continue;
}
gossipPartners.Add(entry.SiloAddress);
bool IsFunctionalForMembership(SiloStatus status)
{
return(status == SiloStatus.Active || status == SiloStatus.ShuttingDown || status == SiloStatus.Stopping);
}
}
try
{
await this.gossiper.GossipToRemoteSilos(updatedSilo, updatedStatus, gossipPartners);
}
catch (Exception exception)
{
this.log.LogWarning("Exception while gossiping status to other silos: {Exception}", exception);
}
}
public async Task SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (this.log.IsEnabled(LogLevel.Trace))
{
this.log.LogTrace("-Received GOSSIP SiloStatusChangeNotification about {Silo} status {Status}. Going to read the table.", updatedSilo, status);
}
try
{
await this.membershipTableManager.Refresh();
}
catch (Exception exception)
{
this.log.LogError(
(int)ErrorCode.MembershipGossipProcessingFailure,
"Error refreshing membership table: {Exception}",
exception);
}
}
public Task GossipToRemoteSilos(
SiloAddress updatedSilo,
SiloStatus updatedStatus,
List <SiloAddress> gossipPartners)
{
if (gossipPartners.Count == 0)
{
return(Task.CompletedTask);
}
this.log.LogInformation(
"Gossiping {Silo} status change to {Status} to {NumPartners} partners",
updatedSilo,
updatedStatus,
gossipPartners.Count);
var systemTarget = this.serviceProvider.GetRequiredService <MembershipSystemTarget>();
return(systemTarget.GossipToRemoteSilos(updatedSilo, updatedStatus, gossipPartners));
}
private MembershipEntry CreateLocalSiloEntry(SiloStatus currentStatus)
{
return(new MembershipEntry
{
SiloAddress = this.localSiloDetails.SiloAddress,
HostName = this.localSiloDetails.DnsHostName,
SiloName = this.localSiloDetails.Name,
Status = currentStatus,
ProxyPort = this.localSiloDetails.GatewayAddress?.Endpoint?.Port ?? 0,
RoleName = (Assembly.GetEntryAssembly() ?? typeof(MembershipTableManager).Assembly).GetName().Name,
SuspectTimes = new List <Tuple <SiloAddress, DateTime> >(),
StartTime = this.siloStartTime,
IAmAliveTime = DateTime.UtcNow
});
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Update list of silos if silos have been added or removed from the deployment
/// - Update the list of active silos
/// - Notify listeners if necessary
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
bool changed = false;
var newSiloNames = deploymentConfig.GetAllSiloInstanceNames();
lock (activeSiloNames)
{
// if silo names has changed, deployment has been changed so we need to update silo names
changed |= UpdateAllSiloNames(newSiloNames);
// if a silo status has changed, update list of active silos
changed |= UpdateActiveSilos(updatedSilo, status);
}
// if no change, don't notify
if (changed)
{
NotifyListeners().Ignore();
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (updatedSilo.Equals(myAddress))
{
if (status.IsTerminating())
{
Stop();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
RemoveServer(updatedSilo);
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Created or SiloStatus.Joining -- wait until it actually becomes active
{
AddServer(updatedSilo);
}
}
}
internal void UpdateMyStatusLocal(SiloStatus status)
{
if (CurrentStatus == status) return;
// make copies
var tmpLocalTableCopy = GetSiloStatuses(st => true, true); // all the silos including me.
var tmpLocalTableCopyOnlyActive = GetSiloStatuses(st => st.Equals(SiloStatus.Active), true); // only active silos including me.
var tmpLocalTableNamesCopy = localTable.ToDictionary(pair => pair.Key, pair => pair.Value.InstanceName); // all the silos excluding me.
CurrentStatus = status;
tmpLocalTableCopy[MyAddress] = status;
if (status.Equals(SiloStatus.Active))
{
tmpLocalTableCopyOnlyActive[MyAddress] = status;
}
else if (tmpLocalTableCopyOnlyActive.ContainsKey(MyAddress))
{
tmpLocalTableCopyOnlyActive.Remove(MyAddress);
}
localTableCopy = tmpLocalTableCopy;
localTableCopyOnlyActive = tmpLocalTableCopyOnlyActive;
localNamesTableCopy = tmpLocalTableNamesCopy;
NotifyLocalSubscribers(MyAddress, CurrentStatus);
}
internal MembershipEntry CreateNewMembershipEntry(NodeConfiguration nodeConf, SiloStatus myStatus)
{
return CreateNewMembershipEntry(nodeConf, MyAddress, MyHostname, myStatus, SiloStartTime);
}
private static MembershipEntry CreateNewMembershipEntry(NodeConfiguration nodeConf, SiloAddress myAddress, string myHostname, SiloStatus myStatus, DateTime startTime)
{
var assy = Assembly.GetEntryAssembly() ?? typeof(MembershipOracleData).GetTypeInfo().Assembly;
var roleName = assy.GetName().Name;
var entry = new MembershipEntry
{
SiloAddress = myAddress,
HostName = myHostname,
InstanceName = nodeConf.SiloName,
Status = myStatus,
ProxyPort = (nodeConf.IsGatewayNode ? nodeConf.ProxyGatewayEndpoint.Port : 0),
RoleName = roleName,
SuspectTimes = new List<Tuple<SiloAddress, DateTime>>(),
StartTime = startTime,
IAmAliveTime = DateTime.UtcNow
};
return entry;
}
private void NotifyLocalSubscribers(SiloAddress siloAddress, SiloStatus newStatus)
{
if (logger.IsVerbose2) logger.Verbose2("-NotifyLocalSubscribers about {0} status {1}", siloAddress.ToLongString(), newStatus);
List<ISiloStatusListener> copy;
lock (statusListeners)
{
copy = statusListeners.ToList();
}
foreach (ISiloStatusListener listener in copy)
{
try
{
listener.SiloStatusChangeNotification(siloAddress, newStatus);
}
catch (Exception exc)
{
logger.Error(ErrorCode.MembershipLocalSubscriberException,
String.Format("Local ISiloStatusListener {0} has thrown an exception when was notified about SiloStatusChangeNotification about silo {1} new status {2}",
listener.GetType().FullName, siloAddress.ToLongString(), newStatus), exc);
}
}
}
protected void RemoveServer(SiloAddress silo, SiloStatus status)
{
lock (membershipCache)
{
if (!membershipCache.Contains(silo))
{
// we have already removed this silo
return;
}
if (CatalogSiloStatusListener != null)
{
try
{
// only call SiloStatusChangeNotification once on the catalog and the order is important: call BEFORE updating membershipRingList.
CatalogSiloStatusListener.SiloStatusChangeNotification(silo, status);
}
catch (Exception exc)
{
log.Error(ErrorCode.Directory_SiloStatusChangeNotification_Exception,
String.Format("CatalogSiloStatusListener.SiloStatusChangeNotification has thrown an exception when notified about removed silo {0}.", silo.ToStringWithHashCode()), exc);
}
}
// the call order is important
HandoffManager.ProcessSiloRemoveEvent(silo);
membershipCache.Remove(silo);
membershipRingList.Remove(silo);
AdjustLocalDirectory(silo);
AdjustLocalCache(silo);
if (log.IsVerbose) log.Verbose("Silo {0} removed silo {1}", MyAddress, silo);
}
}
private static IDbDataParameter CreateStatusParameter(IDbCommand command, SiloStatus status, ParameterDirection direction)
{
var parameter = command.CreateParameter();
parameter.ParameterName = "status";
parameter.Value = (int)status;
parameter.DbType = DbType.Int32;
parameter.Direction = direction;
return parameter;
}
private static IDbDataParameter CreateStatusParameter(IDbCommand command, SiloStatus status)
{
return command.CreateParameter(ParameterDirection.Input, "status", (int)status);
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Update list of silos if silos have been added or removed from the deployment
/// - Update the list of active silos
/// - Notify listeners if necessary
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
bool changed = false;
var newSiloNames = deploymentConfig.GetAllSiloInstanceNames();
lock (activeSiloNames)
{
// if silo names has changed, deployment has been changed so we need to update silo names
changed |= UpdateAllSiloNames(newSiloNames);
// if a silo status has changed, update list of active silos
changed |= UpdateActiveSilos(updatedSilo, status);
}
// if no change, don't notify
if (changed)
{
NotifyListeners().Ignore();
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (!status.IsTerminating()) return;
SiloRuntimeStatistics ignore;
periodicStats.TryRemove(updatedSilo, out ignore);
NotifyAllStatisticsChangeEventsSubscribers(updatedSilo, null);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// ignore joining events and also events on myself.
if (updatedSilo.Equals(LocalSilo)) return;
// We deactivate those activations when silo goes either of ShuttingDown/Stopping/Dead states,
// since this is what Directory is doing as well. Directory removes a silo based on all those 3 statuses,
// thus it will only deliver a "remove" notification for a given silo once to us. Therefore, we need to react the fist time we are notified.
// We may review the directory behaiviour in the future and treat ShuttingDown differently ("drain only") and then this code will have to change a well.
if (!status.IsTerminating()) return;
var activationsToShutdown = new List<ActivationData>();
try
{
// scan all activations in activation directory and deactivate the ones that the removed silo is their primary partition owner.
lock (activations)
{
foreach (var activation in activations)
{
try
{
var activationData = activation.Value;
if (!directory.GetPrimaryForGrain(activationData.Grain).Equals(updatedSilo)) continue;
lock (activationData)
{
// adapted from InsideGarinClient.DeactivateOnIdle().
activationData.ResetKeepAliveRequest();
activationsToShutdown.Add(activationData);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.Catalog_SiloStatusChangeNotification_Exception,
String.Format("Catalog has thrown an exception while executing SiloStatusChangeNotification of silo {0}.", updatedSilo.ToStringWithHashCode()), exc);
}
}
}
logger.Info(ErrorCode.Catalog_SiloStatusChangeNotification,
String.Format("Catalog is deactivating {0} activations due to a failure of silo {1}, since it is a primary directory partiton to these grain ids.",
activationsToShutdown.Count, updatedSilo.ToStringWithHashCode()));
}
finally
{
// outside the lock.
if (activationsToShutdown.Count > 0)
{
DeactivateActivations(activationsToShutdown).Ignore();
}
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
hasToRefreshClusterGrainInterfaceMap = true;
}
/// <summary>
/// Updates active silo names if necessary
/// if silo is active and name is not in active list, add it.
/// if silo is inactive and name is in active list, remove it.
/// </summary>
/// <param name="updatedSilo"></param>
/// <param name="status"></param>
/// <returns>bool, true if active silo names changed</returns>
private bool UpdateActiveSilos(SiloAddress updatedSilo, SiloStatus status)
{
bool changed = false;
string siloName;
// try to get silo name
if (siloStatusOracle.TryGetSiloName(updatedSilo, out siloName))
{
if (status.Equals(SiloStatus.Active) && // if silo state became active
!activeSiloNames.Contains(siloName)) // and silo name is not currently in active silo list
{
changed = true;
activeSiloNames.Add(siloName); // add silo to list of active silos
}
else if (!status.Equals(SiloStatus.Active) && // if silo state became not active
activeSiloNames.Contains(siloName)) // and silo name is currently in active silo list
{
changed = true;
activeSiloNames.Remove(siloName); // remove silo from list of active silos
}
}
return changed;
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
NotifyListeners().Ignore();
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (Equals(updatedSilo, MyAddress))
{
if (status == SiloStatus.Stopping || status.Equals(SiloStatus.ShuttingDown))
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(true), CacheValidator.SchedulingContext).Ignore();
}
else if (status == SiloStatus.Dead)
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(false), CacheValidator.SchedulingContext).Ignore();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => RemoveServer(updatedSilo, status), CacheValidator.SchedulingContext).Ignore();
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Starting -- wait until it actually becomes active
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => AddServer(updatedSilo), CacheValidator.SchedulingContext).Ignore();
}
}
}
