public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (Equals(updatedSilo, MyAddress))
{
if (status == SiloStatus.Stopping || status.Equals(SiloStatus.ShuttingDown))
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(true), CacheValidator.SchedulingContext).Ignore();
}
else if (status == SiloStatus.Dead)
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(false), CacheValidator.SchedulingContext).Ignore();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => RemoveServer(updatedSilo, status), CacheValidator.SchedulingContext).Ignore();
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Starting -- wait until it actually becomes active
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => AddServer(updatedSilo), CacheValidator.SchedulingContext).Ignore();
}
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (!status.Equals(SiloStatus.Stopping) && !status.Equals(SiloStatus.ShuttingDown) &&
!status.Equals(SiloStatus.Dead))
{
return;
}
SiloRuntimeStatistics ignore;
periodicStats.TryRemove(updatedSilo, out ignore);
NotifyAllStatisticsChangeEventsSubscribers(updatedSilo, null);
}
internal void UpdateMyStatusLocal(SiloStatus status)
{
if (CurrentStatus == status)
{
return;
}
// make copies
var tmpLocalTableCopy = GetSiloStatuses(st => true, true); // all the silos including me.
var tmpLocalTableCopyOnlyActive = GetSiloStatuses(st => st.Equals(SiloStatus.Active), true); // only active silos including me.
var tmpLocalTableNamesCopy = localTable.ToDictionary(pair => pair.Key, pair => pair.Value.InstanceName); // all the silos excluding me.
CurrentStatus = status;
tmpLocalTableCopy[MyAddress] = status;
if (status.Equals(SiloStatus.Active))
{
tmpLocalTableCopyOnlyActive[MyAddress] = status;
}
else if (tmpLocalTableCopyOnlyActive.ContainsKey(MyAddress))
{
tmpLocalTableCopyOnlyActive.Remove(MyAddress);
}
localTableCopy = tmpLocalTableCopy;
localTableCopyOnlyActive = tmpLocalTableCopyOnlyActive;
localNamesTableCopy = tmpLocalTableNamesCopy;
NotifyLocalSubscribers(MyAddress, CurrentStatus);
}
// read the table
// find all currently active nodes and test pings to all of them
// try to ping all
// if all pings succeeded
// try to change my status to Active and in the same write transaction update Membership version row, conditioned on both etags
// if failed (on ping or on write exception or on etag) - retry the whole AttemptToJoinActiveNodes
private async Task <bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus)
{
var table = await membershipTableProvider.ReadAll();
if (log.IsEnabled(LogLevel.Debug))
{
log.LogDebug(
"TryUpdateMyStatusGlobalOnce: Read{Selection} Membership table {Table}",
(newStatus.Equals(SiloStatus.Active) ? "All" : " my entry from"),
table.ToString());
}
LogMissedIAmAlives(table);
var(myEntry, myEtag) = this.GetOrCreateLocalSiloEntry(table, newStatus);
if (myEntry.Status == SiloStatus.Dead && myEntry.Status != newStatus)
{
this.log.LogWarning(
(int)ErrorCode.MembershipFoundMyselfDead1,
"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {Entry}.",
myEntry.ToFullString(full: true));
this.KillMyselfLocally($"I should be Dead according to membership table (in TryUpdateMyStatusGlobalOnce): Entry = {(myEntry.ToFullString(full: true))}.");
return(true);
}
var now = GetDateTimeUtcNow();
if (newStatus == SiloStatus.Dead)
{
myEntry.AddSuspector(myAddress, now); // add the killer (myself) to the suspect list, for easier diagnostics later on.
}
myEntry.Status = newStatus;
myEntry.IAmAliveTime = now;
bool ok;
TableVersion next = table.Version.Next();
if (myEtag != null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update.
{
ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next);
}
else
{
ok = await membershipTableProvider.InsertRow(myEntry, next);
}
if (ok)
{
this.CurrentStatus = newStatus;
var entries = table.Members.ToDictionary(e => e.Item1.SiloAddress, e => e);
entries[myEntry.SiloAddress] = Tuple.Create(myEntry, myEtag);
var updatedTable = new MembershipTableData(entries.Values.ToList(), next);
this.ProcessTableUpdate(updatedTable, nameof(TryUpdateMyStatusGlobalOnce));
}
return(ok);
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status.Equals(SiloStatus.Dead))
{
// just clean up garbage from immatureSilos.
bool ignore;
immatureSilos.TryRemove(updatedSilo, out ignore);
}
SiloStatusChangeNotification().Ignore();
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (updatedSilo.Equals(MyAddress))
{
if (status == SiloStatus.Dead || status.Equals(SiloStatus.ShuttingDown) || status == SiloStatus.Stopping)
{
Stop();
}
}
else // Status change for some other silo
{
if (status.Equals(SiloStatus.Dead) || status.Equals(SiloStatus.ShuttingDown) || status.Equals(SiloStatus.Stopping))
{
RemoveServer(updatedSilo);
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Created or SiloStatus.Joining -- wait until it actually becomes active
{
AddServer(updatedSilo);
}
}
}
/// <summary>
/// Updates active silo names if necessary
/// if silo is active and name is not in active list, add it.
/// if silo is inactive and name is in active list, remove it.
/// </summary>
/// <param name="updatedSilo"></param>
/// <param name="status"></param>
/// <returns>bool, true if active silo names changed</returns>
private bool UpdateActiveSilos(SiloAddress updatedSilo, SiloStatus status)
{
bool changed = false;
string siloName;
// try to get silo name
if (siloStatusOracle.TryGetSiloName(updatedSilo, out siloName))
{
if (status.Equals(SiloStatus.Active) && // if silo state became active
!activeSiloNames.Contains(siloName)) // and silo name is not currently in active silo list
{
changed = true;
activeSiloNames.Add(siloName); // add silo to list of active silos
}
else if (!status.Equals(SiloStatus.Active) && // if silo state became not active
activeSiloNames.Contains(siloName)) // and silo name is currently in active silo list
{
changed = true;
activeSiloNames.Remove(siloName); // remove silo from list of active silos
}
}
return(changed);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (Equals(updatedSilo, MyAddress))
{
if (status == SiloStatus.Stopping || status.Equals(SiloStatus.ShuttingDown))
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(true), CacheValidator.SchedulingContext).Ignore();
}
else if (status == SiloStatus.Dead)
{
// QueueAction up the "Stop" to run on a system turn
Scheduler.QueueAction(() => Stop(false), CacheValidator.SchedulingContext).Ignore();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => RemoveServer(updatedSilo, status), CacheValidator.SchedulingContext).Ignore();
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Starting -- wait until it actually becomes active
{
// QueueAction up the "Remove" to run on a system turn
Scheduler.QueueAction(() => AddServer(updatedSilo), CacheValidator.SchedulingContext).Ignore();
}
}
}
/// <summary>
/// Return true if this silo is currently terminating: ShuttingDown, Stopping or Dead.
/// </summary>
public static bool IsTerminating(this SiloStatus siloStatus)
{
return(siloStatus.Equals(SiloStatus.ShuttingDown) || siloStatus.Equals(SiloStatus.Stopping) || siloStatus.Equals(SiloStatus.Dead));
}
/// <summary>
/// Called when the status of a silo in the cluster changes.
/// - Notify listeners
/// </summary>
/// <param name="updatedSilo">Silo which status has changed</param>
/// <param name="status">new silo status</param>
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
if (status.Equals(SiloStatus.Dead))
{
// just clean up garbage from immatureSilos.
bool ignore;
immatureSilos.TryRemove(updatedSilo, out ignore);
}
SiloStatusChangeNotification().Ignore();
}
/// <summary>
/// Updates active silo names if necessary
/// if silo is active and name is not in active list, add it.
/// if silo is inactive and name is in active list, remove it.
/// </summary>
/// <param name="updatedSilo"></param>
/// <param name="status"></param>
/// <returns>bool, true if active silo names changed</returns>
private bool UpdateActiveSilos(SiloAddress updatedSilo, SiloStatus status)
{
bool changed = false;
string siloName;
// try to get silo name
if (siloStatusOracle.TryGetSiloName(updatedSilo, out siloName))
{
if (status.Equals(SiloStatus.Active) && // if silo state became active
!activeSiloNames.Contains(siloName)) // and silo name is not currently in active silo list
{
changed = true;
activeSiloNames.Add(siloName); // add silo to list of active silos
}
else if (!status.Equals(SiloStatus.Active) && // if silo state became not active
activeSiloNames.Contains(siloName)) // and silo name is currently in active silo list
{
changed = true;
activeSiloNames.Remove(siloName); // remove silo from list of active silos
}
}
return changed;
}
internal bool IsFunctional(SiloStatus status)
{
return(status.Equals(SiloStatus.Active) || status.Equals(SiloStatus.ShuttingDown) || status.Equals(SiloStatus.Stopping));
}
internal void UpdateMyStatusLocal(SiloStatus status)
{
if (CurrentStatus == status) return;
// make copies
var tmpLocalTableCopy = GetSiloStatuses(st => true, true); // all the silos including me.
var tmpLocalTableCopyOnlyActive = GetSiloStatuses(st => st.Equals(SiloStatus.Active), true); // only active silos including me.
var tmpLocalTableNamesCopy = localTable.ToDictionary(pair => pair.Key, pair => pair.Value.InstanceName); // all the silos excluding me.
CurrentStatus = status;
tmpLocalTableCopy[MyAddress] = status;
if (status.Equals(SiloStatus.Active))
{
tmpLocalTableCopyOnlyActive[MyAddress] = status;
}
else if (tmpLocalTableCopyOnlyActive.ContainsKey(MyAddress))
{
tmpLocalTableCopyOnlyActive.Remove(MyAddress);
}
localTableCopy = tmpLocalTableCopy;
localTableCopyOnlyActive = tmpLocalTableCopyOnlyActive;
localNamesTableCopy = tmpLocalTableNamesCopy;
NotifyLocalSubscribers(MyAddress, CurrentStatus);
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// This silo's status has changed
if (updatedSilo.Equals(MyAddress))
{
if (status.IsTerminating())
{
Stop();
}
}
else // Status change for some other silo
{
if (status.IsTerminating())
{
RemoveServer(updatedSilo);
}
else if (status.Equals(SiloStatus.Active)) // do not do anything with SiloStatus.Created or SiloStatus.Joining -- wait until it actually becomes active
{
AddServer(updatedSilo);
}
}
}