internal VirtualBucketsRingProvider(SiloAddress siloAddr, int nBucketsPerSilo)
{
if (nBucketsPerSilo <= 0 )
throw new IndexOutOfRangeException("numBucketsPerSilo is out of the range. numBucketsPerSilo = " + nBucketsPerSilo);
logger = TraceLogger.GetLogger(typeof(VirtualBucketsRingProvider).Name);
statusListeners = new List<IRingRangeListener>();
bucketsMap = new SortedDictionary<uint, SiloAddress>();
sortedBucketsList = new List<Tuple<uint, SiloAddress>>();
myAddress = siloAddr;
numBucketsPerSilo = nBucketsPerSilo;
lockable = new object();
running = true;
myRange = RangeFactory.CreateFullRange();
logger.Info("Starting {0} on silo {1}.", typeof(VirtualBucketsRingProvider).Name, siloAddr.ToStringWithHashCode());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RING, ToString);
IntValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RINGSIZE, () => GetRingSize());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGDISTANCE, () => String.Format("x{0,8:X8}", ((IRingRangeInternal)myRange).RangeSize()));
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGPERCENTAGE, () => (float)((IRingRangeInternal)myRange).RangePercentage());
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_AVERAGERINGPERCENTAGE, () =>
{
int size = GetRingSize();
return size == 0 ? 0 : ((float)100.0/(float) size);
});
// add myself to the list of members
AddServer(myAddress);
}
internal void AddServer(SiloAddress silo)
{
lock (membershipRingList)
{
if (membershipRingList.Contains(silo))
{
return; // we already have this silo
}
int myOldIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (!(membershipRingList.Count == 0 || myOldIndex != -1))
{
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, Utils.EnumerableToString(membershipRingList)));
}
// insert new silo in the sorted order
int hash = silo.GetConsistentHashCode();
// Find the last silo with hash smaller than the new silo, and insert the latter after (this is why we have +1 here) the former.
// Notice that FindLastIndex might return -1 if this should be the first silo in the list, but then
// 'index' will get 0, as needed.
int index = membershipRingList.FindLastIndex(siloAddr => siloAddr.GetConsistentHashCode() < hash) + 1;
membershipRingList.Insert(index, silo);
// relating to triggering handler ... new node took over some of my responsibility
if (index == myOldIndex || // new node was inserted in my place
(myOldIndex == 0 && index == membershipRingList.Count - 1)) // I am the first node, and the new server is the last node
{
IRingRange oldRange = MyRange;
try
{
MyRange = RangeFactory.CreateRange(unchecked ((uint)hash), unchecked ((uint)myKey));
}
catch (OverflowException exc)
{
log.Error(ErrorCode.ConsistentRingProviderBase + 5,
String.Format("OverflowException: hash as int= x{0, 8:X8}, hash as uint= x{1, 8:X8}, myKey as int x{2, 8:X8}, myKey as uint x{3, 8:X8}.",
hash, (uint)hash, myKey, (uint)myKey), exc);
}
NotifyLocalRangeSubscribers(oldRange, MyRange, false);
}
log.Info("Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
}
}
public override string ToString()
{
lock (membershipRingList)
{
if (membershipRingList.Count == 1)
{
return(Utils.EnumerableToString(membershipRingList, silo => String.Format("{0} -> {1}", silo.ToStringWithHashCode(), RangeFactory.CreateFullRange())));
}
var sb = new StringBuilder("[");
for (int i = 0; i < membershipRingList.Count; i++)
{
SiloAddress curr = membershipRingList[i];
SiloAddress next = membershipRingList[(i + 1) % membershipRingList.Count];
IRingRange range = RangeFactory.CreateRange(unchecked ((uint)curr.GetConsistentHashCode()), unchecked ((uint)next.GetConsistentHashCode()));
sb.Append(String.Format("{0} -> {1}, ", curr.ToStringWithHashCode(), range));
}
sb.Append("]");
return(sb.ToString());
}
}
protected void RemoveServer(SiloAddress silo, SiloStatus status)
{
lock (membershipCache)
{
if (!membershipCache.Contains(silo))
{
// we have already removed this silo
return;
}
if (CatalogSiloStatusListener != null)
{
try
{
// only call SiloStatusChangeNotification once on the catalog and the order is important: call BEFORE updating membershipRingList.
CatalogSiloStatusListener.SiloStatusChangeNotification(silo, status);
}
catch (Exception exc)
{
log.Error(ErrorCode.Directory_SiloStatusChangeNotification_Exception,
String.Format("CatalogSiloStatusListener.SiloStatusChangeNotification has thrown an exception when notified about removed silo {0}.", silo.ToStringWithHashCode()), exc);
}
}
// the call order is important
HandoffManager.ProcessSiloRemoveEvent(silo);
membershipCache.Remove(silo);
membershipRingList.Remove(silo);
AdjustLocalDirectory(silo);
AdjustLocalCache(silo);
if (log.IsVerbose)
{
log.Verbose("Silo {0} removed silo {1}", MyAddress, silo);
}
}
}
public void SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status)
{
// ignore joining events and also events on myself.
if (updatedSilo.Equals(LocalSilo)) return;
// We deactivate those activations when silo goes either of ShuttingDown/Stopping/Dead states,
// since this is what Directory is doing as well. Directory removes a silo based on all those 3 statuses,
// thus it will only deliver a "remove" notification for a given silo once to us. Therefore, we need to react the fist time we are notified.
// We may review the directory behaiviour in the future and treat ShuttingDown differently ("drain only") and then this code will have to change a well.
if (!status.IsTerminating()) return;
var activationsToShutdown = new List<ActivationData>();
try
{
// scan all activations in activation directory and deactivate the ones that the removed silo is their primary partition owner.
lock (activations)
{
foreach (var activation in activations)
{
try
{
var activationData = activation.Value;
if (!directory.GetPrimaryForGrain(activationData.Grain).Equals(updatedSilo)) continue;
lock (activationData)
{
// adapted from InsideGarinClient.DeactivateOnIdle().
activationData.ResetKeepAliveRequest();
activationsToShutdown.Add(activationData);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.Catalog_SiloStatusChangeNotification_Exception,
String.Format("Catalog has thrown an exception while executing SiloStatusChangeNotification of silo {0}.", updatedSilo.ToStringWithHashCode()), exc);
}
}
}
logger.Info(ErrorCode.Catalog_SiloStatusChangeNotification,
String.Format("Catalog is deactivating {0} activations due to a failure of silo {1}, since it is a primary directory partiton to these grain ids.",
activationsToShutdown.Count, updatedSilo.ToStringWithHashCode()));
}
finally
{
// outside the lock.
if (activationsToShutdown.Count > 0)
{
DeactivateActivations(activationsToShutdown).Ignore();
}
}
}
internal VirtualBucketsRingProvider(SiloAddress siloAddress, ILoggerFactory loggerFactory, int numVirtualBuckets)
{
numBucketsPerSilo = numVirtualBuckets;
if (numBucketsPerSilo <= 0)
{
throw new IndexOutOfRangeException("numBucketsPerSilo is out of the range. numBucketsPerSilo = " + numBucketsPerSilo);
}
logger = loggerFactory.CreateLogger <VirtualBucketsRingProvider>();
statusListeners = new List <IRingRangeListener>();
bucketsMap = new SortedDictionary <uint, SiloAddress>();
sortedBucketsList = new List <Tuple <uint, SiloAddress> >();
myAddress = siloAddress;
lockable = new object();
running = true;
myRange = RangeFactory.CreateFullRange();
logger.Info("Starting {0} on silo {1}.", typeof(VirtualBucketsRingProvider).Name, siloAddress.ToStringWithHashCode());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RING, ToString);
IntValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RINGSIZE, () => GetRingSize());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGDISTANCE, () => String.Format("x{0,8:X8}", ((IRingRangeInternal)myRange).RangeSize()));
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGPERCENTAGE, () => (float)((IRingRangeInternal)myRange).RangePercentage());
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_AVERAGERINGPERCENTAGE, () =>
{
int size = GetRingSize();
return(size == 0 ? 0 : ((float)100.0 / (float)size));
});
// add myself to the list of members
AddServer(myAddress);
}
internal void RemoveServer(SiloAddress silo)
{
lock (lockable)
{
if (!bucketsMap.ContainsValue(silo))
{
return; // we have already removed this silo
}
List <uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
bucketsMap.Remove(hash);
}
var myOldRange = this.myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple <uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Removed_Silo, "Removed Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
private void AddServer(SiloAddress silo)
{
lock (lockable)
{
List <uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
if (bucketsMap.TryGetValue(hash, out var other))
{
// If two silos conflict, take the lesser of the two (usually the older one; that is, the lower epoch)
if (silo.CompareTo(other) > 0)
{
continue;
}
}
bucketsMap[hash] = silo;
}
var myOldRange = myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple <uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Added_Silo, "Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
protected void RemoveServer(SiloAddress silo, SiloStatus status)
{
lock (membershipCache)
{
if (!membershipCache.Contains(silo))
{
// we have already removed this silo
return;
}
if (CatalogSiloStatusListener != null)
{
try
{
// only call SiloStatusChangeNotification once on the catalog and the order is important: call BEFORE updating membershipRingList.
CatalogSiloStatusListener.SiloStatusChangeNotification(silo, status);
}
catch (Exception exc)
{
log.Error(ErrorCode.Directory_SiloStatusChangeNotification_Exception,
String.Format("CatalogSiloStatusListener.SiloStatusChangeNotification has thrown an exception when notified about removed silo {0}.", silo.ToStringWithHashCode()), exc);
}
}
// the call order is important
HandoffManager.ProcessSiloRemoveEvent(silo);
membershipCache.Remove(silo);
membershipRingList.Remove(silo);
AdjustLocalDirectory(silo);
AdjustLocalCache(silo);
if (log.IsVerbose) log.Verbose("Silo {0} removed silo {1}", MyAddress, silo);
}
}
internal void AddServer(SiloAddress silo)
{
lock (membershipRingList)
{
if (membershipRingList.Contains(silo)) return; // we already have this silo
int myOldIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (!(membershipRingList.Count == 0 || myOldIndex != -1))
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, Utils.EnumerableToString(membershipRingList)));
// insert new silo in the sorted order
int hash = silo.GetConsistentHashCode();
// Find the last silo with hash smaller than the new silo, and insert the latter after (this is why we have +1 here) the former.
// Notice that FindLastIndex might return -1 if this should be the first silo in the list, but then
// 'index' will get 0, as needed.
int index = membershipRingList.FindLastIndex(siloAddr => siloAddr.GetConsistentHashCode() < hash) + 1;
membershipRingList.Insert(index, silo);
// relating to triggering handler ... new node took over some of my responsibility
if (index == myOldIndex || // new node was inserted in my place
(myOldIndex == 0 && index == membershipRingList.Count - 1)) // I am the first node, and the new server is the last node
{
IRingRange oldRange = MyRange;
try
{
MyRange = RangeFactory.CreateRange(unchecked((uint)hash), unchecked((uint)myKey));
}
catch (OverflowException exc)
{
log.Error(ErrorCode.ConsistentRingProviderBase + 5,
String.Format("OverflowException: hash as int= x{0, 8:X8}, hash as uint= x{1, 8:X8}, myKey as int x{2, 8:X8}, myKey as uint x{3, 8:X8}.",
hash, (uint)hash, myKey, (uint)myKey), exc);
}
NotifyLocalRangeSubscribers(oldRange, MyRange, false);
}
log.Info("Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
}
}
protected void RemoveServer(SiloAddress silo, SiloStatus status)
{
lock (this.writeLock)
{
try
{
// Only notify the catalog once. Order is important: call BEFORE updating membershipRingList.
this.catalogOnSiloRemoved?.Invoke(silo, status);
}
catch (Exception exc)
{
log.Error(ErrorCode.Directory_SiloStatusChangeNotification_Exception,
String.Format("CatalogSiloStatusListener.SiloStatusChangeNotification has thrown an exception when notified about removed silo {0}.", silo.ToStringWithHashCode()), exc);
}
var existing = this.directoryMembership;
if (!existing.MembershipCache.Contains(silo))
{
// we have already removed this silo
return;
}
// the call order is important
HandoffManager.ProcessSiloRemoveEvent(silo);
this.directoryMembership = new DirectoryMembership(
existing.MembershipRingList.Remove(silo),
existing.MembershipCache.Remove(silo));
AdjustLocalDirectory(silo, dead: true);
AdjustLocalCache(silo, dead: true);
if (log.IsEnabled(LogLevel.Debug))
{
log.Debug("Silo {0} removed silo {1}", MyAddress, silo);
}
}
}
internal void RemoveServer(SiloAddress silo)
{
lock (lockable)
{
if (!bucketsMap.ContainsValue(silo)) return; // we have already removed this silo
List<uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
bucketsMap.Remove(hash);
}
var myOldRange = this.myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple<uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Removed_Silo, "Removed Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
private void AddServer(SiloAddress silo)
{
lock (lockable)
{
List<uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
if (bucketsMap.ContainsKey(hash))
{
var other = bucketsMap[hash];
// If two silos conflict, take the lesser of the two (usually the older one; that is, the lower epoch)
if (silo.CompareTo(other) > 0) continue;
}
bucketsMap[hash] = silo;
}
var myOldRange = myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple<uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Added_Silo, "Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
