/// <summary>
/// Attempt to retrieve reminders from the global reminder table
/// </summary>
private async Task ReadAndUpdateReminders()
{
// try to retrieve reminder from all my subranges
myRange = ring.GetMyRange();
if (logger.IsVerbose2)
{
logger.Verbose2("My range= {0}", myRange);
}
var acks = new List <Task>();
foreach (SingleRange range in RangeFactory.GetSubRanges(myRange))
{
if (logger.IsVerbose2)
{
logger.Verbose2("Reading rows for range {0}", range);
}
acks.Add(ReadTableAndStartTimers(range));
}
await Task.WhenAll(acks);
if (logger.IsVerbose3)
{
PrintReminders();
}
}
internal VirtualBucketsRingProvider(SiloAddress siloAddress, ILoggerFactory loggerFactory, int numVirtualBuckets)
{
numBucketsPerSilo = numVirtualBuckets;
if (numBucketsPerSilo <= 0)
{
throw new IndexOutOfRangeException("numBucketsPerSilo is out of the range. numBucketsPerSilo = " + numBucketsPerSilo);
}
logger = loggerFactory.CreateLogger <VirtualBucketsRingProvider>();
statusListeners = new List <IRingRangeListener>();
bucketsMap = new SortedDictionary <uint, SiloAddress>();
sortedBucketsList = new List <Tuple <uint, SiloAddress> >();
myAddress = siloAddress;
lockable = new object();
running = true;
myRange = RangeFactory.CreateFullRange();
logger.Info("Starting {0} on silo {1}.", typeof(VirtualBucketsRingProvider).Name, siloAddress.ToStringWithHashCode());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RING, ToString);
IntValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RINGSIZE, () => GetRingSize());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGDISTANCE, () => String.Format("x{0,8:X8}", ((IRingRangeInternal)myRange).RangeSize()));
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGPERCENTAGE, () => (float)((IRingRangeInternal)myRange).RangePercentage());
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_AVERAGERINGPERCENTAGE, () =>
{
int size = GetRingSize();
return(size == 0 ? 0 : ((float)100.0 / (float)size));
});
// add myself to the list of members
AddServer(myAddress);
}
/// <summary>
/// Attempt to retrieve reminders, that are my responsibility, from the global reminder table when starting this silo (reminder service instance)
/// </summary>
/// <returns></returns>
public async Task Start()
{
myRange = ring.GetMyRange();
logger.Info(ErrorCode.RS_ServiceStarting, "Starting reminder system target on: {0} x{1,8:X8}, with range {2}", Silo, Silo.GetConsistentHashCode(), myRange);
// in case reminderTable is as grain, poke the grain to activate it, before slamming it with multipel parallel requests, which may create duplicate activations.
await reminderTable.Init();
await ReadAndUpdateReminders();
logger.Info(ErrorCode.RS_ServiceStarted, "Reminder system target started OK on: {0} x{1,8:X8}, with range {2}", Silo, Silo.GetConsistentHashCode(), myRange);
status = ReminderServiceStatus.Started;
startedTask.TrySetResult(true);
var random = new SafeRandom();
var dueTime = random.NextTimeSpan(Constants.RefreshReminderList);
listRefresher = GrainTimer.FromTaskCallback(
_ => ReadAndUpdateReminders(),
null,
dueTime,
Constants.RefreshReminderList,
name: "ReminderService.ReminderListRefresher");
listRefresher.Start();
ring.SubscribeToRangeChangeEvents(this);
}
internal VirtualBucketsRingProvider(SiloAddress siloAddr, int nBucketsPerSilo)
{
if (nBucketsPerSilo <= 0 )
throw new IndexOutOfRangeException("numBucketsPerSilo is out of the range. numBucketsPerSilo = " + nBucketsPerSilo);
logger = TraceLogger.GetLogger(typeof(VirtualBucketsRingProvider).Name);
statusListeners = new List<IRingRangeListener>();
bucketsMap = new SortedDictionary<uint, SiloAddress>();
sortedBucketsList = new List<Tuple<uint, SiloAddress>>();
myAddress = siloAddr;
numBucketsPerSilo = nBucketsPerSilo;
lockable = new object();
running = true;
myRange = RangeFactory.CreateFullRange();
logger.Info("Starting {0} on silo {1}.", typeof(VirtualBucketsRingProvider).Name, siloAddr.ToStringWithHashCode());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RING, ToString);
IntValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_RINGSIZE, () => GetRingSize());
StringValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGDISTANCE, () => String.Format("x{0,8:X8}", ((IRingRangeInternal)myRange).RangeSize()));
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_MYRANGE_RINGPERCENTAGE, () => (float)((IRingRangeInternal)myRange).RangePercentage());
FloatValueStatistic.FindOrCreate(StatisticNames.CONSISTENTRING_AVERAGERINGPERCENTAGE, () =>
{
int size = GetRingSize();
return size == 0 ? 0 : ((float)100.0/(float) size);
});
// add myself to the list of members
AddServer(myAddress);
}
private void AddServer(SiloAddress silo)
{
lock (lockable)
{
List <uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
if (bucketsMap.TryGetValue(hash, out var other))
{
// If two silos conflict, take the lesser of the two (usually the older one; that is, the lower epoch)
if (silo.CompareTo(other) > 0)
{
continue;
}
}
bucketsMap[hash] = silo;
}
var myOldRange = myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple <uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Added_Silo, "Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
internal LocalReminderService(
SiloAddress addr,
GrainId id,
IConsistentRingProvider ring,
OrleansTaskScheduler localScheduler,
IReminderTable reminderTable,
GlobalConfiguration config,
TimeSpan initTimeout)
: base(id, addr)
{
logger = TraceLogger.GetLogger("ReminderService", TraceLogger.LoggerType.Runtime);
localReminders = new Dictionary <ReminderIdentity, LocalReminderData>();
this.ring = ring;
scheduler = localScheduler;
this.reminderTable = reminderTable;
this.config = config;
this.initTimeout = initTimeout;
status = ReminderServiceStatus.Booting;
myRange = null;
localTableSequence = 0;
tardinessStat = AverageTimeSpanStatistic.FindOrCreate(StatisticNames.REMINDERS_AVERAGE_TARDINESS_SECONDS);
IntValueStatistic.FindOrCreate(StatisticNames.REMINDERS_NUMBER_ACTIVE_REMINDERS, () => localReminders.Count);
ticksDeliveredStat = CounterStatistic.FindOrCreate(StatisticNames.REMINDERS_COUNTERS_TICKS_DELIVERED);
startedTask = new TaskCompletionSource <bool>();
}
/// <summary>
/// Attempt to retrieve reminders, that are my responsibility, from the global reminder table when starting this silo (reminder service instance)
/// </summary>
/// <returns></returns>
public async Task Start()
{
myRange = ring.GetMyRange();
logger.Info(ErrorCode.RS_ServiceStarting, "Starting reminder system target on: {0} x{1,8:X8}, with range {2}", Silo, Silo.GetConsistentHashCode(), myRange);
await reminderTable.Init(config, logger).WithTimeout(initTimeout);
await ReadAndUpdateReminders();
logger.Info(ErrorCode.RS_ServiceStarted, "Reminder system target started OK on: {0} x{1,8:X8}, with range {2}", Silo, Silo.GetConsistentHashCode(), myRange);
status = ReminderServiceStatus.Started;
startedTask.TrySetResult(true);
var random = new SafeRandom();
var dueTime = random.NextTimeSpan(Constants.RefreshReminderList);
listRefresher = GrainTimer.FromTaskCallback(
_ => ReadAndUpdateReminders(),
null,
dueTime,
Constants.RefreshReminderList,
name: "ReminderService.ReminderListRefresher");
listRefresher.Start();
ring.SubscribeToRangeChangeEvents(this);
}
internal void RemoveServer(SiloAddress silo)
{
lock (lockable)
{
if (!bucketsMap.ContainsValue(silo))
{
return; // we have already removed this silo
}
List <uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
bucketsMap.Remove(hash);
}
var myOldRange = this.myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple <uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Removed_Silo, "Removed Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
/// <summary>Invoked when the ring range owned by the service instance changes because of a change in the cluster state</summary>
public virtual Task OnRangeChange(IRingRange oldRange, IRingRange newRange, bool increased)
{
Logger.Info(ErrorCode.RS_RangeChanged, "My range changed from {0} to {1} increased = {2}", oldRange, newRange, increased);
RingRange = newRange;
RangeSerialNumber++;
return(Task.CompletedTask);
}
public IEnumerable <QueueId> GetQueuesForRange(IRingRange range)
{
return
(from ring in _queueMap.Values
from queueId in ring.GetAllRingMembers()
where range.InRange(queueId.GetUniformHashCode())
select QueueId.GetQueueId(queueId.QueueNamePrefix, queueId.QueueId, queueId.UniformHashCache));
}
public IEnumerable <QueueId> GetQueuesForRange(IRingRange range)
{
foreach (QueueId queueId in hashRing.GetAllRingMembers())
{
if (range.InRange(queueId.GetUniformHashCode()))
{
yield return(queueId);
}
}
}
/// <summary>
/// Attempt to retrieve reminders, that are my responsibility, from the global reminder table when starting this silo (reminder service instance)
/// </summary>
/// <returns></returns>
public async Task Start()
{
myRange = ring.GetMyRange();
logger.Info(ErrorCode.RS_ServiceStarting, "Starting reminder system target on: {0} x{1,8:X8}, with range {2}", Silo, Silo.GetConsistentHashCode(), myRange);
// confirm that it can access the underlying store, as after this the ReminderService will load in the background, without the opportunity to prevent the Silo from starting
await reminderTable.Init(config, logger).WithTimeout(initTimeout);
StartInBackground().Ignore();
}
public ConsistentRingQueueBalancer(IConsistentRingProvider consistentRingProvider, ILoggerFactory loggerFactory, IServiceProvider services, ILogger <ConsistentRingQueueBalancer> logger)
: base(services, logger)
{
if (consistentRingProvider == null)
{
throw new ArgumentNullException("streamProviderRuntime");
}
_myRange = consistentRingProvider.GetMyRange();
consistentRingProvider.SubscribeToRangeChangeEvents(this);
}
public Task RangeChangeNotification(IRingRange old, IRingRange now)
{
_myRange = now;
var notificatioTasks = new List <Task>(_queueBalanceListeners.Count);
foreach (IStreamQueueBalanceListener listener in _queueBalanceListeners)
{
notificatioTasks.Add(listener.QueueDistributionChangeNotification());
}
return(Task.WhenAll(notificatioTasks));
}
public ConsistentRingQueueBalancer(IStreamProviderRuntime streamProviderRuntime)
{
if (streamProviderRuntime == null)
{
throw new ArgumentNullException("streamProviderRuntime");
}
var ringProvider = streamProviderRuntime.GetConsistentRingProvider(0, 1);
myRange = ringProvider.GetMyRange();
ringProvider.SubscribeToRangeChangeEvents(this);
}
public static IEnumerable <SingleRange> GetSubRanges(IRingRange range)
{
if (range is SingleRange)
{
return(new SingleRange[] { (SingleRange)range });
}
else if (range is GeneralMultiRange)
{
return(((GeneralMultiRange)range).Ranges);
}
return(null);
}
/// <summary>
/// Invoked when the ring range owned by the service instance changes because of a change in the cluster state
/// </summary>
/// <param name="oldRange">The old range.</param>
/// <param name="newRange">The new range.</param>
/// <param name="increased">A value indicating whether the range has increased.</param>
/// <returns>A <see cref="Task"/> representing the work performed.</returns>
public virtual Task OnRangeChange(IRingRange oldRange, IRingRange newRange, bool increased)
{
Logger.LogInformation(
(int)ErrorCode.RS_RangeChanged,
"My range changed from {OldRange} to {NewRange} increased = {Increased}",
oldRange,
newRange,
increased);
RingRange = newRange;
RangeSerialNumber++;
return(Task.CompletedTask);
}
/// <inheritdoc/>
public IEnumerable <QueueId> GetQueuesForRange(IRingRange range)
{
var ls = new List <QueueId>();
foreach (QueueId queueId in hashRing.GetAllRingMembers())
{
if (range.InRange(queueId.GetUniformHashCode()))
{
ls.Add(queueId);
}
}
return(ls);
}
public ConsistentRingProvider(SiloAddress siloAddr)
{
log = LogManager.GetLogger(typeof(ConsistentRingProvider).Name);
membershipRingList = new List<SiloAddress>();
MyAddress = siloAddr;
myKey = MyAddress.GetConsistentHashCode();
// add myself to the list of members
AddServer(MyAddress);
MyRange = RangeFactory.CreateFullRange(); // i am responsible for the whole range
statusListeners = new List<IRingRangeListener>();
Start();
}
public override async Task OnRangeChange(IRingRange oldRange, IRingRange newRange, bool increased)
{
await base.OnRangeChange(oldRange, newRange, increased);
if (Status == GrainServiceStatus.Started)
{
await ReadAndUpdateReminders();
}
else
if (Logger.IsVerbose)
{
Logger.Verbose("Ignoring range change until ReminderService is Started -- Current status = {0}", Status);
}
}
public ConsistentRingProvider(SiloAddress siloAddr)
{
log = TraceLogger.GetLogger(typeof(ConsistentRingProvider).Name);
membershipRingList = new List <SiloAddress>();
MyAddress = siloAddr;
myKey = MyAddress.GetConsistentHashCode();
// add myself to the list of members
AddServer(MyAddress);
MyRange = RangeFactory.CreateFullRange(); // i am responsible for the whole range
statusListeners = new List <IRingRangeListener>();
Start();
}
private async Task OnRangeChange(IRingRange oldRange, IRingRange newRange, bool increased)
{
logger.Info(ErrorCode.RS_RangeChanged, "My range changed from {0} to {1} increased = {2}", oldRange, newRange, increased);
myRange = newRange;
if (status == ReminderServiceStatus.Started)
{
await ReadAndUpdateReminders();
}
else
if (logger.IsVerbose)
{
logger.Verbose("Ignoring range change until ReminderService is Started -- Current status = {0}", status);
}
}
public Task RangeChangeNotification(IRingRange old, IRingRange now)
{
myRange = now;
List<IStreamQueueBalanceListener> queueBalanceListenersCopy;
lock (queueBalanceListeners)
{
queueBalanceListenersCopy = queueBalanceListeners.ToList();
}
var notificatioTasks = new List<Task>(queueBalanceListenersCopy.Count);
foreach (IStreamQueueBalanceListener listener in queueBalanceListenersCopy)
{
notificatioTasks.Add(listener.QueueDistributionChangeNotification());
}
return Task.WhenAll(notificatioTasks);
}
// This class takes a range and devides it into X (x being numSubRanges) equal ranges.
public EquallyDividedMultiRange(IRingRange range, int numSubRanges)
{
multiRanges = new Dictionary <int, IRingRangeInternal>();
if (range is SingleRange)
{
var fullSingleRange = range as SingleRange;
var singleDevided = new EquallyDividedSingleRange(fullSingleRange, numSubRanges);
for (int i = 0; i < numSubRanges; i++)
{
var singleRange = singleDevided.GetSubRange(i);
multiRanges[i] = singleRange;
}
}
else if (range is GeneralMultiRange)
{
var fullMultiRange = range as GeneralMultiRange;
// Take each of the single ranges in the multi range and divide each into equal sub ranges.
// Then go over all those and group them by sub range index.
var allSinglesDevided = new List <EquallyDividedSingleRange>();
foreach (var singleRange in fullMultiRange.Ranges)
{
var singleDevided = new EquallyDividedSingleRange(singleRange, numSubRanges);
allSinglesDevided.Add(singleDevided);
}
for (int i = 0; i < numSubRanges; i++)
{
var singlesForThisIndex = new List <IRingRange>();
foreach (var singleDevided in allSinglesDevided)
{
IRingRange singleRange = singleDevided.GetSubRange(i);
singlesForThisIndex.Add(singleRange);
}
IRingRangeInternal multi = (IRingRangeInternal)RangeFactory.CreateRange(singlesForThisIndex);
multiRanges[i] = multi;
}
}
if (multiRanges.Count == 0)
{
rangeSize = 0;
rangePercentage = 0;
}
else
{
rangeSize = multiRanges.Values.Sum(r => r.RangeSize());
rangePercentage = multiRanges.Values.Sum(r => r.RangePercentage());
}
}
internal void AddServer(SiloAddress silo)
{
lock (membershipRingList)
{
if (membershipRingList.Contains(silo))
{
return; // we already have this silo
}
int myOldIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (!(membershipRingList.Count == 0 || myOldIndex != -1))
{
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, Utils.EnumerableToString(membershipRingList)));
}
// insert new silo in the sorted order
int hash = silo.GetConsistentHashCode();
// Find the last silo with hash smaller than the new silo, and insert the latter after (this is why we have +1 here) the former.
// Notice that FindLastIndex might return -1 if this should be the first silo in the list, but then
// 'index' will get 0, as needed.
int index = membershipRingList.FindLastIndex(siloAddr => siloAddr.GetConsistentHashCode() < hash) + 1;
membershipRingList.Insert(index, silo);
// relating to triggering handler ... new node took over some of my responsibility
if (index == myOldIndex || // new node was inserted in my place
(myOldIndex == 0 && index == membershipRingList.Count - 1)) // I am the first node, and the new server is the last node
{
IRingRange oldRange = MyRange;
try
{
MyRange = RangeFactory.CreateRange(unchecked ((uint)hash), unchecked ((uint)myKey));
}
catch (OverflowException exc)
{
log.Error(ErrorCode.ConsistentRingProviderBase + 5,
String.Format("OverflowException: hash as int= x{0, 8:X8}, hash as uint= x{1, 8:X8}, myKey as int x{2, 8:X8}, myKey as uint x{3, 8:X8}.",
hash, (uint)hash, myKey, (uint)myKey), exc);
}
NotifyLocalRangeSubscribers(oldRange, MyRange, false);
}
log.Info("Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
}
}
public void RangeChangeNotification(IRingRange old, IRingRange now, bool increased)
{
myRange = CalcMyRange();
var oldMultiRange = RangeFactory.CreateEquallyDividedMultiRange(old, numSubRanges);
IRingRange oldSubRange = oldMultiRange.GetSubRange(mySubRangeIndex);
var newMultiRange = RangeFactory.CreateEquallyDividedMultiRange(now, numSubRanges);
IRingRange newSubRange = newMultiRange.GetSubRange(mySubRangeIndex);
if (oldSubRange.Equals(newSubRange))
{
return;
}
// For now, always say your range increased and the listeners need to deal with the situation when they get the same range again anyway.
// In the future, check sub range inclusion. Note that it is NOT correct to just return the passed increased argument.
// It will be wrong: the multi range may have decreased, while some individual sub range may partialy increase (shift).
logger.Info("-NotifyLocal GrainRangeSubscribers about old {0} and new {1} increased? {2}.", oldSubRange.ToString(), newSubRange.ToString(), increased);
List <IAsyncRingRangeListener> copy;
lock (grainStatusListeners)
{
copy = grainStatusListeners.ToList();
}
foreach (IAsyncRingRangeListener listener in copy)
{
try
{
Task task = listener.RangeChangeNotification(oldSubRange, newSubRange);
// We don't want to await it here since it will delay delivering notifications to other listeners.
// We only want to log an error if it happends, so use ContinueWith.
task.LogException(logger, ErrorCode.CRP_ForGrains_Local_Subscriber_Exception_1,
String.Format("Local IGrainRingRangeListener {0} has thrown an asynchronous exception when was notified about RangeChangeNotification about old {1} new {2}.",
listener.GetType().FullName, oldSubRange, newSubRange))
.Ignore();
}
catch (Exception exc)
{
logger.Error(ErrorCode.CRP_ForGrains_Local_Subscriber_Exception_2,
String.Format("Local IGrainRingRangeListener {0} has thrown an exception when was notified about RangeChangeNotification about old {1} new {2}.",
listener.GetType().FullName, oldSubRange, newSubRange), exc);
}
}
}
private static IRingRange CalculateRange(List <Tuple <uint, SiloAddress> > list, SiloAddress silo)
{
var ranges = new List <IRingRange>();
for (int i = 0; i < list.Count; i++)
{
var curr = list[i];
var next = list[(i + 1) % list.Count];
// 'backward/clockwise' definition to assign responsibilities on the ring.
if (next.Item2.Equals(silo))
{
IRingRange range = RangeFactory.CreateRange(curr.Item1, next.Item1);
ranges.Add(range);
}
}
return(RangeFactory.CreateRange(ranges));
}
public Task RangeChangeNotification(IRingRange old, IRingRange now)
{
myRange = now;
List <IStreamQueueBalanceListener> queueBalanceListenersCopy;
lock (queueBalanceListeners)
{
queueBalanceListenersCopy = queueBalanceListeners.ToList();
}
var notificatioTasks = new List <Task>(queueBalanceListenersCopy.Count);
foreach (IStreamQueueBalanceListener listener in queueBalanceListenersCopy)
{
notificatioTasks.Add(listener.QueueDistributionChangeNotification());
}
return(Task.WhenAll(notificatioTasks));
}
internal void RemoveServer(SiloAddress silo)
{
lock (membershipRingList)
{
int indexOfFailedSilo = membershipRingList.FindIndex(elem => elem.Equals(silo));
if (indexOfFailedSilo < 0)
{
return; // we have already removed this silo
}
membershipRingList.Remove(silo);
// related to triggering handler
int myNewIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (myNewIndex == -1)
{
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, this.ToString()));
}
bool wasMyPred = ((myNewIndex == indexOfFailedSilo) || (myNewIndex == 0 && indexOfFailedSilo == membershipRingList.Count)); // no need for '- 1'
if (wasMyPred) // failed node was our predecessor
{
if (log.IsVerbose)
{
log.Verbose("Failed server was my pred? {0}, updated view {1}", wasMyPred, this.ToString());
}
IRingRange oldRange = MyRange;
if (membershipRingList.Count == 1) // i'm the only one left
{
MyRange = RangeFactory.CreateFullRange();
NotifyLocalRangeSubscribers(oldRange, MyRange, true);
}
else
{
int myNewPredIndex = myNewIndex == 0 ? membershipRingList.Count - 1 : myNewIndex - 1;
int myPredecessorsHash = membershipRingList[myNewPredIndex].GetConsistentHashCode();
MyRange = RangeFactory.CreateRange(unchecked ((uint)myPredecessorsHash), unchecked ((uint)myKey));
NotifyLocalRangeSubscribers(oldRange, MyRange, true);
}
}
log.Info("Removed Server {0} hash {1}. Current view {2}", silo, silo.GetConsistentHashCode(), this.ToString());
}
}
public ConsistentRingQueueBalancer(
IConsistentRingProviderForGrains ringProvider,
IStreamQueueMapper queueMapper)
{
if (ringProvider == null)
{
throw new ArgumentNullException("ringProvider");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
_streamQueueMapper = queueMapper;
_myRange = ringProvider.GetMyRange();
ringProvider.SubscribeToRangeChangeEvents(this);
}
public void RangeChangeNotification(IRingRange old, IRingRange now, bool increased)
{
myRange = CalcMyRange();
var oldMultiRange = new EquallyDevidedMultiRange(old, numSubRanges);
IRingRange oldSubRange = oldMultiRange.GetSubRange(mySubRangeIndex);
var newMultiRange = new EquallyDevidedMultiRange(now, numSubRanges);
IRingRange newSubRange = newMultiRange.GetSubRange(mySubRangeIndex);
if (oldSubRange.Equals(newSubRange)) return;
// For now, always say your range increased and the listeners need to deal with the situation when they get the same range again anyway.
// In the future, check sub range inclusion. Note that it is NOT correct to just return the passed increased argument.
// It will be wrong: the multi range may have decreased, while some individual sub range may partialy increase (shift).
logger.Info("-NotifyLocal GrainRangeSubscribers about old {0} and new {1} increased? {2}.", oldSubRange.ToString(), newSubRange.ToString(), increased);
List<IAsyncRingRangeListener> copy;
lock (grainStatusListeners)
{
copy = grainStatusListeners.ToList();
}
foreach (IAsyncRingRangeListener listener in copy)
{
try
{
Task task = listener.RangeChangeNotification(oldSubRange, newSubRange);
// We don't want to await it here since it will delay delivering notifications to other listeners.
// We only want to log an error if it happends, so use ContinueWith.
task.LogException(logger, ErrorCode.CRP_ForGrains_Local_Subscriber_Exception_1,
String.Format("Local IGrainRingRangeListener {0} has thrown an asynchronous exception when was notified about RangeChangeNotification about old {1} new {2}.",
listener.GetType().FullName, oldSubRange, newSubRange))
.Ignore();
}
catch (Exception exc)
{
logger.Error(ErrorCode.CRP_ForGrains_Local_Subscriber_Exception_2,
String.Format("Local IGrainRingRangeListener {0} has thrown an exception when was notified about RangeChangeNotification about old {1} new {2}.",
listener.GetType().FullName, oldSubRange, newSubRange), exc);
}
}
}
// Takes a range and devides it into numSubRanges equal ranges and returns the subrange at mySubRangeIndex.
public static IRingRange GetEquallyDividedSubRange(IRingRange range, int numSubRanges, int mySubRangeIndex)
{
if (numSubRanges <= 0)
{
throw new ArgumentException(nameof(numSubRanges));
}
if ((uint)mySubRangeIndex >= (uint)numSubRanges)
{
throw new ArgumentException(nameof(mySubRangeIndex));
}
if (numSubRanges == 1)
{
return(range);
}
switch (range)
{
case SingleRange singleRange:
return(GetEquallyDividedSubRange(singleRange, numSubRanges, mySubRangeIndex));
case GeneralMultiRange multiRange:
switch (multiRange.Ranges.Count)
{
case 0: return(multiRange);
case 1: return(GetEquallyDividedSubRange(multiRange.Ranges[0], numSubRanges, mySubRangeIndex));
default:
// Take each of the single ranges in the multi range and divide each into equal sub ranges.
var singlesForThisIndex = new List <SingleRange>(multiRange.Ranges.Count);
foreach (var singleRange in multiRange.Ranges)
{
singlesForThisIndex.Add(GetEquallyDividedSubRange(singleRange, numSubRanges, mySubRangeIndex));
}
return(new GeneralMultiRange(singlesForThisIndex));
}
default: throw new ArgumentException(nameof(range));
}
}
public override string ToString()
{
lock (membershipRingList)
{
if (membershipRingList.Count == 1)
{
return(Utils.EnumerableToString(membershipRingList, silo => String.Format("{0} -> {1}", silo.ToStringWithHashCode(), RangeFactory.CreateFullRange())));
}
var sb = new StringBuilder("[");
for (int i = 0; i < membershipRingList.Count; i++)
{
SiloAddress curr = membershipRingList[i];
SiloAddress next = membershipRingList[(i + 1) % membershipRingList.Count];
IRingRange range = RangeFactory.CreateRange(unchecked ((uint)curr.GetConsistentHashCode()), unchecked ((uint)next.GetConsistentHashCode()));
sb.Append(String.Format("{0} -> {1}, ", curr.ToStringWithHashCode(), range));
}
sb.Append("]");
return(sb.ToString());
}
}
public bool Remove(IRingRange range)
{
bool wholerange = true;
foreach (SingleRange s in RangeFactory.GetSubRanges(range))
{
bool found = false;
foreach (SingleRange m in ranges)
{
if (m.Begin == m.End) // treat full range as special case
{
found = true;
ranges.Remove(m);
if (s.Begin != s.End) // if s is full range as well, then end of story ... whole range is covered
{
ranges.Add(new SingleRange(s.End, s.Begin));
}
break;
}
if (m.InRange(s.Begin + 1) && m.InRange(s.End)) // s cant overlap two singleranges
{
found = true;
ranges.Remove(m);
if (s.Begin != m.Begin)
{
ranges.Add(new SingleRange(m.Begin, s.Begin));
}
if (s.End != m.End)
{
ranges.Add(new SingleRange(s.End, m.End));
}
break;
}
}
wholerange = wholerange && found;
}
return(wholerange);
}
public ConsistentRingQueueBalancer(
IConsistentRingProviderForGrains ringProvider,
IStreamQueueMapper queueMapper)
{
if (ringProvider == null)
{
throw new ArgumentNullException("ringProvider");
}
if (queueMapper == null)
{
throw new ArgumentNullException("queueMapper");
}
if (!(queueMapper is IConsistentRingStreamQueueMapper))
{
throw new ArgumentException("queueMapper for ConsistentRingQueueBalancer should implement IConsistentRingStreamQueueMapper", "queueMapper");
}
streamQueueMapper = (IConsistentRingStreamQueueMapper)queueMapper;
myRange = ringProvider.GetMyRange();
ringProvider.SubscribeToRangeChangeEvents(this);
}
internal void AddServer(SiloAddress silo)
{
lock (membershipRingList)
{
if (membershipRingList.Contains(silo)) return; // we already have this silo
int myOldIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (!(membershipRingList.Count == 0 || myOldIndex != -1))
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, Utils.EnumerableToString(membershipRingList)));
// insert new silo in the sorted order
int hash = silo.GetConsistentHashCode();
// Find the last silo with hash smaller than the new silo, and insert the latter after (this is why we have +1 here) the former.
// Notice that FindLastIndex might return -1 if this should be the first silo in the list, but then
// 'index' will get 0, as needed.
int index = membershipRingList.FindLastIndex(siloAddr => siloAddr.GetConsistentHashCode() < hash) + 1;
membershipRingList.Insert(index, silo);
// relating to triggering handler ... new node took over some of my responsibility
if (index == myOldIndex || // new node was inserted in my place
(myOldIndex == 0 && index == membershipRingList.Count - 1)) // I am the first node, and the new server is the last node
{
IRingRange oldRange = MyRange;
try
{
MyRange = RangeFactory.CreateRange(unchecked((uint)hash), unchecked((uint)myKey));
}
catch (OverflowException exc)
{
log.Error(ErrorCode.ConsistentRingProviderBase + 5,
String.Format("OverflowException: hash as int= x{0, 8:X8}, hash as uint= x{1, 8:X8}, myKey as int x{2, 8:X8}, myKey as uint x{3, 8:X8}.",
hash, (uint)hash, myKey, (uint)myKey), exc);
}
NotifyLocalRangeSubscribers(oldRange, MyRange, false);
}
log.Info("Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
}
}
internal void RemoveServer(SiloAddress silo)
{
lock (lockable)
{
if (!bucketsMap.ContainsValue(silo)) return; // we have already removed this silo
List<uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
bucketsMap.Remove(hash);
}
var myOldRange = this.myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple<uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Removed_Silo, "Removed Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
private void AddServer(SiloAddress silo)
{
lock (lockable)
{
List<uint> hashes = silo.GetUniformHashCodes(numBucketsPerSilo);
foreach (var hash in hashes)
{
if (bucketsMap.ContainsKey(hash))
{
var other = bucketsMap[hash];
// If two silos conflict, take the lesser of the two (usually the older one; that is, the lower epoch)
if (silo.CompareTo(other) > 0) continue;
}
bucketsMap[hash] = silo;
}
var myOldRange = myRange;
var bucketsList = bucketsMap.Select(pair => new Tuple<uint, SiloAddress>(pair.Key, pair.Value)).ToList();
var myNewRange = CalculateRange(bucketsList, myAddress);
// capture my range and sortedBucketsList for later lock-free access.
myRange = myNewRange;
sortedBucketsList = bucketsList;
logger.Info(ErrorCode.CRP_Added_Silo, "Added Server {0}. Current view: {1}", silo.ToStringWithHashCode(), this.ToString());
NotifyLocalRangeSubscribers(myOldRange, myNewRange, true);
}
}
public IEnumerable<QueueId> GetQueuesForRange(IRingRange range)
{
foreach (QueueId queueId in hashRing.GetAllRingMembers())
if (range.InRange(queueId.GetUniformHashCode()))
yield return queueId;
}
private void NotifyLocalRangeSubscribers(IRingRange old, IRingRange now, bool increased)
{
log.Info("-NotifyLocalRangeSubscribers about old {0} new {1} increased? {2}", old, now, increased);
List<IRingRangeListener> copy;
lock (statusListeners)
{
copy = statusListeners.ToList();
}
foreach (IRingRangeListener listener in copy)
{
try
{
listener.RangeChangeNotification(old, now, increased);
}
catch (Exception exc)
{
log.Error(ErrorCode.CRP_Local_Subscriber_Exception,
String.Format("Local IRangeChangeListener {0} has thrown an exception when was notified about RangeChangeNotification about old {1} new {2} increased? {3}",
listener.GetType().FullName, old, now, increased), exc);
}
}
}
internal void RemoveServer(SiloAddress silo)
{
lock (membershipRingList)
{
int indexOfFailedSilo = membershipRingList.FindIndex(elem => elem.Equals(silo));
if (indexOfFailedSilo < 0) return; // we have already removed this silo
membershipRingList.Remove(silo);
// related to triggering handler
int myNewIndex = membershipRingList.FindIndex(elem => elem.Equals(MyAddress));
if (myNewIndex == -1)
throw new OrleansException(string.Format("{0}: Couldn't find my position in the ring {1}.", MyAddress, this.ToString()));
bool wasMyPred = ((myNewIndex == indexOfFailedSilo) || (myNewIndex == 0 && indexOfFailedSilo == membershipRingList.Count)); // no need for '- 1'
if (wasMyPred) // failed node was our predecessor
{
if (log.IsVerbose) log.Verbose("Failed server was my pred? {0}, updated view {1}", wasMyPred, this.ToString());
IRingRange oldRange = MyRange;
if (membershipRingList.Count == 1) // i'm the only one left
{
MyRange = RangeFactory.CreateFullRange();
NotifyLocalRangeSubscribers(oldRange, MyRange, true);
}
else
{
int myNewPredIndex = myNewIndex == 0 ? membershipRingList.Count - 1 : myNewIndex - 1;
int myPredecessorsHash = membershipRingList[myNewPredIndex].GetConsistentHashCode();
MyRange = RangeFactory.CreateRange(unchecked((uint)myPredecessorsHash), unchecked((uint)myKey));
NotifyLocalRangeSubscribers(oldRange, MyRange, true);
}
}
log.Info("Removed Server {0} hash {1}. Current view {2}", silo, silo.GetConsistentHashCode(), this.ToString());
}
}
public IRingRange GetMyRange()
{
return myRange ?? (myRange = CalcMyRange());
}
public bool Remove(IRingRange range)
{
bool wholerange = true;
foreach (SingleRange s in RangeFactory.GetSubRanges(range))
{
bool found = false;
foreach (SingleRange m in ranges)
{
if (m.Begin == m.End) // treat full range as special case
{
found = true;
ranges.Remove(m);
if (s.Begin != s.End) // if s is full range as well, then end of story ... whole range is covered
{
ranges.Add(new SingleRange(s.End, s.Begin));
}
break;
}
if (m.InRange(s.Begin + 1) && m.InRange(s.End)) // s cant overlap two singleranges
{
found = true;
ranges.Remove(m);
if (s.Begin != m.Begin)
{
ranges.Add(new SingleRange(m.Begin, s.Begin));
}
if (s.End != m.End)
{
ranges.Add(new SingleRange(s.End, m.End));
}
break;
}
}
wholerange = wholerange && found;
}
return wholerange;
}