private static readonly TimeSpan RETRY_DELAY = TimeSpan.FromSeconds(5); // Pause 5 seconds between forwards to let the membership directory settle down
internal RemoteGrainDirectory(LocalGrainDirectory r, GrainId id)
: base(id, r.MyAddress)
{
router = r;
partition = r.DirectoryPartition;
logger = TraceLogger.GetLogger("Orleans.GrainDirectory.CacheValidator", TraceLogger.LoggerType.Runtime);
}
public GlobalSingleInstanceRegistrar(GrainDirectoryPartition partition, Logger logger, GlobalSingleInstanceActivationMaintainer gsiActivationMaintainer, int numRetries)
{
this.directoryPartition = partition;
this.logger = logger;
this.gsiActivationMaintainer = gsiActivationMaintainer;
this.numRetries = numRetries;
}
public ClusterLocalRegistrar(GrainDirectoryPartition partition)
{
directoryPartition = partition;
}
/// <summary>
/// Runs through all entries in the partition and moves/copies (depending on the given flag) the
/// entries satisfying the given predicate into a new partition.
/// This method is supposed to be used by handoff manager to update the partitions when the system view (set of live silos) changes.
/// </summary>
/// <param name="predicate">filter predicate (usually if the given grain is owned by particular silo)</param>
/// <param name="modifyOrigin">flag controling whether the source partition should be modified (i.e., the entries should be moved or just copied) </param>
/// <returns>new grain directory partition containing entries satisfying the given predicate</returns>
internal GrainDirectoryPartition Split(Predicate<GrainId> predicate, bool modifyOrigin)
{
var newDirectory = new GrainDirectoryPartition();
if (modifyOrigin)
{
// SInce we use the "pairs" list to modify the underlying collection below, we need to turn it into an actual list here
List<KeyValuePair<GrainId, IGrainInfo>> pairs;
lock (lockable)
{
pairs = partitionData.Where(pair => predicate(pair.Key)).ToList();
}
foreach (var pair in pairs)
{
newDirectory.partitionData.Add(pair.Key, pair.Value);
}
lock (lockable)
{
foreach (var pair in pairs)
{
partitionData.Remove(pair.Key);
}
}
}
else
{
lock (lockable)
{
foreach (var pair in partitionData.Where(pair => predicate(pair.Key)))
{
newDirectory.partitionData.Add(pair.Key, pair.Value);
}
}
}
return newDirectory;
}
/// <summary>
/// Merges one partition into another, asuuming partitions are disjoint.
/// This method is supposed to be used by handoff manager to update the partitions when the system view (set of live silos) changes.
/// </summary>
/// <param name="other"></param>
internal void Merge(GrainDirectoryPartition other)
{
lock (lockable)
{
foreach (var pair in other.partitionData)
{
if (partitionData.ContainsKey(pair.Key))
{
if (log.IsVerbose) log.Verbose("While merging two disjoint partitions, same grain " + pair.Key + " was found in both partitions");
partitionData[pair.Key].Merge(pair.Key, pair.Value);
}
else
{
partitionData.Add(pair.Key, pair.Value);
}
}
}
}
internal void AcceptHandoffPartition(SiloAddress source, Dictionary<GrainId, IGrainInfo> partition, bool isFullCopy)
{
if (logger.IsVerbose) logger.Verbose("Got request to register " + (isFullCopy ? "FULL" : "DELTA") + " directory partition with " + partition.Count + " elements from " + source);
if (!directoryPartitionsMap.ContainsKey(source))
{
if (!isFullCopy)
{
logger.Warn(ErrorCode.DirectoryUnexpectedDelta,
String.Format("Got delta of the directory partition from silo {0} (Membership status {1}) while not holding a full copy. Membership active cluster size is {2}",
source, localDirectory.Membership.GetApproximateSiloStatus(source),
localDirectory.Membership.GetApproximateSiloStatuses(true).Count));
}
directoryPartitionsMap[source] = new GrainDirectoryPartition();
}
if (isFullCopy)
{
directoryPartitionsMap[source].Set(partition);
}
else
{
directoryPartitionsMap[source].Update(partition);
}
}
public LocalGrainDirectory(Silo silo)
{
log = TraceLogger.GetLogger("Orleans.GrainDirectory.LocalGrainDirectory");
MyAddress = silo.LocalMessageCenter.MyAddress;
Scheduler = silo.LocalScheduler;
membershipRingList = new List<SiloAddress>();
membershipCache = new HashSet<SiloAddress>();
silo.OrleansConfig.OnConfigChange("Globals/Caching", () =>
{
lock (membershipCache)
{
DirectoryCache = GrainDirectoryCacheFactory<List<Tuple<SiloAddress, ActivationId>>>.CreateGrainDirectoryCache(silo.GlobalConfig);
}
});
maintainer = GrainDirectoryCacheFactory<List<Tuple<SiloAddress, ActivationId>>>.CreateGrainDirectoryCacheMaintainer(this, DirectoryCache);
if (silo.GlobalConfig.SeedNodes.Count > 0)
{
seed = silo.GlobalConfig.SeedNodes.Contains(MyAddress.Endpoint) ? MyAddress : SiloAddress.New(silo.GlobalConfig.SeedNodes[0], 0);
}
stopPreparationResolver = new TaskCompletionSource<bool>();
DirectoryPartition = new GrainDirectoryPartition();
HandoffManager = new GrainDirectoryHandoffManager(this, silo.GlobalConfig);
RemGrainDirectory = new RemoteGrainDirectory(this, Constants.DirectoryServiceId);
CacheValidator = new RemoteGrainDirectory(this, Constants.DirectoryCacheValidatorId);
// add myself to the list of members
AddServer(MyAddress);
Func<SiloAddress, string> siloAddressPrint = (SiloAddress addr) =>
String.Format("{0}/{1:X}", addr.ToLongString(), addr.GetConsistentHashCode());
localLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED);
localSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_SUCCESSES);
fullLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_FULL_ISSUED);
RemoteLookupsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_SENT);
RemoteLookupsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_RECEIVED);
LocalDirectoryLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED);
LocalDirectorySuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES);
cacheLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_ISSUED);
cacheSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_SUCCESSES);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_HITRATIO, () =>
{
long delta1, delta2;
long curr1 = cacheSuccesses.GetCurrentValueAndDelta(out delta1);
long curr2 = cacheLookups.GetCurrentValueAndDelta(out delta2);
return String.Format("{0}, Delta={1}",
(curr2 != 0 ? (float)curr1 / (float)curr2 : 0)
,(delta2 !=0 ? (float)delta1 / (float)delta2 : 0));
});
CacheValidationsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_SENT);
CacheValidationsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_RECEIVED);
registrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_ISSUED);
RegistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_LOCAL);
RegistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_SENT);
RegistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_RECEIVED);
registrationsSingleActIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED);
RegistrationsSingleActLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL);
RegistrationsSingleActRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT);
RegistrationsSingleActRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED);
unregistrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_ISSUED);
UnregistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_LOCAL);
UnregistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT);
UnregistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED);
unregistrationsManyIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_ISSUED);
UnregistrationsManyRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT);
UnregistrationsManyRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED);
directoryPartitionCount = IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_PARTITION_SIZE, () => DirectoryPartition.Count);
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, () => RingDistanceToSuccessor());
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, () => (((float)RingDistanceToSuccessor()) / ((float)(int.MaxValue * 2L))) * 100);
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, () => membershipRingList.Count == 0 ? 0 : ((float)100 / (float)membershipRingList.Count));
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_RINGSIZE, () => membershipRingList.Count);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING, () =>
{
lock (membershipCache)
{
return Utils.EnumerableToString(membershipRingList, siloAddressPrint);
}
});
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_PREDECESSORS, () => Utils.EnumerableToString(FindPredecessors(MyAddress, 1), siloAddressPrint));
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_SUCCESSORS, () => Utils.EnumerableToString(FindSuccessors(MyAddress, 1), siloAddressPrint));
}
public ClusterLocalRegistrar(GrainDirectoryPartition partition)
{
directoryPartition = partition;
}
public ClusterLocalRegistrar(LocalGrainDirectory directory)
{
directoryPartition = directory.DirectoryPartition;
}
public LocalGrainDirectory(Silo silo)
{
log = TraceLogger.GetLogger("Orleans.GrainDirectory.LocalGrainDirectory");
MyAddress = silo.LocalMessageCenter.MyAddress;
Scheduler = silo.LocalScheduler;
membershipRingList = new List <SiloAddress>();
membershipCache = new HashSet <SiloAddress>();
silo.OrleansConfig.OnConfigChange("Globals/Caching", () =>
{
lock (membershipCache)
{
DirectoryCache = GrainDirectoryCacheFactory <List <Tuple <SiloAddress, ActivationId> > > .CreateGrainDirectoryCache(silo.GlobalConfig);
}
});
maintainer = GrainDirectoryCacheFactory <List <Tuple <SiloAddress, ActivationId> > > .CreateGrainDirectoryCacheMaintainer(this, DirectoryCache);
if (silo.GlobalConfig.SeedNodes.Count > 0)
{
seed = silo.GlobalConfig.SeedNodes.Contains(MyAddress.Endpoint) ? MyAddress : SiloAddress.New(silo.GlobalConfig.SeedNodes[0], 0);
}
stopPreparationResolver = new TaskCompletionSource <bool>();
DirectoryPartition = new GrainDirectoryPartition();
HandoffManager = new GrainDirectoryHandoffManager(this, silo.GlobalConfig);
RemGrainDirectory = new RemoteGrainDirectory(this, Constants.DirectoryServiceId);
CacheValidator = new RemoteGrainDirectory(this, Constants.DirectoryCacheValidatorId);
// add myself to the list of members
AddServer(MyAddress);
Func <SiloAddress, string> siloAddressPrint = (SiloAddress addr) =>
String.Format("{0}/{1:X}", addr.ToLongString(), addr.GetConsistentHashCode());
localLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_ISSUED);
localSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCAL_SUCCESSES);
fullLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_FULL_ISSUED);
RemoteLookupsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_SENT);
RemoteLookupsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_REMOTE_RECEIVED);
LocalDirectoryLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED);
LocalDirectorySuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES);
cacheLookups = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_ISSUED);
cacheSuccesses = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_SUCCESSES);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_LOOKUPS_CACHE_HITRATIO, () =>
{
long delta1, delta2;
long curr1 = cacheSuccesses.GetCurrentValueAndDelta(out delta1);
long curr2 = cacheLookups.GetCurrentValueAndDelta(out delta2);
return(String.Format("{0}, Delta={1}",
(curr2 != 0 ? (float)curr1 / (float)curr2 : 0)
, (delta2 != 0 ? (float)delta1 / (float)delta2 : 0)));
});
CacheValidationsSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_SENT);
CacheValidationsReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_VALIDATIONS_CACHE_RECEIVED);
registrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_ISSUED);
RegistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_LOCAL);
RegistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_SENT);
RegistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_REMOTE_RECEIVED);
registrationsSingleActIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED);
RegistrationsSingleActLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL);
RegistrationsSingleActRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT);
RegistrationsSingleActRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED);
unregistrationsIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_ISSUED);
UnregistrationsLocal = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_LOCAL);
UnregistrationsRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_SENT);
UnregistrationsRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED);
unregistrationsManyIssued = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_ISSUED);
UnregistrationsManyRemoteSent = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT);
UnregistrationsManyRemoteReceived = CounterStatistic.FindOrCreate(StatisticNames.DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED);
directoryPartitionCount = IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_PARTITION_SIZE, () => DirectoryPartition.Count);
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGDISTANCE, () => RingDistanceToSuccessor());
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_RINGPERCENTAGE, () => (((float)RingDistanceToSuccessor()) / ((float)(int.MaxValue * 2L))) * 100);
FloatValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE, () => membershipRingList.Count == 0 ? 0 : ((float)100 / (float)membershipRingList.Count));
IntValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_RINGSIZE, () => membershipRingList.Count);
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING, () =>
{
lock (membershipCache)
{
return(Utils.EnumerableToString(membershipRingList, siloAddressPrint));
}
});
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_PREDECESSORS, () => Utils.EnumerableToString(FindPredecessors(MyAddress, 1), siloAddressPrint));
StringValueStatistic.FindOrCreate(StatisticNames.DIRECTORY_RING_SUCCESSORS, () => Utils.EnumerableToString(FindSuccessors(MyAddress, 1), siloAddressPrint));
}
internal void ProcessSiloAddEvent(SiloAddress addedSilo)
{
if (logger.IsVerbose)
{
logger.Verbose("Processing silo add event for " + addedSilo);
}
// Reset our follower list to take the changes into account
ResetFollowers();
// check if this is one of our successors (i.e., if I should hold this silo's copy)
// (if yes, adjust local and/or copied directory partitions by splitting them between old successors and the new one)
// NOTE: We need to move part of our local directory to the new silo if it is an immediate successor.
List <SiloAddress> successors = localDirectory.FindSuccessors(localDirectory.MyAddress, 1);
if (!successors.Contains(addedSilo))
{
return;
}
// check if this is an immediate successor
if (successors[0].Equals(addedSilo))
{
// split my local directory and send to my new immediate successor his share
if (logger.IsVerbose)
{
logger.Verbose("Splitting my partition between me and " + addedSilo);
}
GrainDirectoryPartition splitPart = localDirectory.DirectoryPartition.Split(
grain =>
{
var s = localDirectory.CalculateTargetSilo(grain);
return((s != null) && !localDirectory.MyAddress.Equals(s));
}, false);
List <ActivationAddress> splitPartListSingle = splitPart.ToListOfActivations(true);
List <ActivationAddress> splitPartListMulti = splitPart.ToListOfActivations(false);
if (splitPartListSingle.Count > 0)
{
if (logger.IsVerbose)
{
logger.Verbose("Sending " + splitPartListSingle.Count + " single activation entries to " + addedSilo);
}
localDirectory.Scheduler.QueueTask(async() =>
{
await localDirectory.GetDirectoryReference(successors[0]).RegisterManySingleActivation(
splitPartListSingle, LocalGrainDirectory.NUM_RETRIES);
splitPartListSingle.ForEach(
activationAddress =>
localDirectory.DirectoryPartition.RemoveGrain(activationAddress.Grain));
}, localDirectory.RemGrainDirectory.SchedulingContext).Ignore();
}
if (splitPartListMulti.Count > 0)
{
if (logger.IsVerbose)
{
logger.Verbose("Sending " + splitPartListMulti.Count + " entries to " + addedSilo);
}
localDirectory.Scheduler.QueueTask(async() =>
{
await localDirectory.GetDirectoryReference(successors[0]).RegisterMany(splitPartListMulti);
splitPartListMulti.ForEach(
activationAddress =>
localDirectory.DirectoryPartition.RemoveGrain(activationAddress.Grain));
}, localDirectory.RemGrainDirectory.SchedulingContext).Ignore();
}
}
else
{
// adjust partitions by splitting them accordingly between new and old silos
SiloAddress predecessorOfNewSilo = localDirectory.FindPredecessors(addedSilo, 1)[0];
if (!directoryPartitionsMap.ContainsKey(predecessorOfNewSilo))
{
// we should have the partition of the predcessor of our new successor
logger.Warn(ErrorCode.DirectoryPartitionPredecessorExpected, "This silo is expected to hold directory partition of " + predecessorOfNewSilo);
}
else
{
if (logger.IsVerbose)
{
logger.Verbose("Splitting partition of " + predecessorOfNewSilo + " and creating a copy for " + addedSilo);
}
GrainDirectoryPartition splitPart = directoryPartitionsMap[predecessorOfNewSilo].Split(
grain =>
{
// Need to review the 2nd line condition.
var s = localDirectory.CalculateTargetSilo(grain);
return((s != null) && !predecessorOfNewSilo.Equals(s));
}, true);
directoryPartitionsMap[addedSilo] = splitPart;
}
}
// remove partition of one of the old successors that we do not need to now
SiloAddress oldSuccessor = directoryPartitionsMap.FirstOrDefault(pair => !successors.Contains(pair.Key)).Key;
if (oldSuccessor == null)
{
return;
}
if (logger.IsVerbose)
{
logger.Verbose("Removing copy of the directory partition of silo " + oldSuccessor + " (holding copy of " + addedSilo + " instead)");
}
directoryPartitionsMap.Remove(oldSuccessor);
}
public ClusterLocalRegistrar(GrainDirectoryPartition directoryPartition)
{
this.directoryPartition = directoryPartition;
}
