/// Adjust local cache following the removal of a silo by droping: /// 1) entries that point to activations located on the removed silo /// 2) entries for grains that are now owned by this silo (me) /// 3) entries for grains that were owned by this removed silo - we currently do NOT do that. /// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership). /// We don't do that since first cache refresh handles that. /// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo. /// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner). protected void AdjustLocalCache(SiloAddress removedSilo) { // remove all records of activations located on the removed silo foreach (Tuple <GrainId, List <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues) { // 2) remove entries owned by me (they should be retrieved from my directory partition) if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1))) { DirectoryCache.Remove(tuple.Item1); } // 1) remove entries that point to activations located on the removed silo if (tuple.Item2.RemoveAll(tuple2 => tuple2.Item1.Equals(removedSilo)) <= 0) { continue; } if (tuple.Item2.Count > 0) { DirectoryCache.AddOrUpdate(tuple.Item1, tuple.Item2, tuple.Item3); } else { DirectoryCache.Remove(tuple.Item1); } } }
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount) { var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued; counterStatistic.Increment(); // see if the owner is somewhere else (returns null if we are owner) var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync"); // on all silos other than first, we insert a retry delay and recheck owner before forwarding if (hopCount > 0 && forwardAddress != null) { await Task.Delay(RETRY_DELAY); forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync"); if (forwardAddress is object) { int hash = unchecked ((int)address.GrainId.GetUniformHashCode()); this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})"); } } if (forwardAddress == null) { RegistrationsSingleActLocal.Increment(); var result = DirectoryPartition.AddSingleActivation(address); return(result); } else { RegistrationsSingleActRemoteSent.Increment(); // otherwise, notify the owner AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1); // Caching optimization: // cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! if (result.Address == null) { return(result); } if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress)) { return(result); } // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address, result.VersionTag); return(result); } }
public async Task RegisterAsync(ActivationAddress address) { registrationsIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsLocal.Increment(); // if I am the owner, store the new activation locally DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo); } else { RegistrationsRemoteSent.Increment(); // otherwise, notify the owner int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES); if (IsValidSilo(address.Silo)) { // Caching optimization: // cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached; if (!DirectoryCache.LookUp(address.Grain, out cached)) { cached = new List <Tuple <SiloAddress, ActivationId> >(1) { Tuple.Create(address.Silo, address.Activation) }; } else { var newcached = new List <Tuple <SiloAddress, ActivationId> >(cached.Count + 1); newcached.AddRange(cached); newcached.Add(Tuple.Create(address.Silo, address.Activation)); cached = newcached; } // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, eTag); } } }
/// <summary> /// Registers a new activation, in single activation mode, with the directory service. /// If there is already an activation registered for this grain, then the new activation will /// not be registered and the address of the existing activation will be returned. /// Otherwise, the passed-in address will be returned. /// <para>This method must be called from a scheduler thread.</para> /// </summary> /// <param name="address">The address of the potential new activation.</param> /// <returns>The address registered for the grain's single activation.</returns> public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address) { registrationsSingleActIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsSingleActLocal.Increment(); // if I am the owner, store the new activation locally Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo); return(returnedAddress == null ? null : returnedAddress.Item1); } else { RegistrationsSingleActRemoteSent.Increment(); // otherwise, notify the owner Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES); // Caching optimization: // cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! if (returnedAddress == null || returnedAddress.Item1 == null) { return(null); } if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo)) { return(returnedAddress.Item1); } var cached = new List <Tuple <SiloAddress, ActivationId> >(1) { Tuple.Create(address.Silo, address.Activation) }; // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2); return(returnedAddress.Item1); } }
public async Task <List <ActivationAddress> > FullLookup(GrainId grain) { fullLookups.Increment(); SiloAddress silo = CalculateTargetSilo(grain, false); // No need to check that silo != null since we're passing excludeThisSiloIfStopping = false if (log.IsVerbose) { log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode()); } // We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()). // We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called. if (silo.Equals(MyAddress)) { LocalDirectoryLookups.Increment(); var localResult = DirectoryPartition.LookUpGrain(grain); if (localResult == null) { // it can happen that we cannot find the grain in our partition if there were // some recent changes in the membership if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}=none", grain); } return(new List <ActivationAddress>()); } var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings()); } LocalDirectorySuccesses.Increment(); return(a); } // Just a optimization. Why sending a message to someone we know is not valid. if (!IsValidSilo(silo)) { throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo)); } RemoteLookupsSent.Increment(); Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES); // update the cache List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList(); List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings()); } if (entries.Count > 0) { DirectoryCache.AddOrUpdate(grain, entries, result.Item2); } return(addresses); }
public async Task <AddressAndTag> LookupAsync(GrainId grainId, int hopCount = 0) { (hopCount > 0 ? RemoteLookupsReceived : fullLookups).Increment(); // see if the owner is somewhere else (returns null if we are owner) var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync"); // on all silos other than first, we insert a retry delay and recheck owner before forwarding if (hopCount > 0 && forwardAddress != null) { await Task.Delay(RETRY_DELAY); forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "LookUpAsync"); if (forwardAddress is object) { int hash = unchecked ((int)grainId.GetUniformHashCode()); this.log.LogWarning($"LookupAsync - It seems we are not the owner of grain {grainId} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})"); } } if (forwardAddress == null) { // we are the owner LocalDirectoryLookups.Increment(); var localResult = DirectoryPartition.LookUpActivation(grainId); if (localResult.Address == null) { // it can happen that we cannot find the grain in our partition if there were // some recent changes in the membership if (log.IsEnabled(LogLevel.Trace)) { log.Trace("FullLookup mine {0}=none", grainId); } localResult.Address = default; localResult.VersionTag = GrainInfo.NO_ETAG; return(localResult); } if (log.IsEnabled(LogLevel.Trace)) { log.Trace("FullLookup mine {0}={1}", grainId, localResult.Address); } LocalDirectorySuccesses.Increment(); return(localResult); } else { // Just a optimization. Why sending a message to someone we know is not valid. if (!IsValidSilo(forwardAddress)) { throw new OrleansException($"Current directory at {MyAddress} is not stable to perform the lookup for grainId {grainId} (it maps to {forwardAddress}, which is not a valid silo). Retry later."); } RemoteLookupsSent.Increment(); var result = await GetDirectoryReference(forwardAddress).LookupAsync(grainId, hopCount + 1); // update the cache if (result.Address is { } address&& IsValidSilo(address.SiloAddress)) { DirectoryCache.AddOrUpdate(address, result.VersionTag); } if (log.IsEnabled(LogLevel.Trace)) { log.Trace("FullLookup remote {0}={1}", grainId, result.Address); } return(result); } }