Example #1
0
        // TODO move this logic in the LocalGrainDirectory
        private void OnSiloStatusChange(SiloAddress updatedSilo, SiloStatus status)
        {
            // ignore joining events and also events on myself.
            if (updatedSilo.Equals(LocalSilo))
            {
                return;
            }

            // We deactivate those activations when silo goes either of ShuttingDown/Stopping/Dead states,
            // since this is what Directory is doing as well. Directory removes a silo based on all those 3 statuses,
            // thus it will only deliver a "remove" notification for a given silo once to us. Therefore, we need to react the fist time we are notified.
            // We may review the directory behavior in the future and treat ShuttingDown differently ("drain only") and then this code will have to change a well.
            if (!status.IsTerminating())
            {
                return;
            }
            if (status == SiloStatus.Dead)
            {
                this.RuntimeClient.BreakOutstandingMessagesToDeadSilo(updatedSilo);
            }

            var activationsToShutdown = new List <IGrainContext>();

            try
            {
                // scan all activations in activation directory and deactivate the ones that the removed silo is their primary partition owner.
                lock (activations)
                {
                    foreach (var activation in activations)
                    {
                        try
                        {
                            var activationData = activation.Value;
                            if (!activationData.PlacementStrategy.IsUsingGrainDirectory || grainDirectoryResolver.HasNonDefaultDirectory(activationData.GrainId.Type))
                            {
                                continue;
                            }
                            if (!updatedSilo.Equals(directory.GetPrimaryForGrain(activationData.GrainId)))
                            {
                                continue;
                            }

                            activationsToShutdown.Add(activationData);
                        }
                        catch (Exception exc)
                        {
                            logger.LogError(
                                (int)ErrorCode.Catalog_SiloStatusChangeNotification_Exception,
                                exc,
                                "Catalog has thrown an exception while handling removal of silo {Silo}", updatedSilo.ToStringWithHashCode());
                        }
                    }
                }

                logger.LogInformation(
                    (int)ErrorCode.Catalog_SiloStatusChangeNotification,
                    "Catalog is deactivating {Count} activations due to a failure of silo {Silo}, since it is a primary directory partition to these grain ids.",
                    activationsToShutdown.Count,
                    updatedSilo.ToStringWithHashCode());
            }
            finally
            {
                // outside the lock.
                if (activationsToShutdown.Count > 0)
                {
                    DeactivateActivations(activationsToShutdown).Ignore();
                }
            }
        }