private async Task RunBatchedActivationRequests(List<string> remoteClusters, List<GrainId> grains)
        {
            var addresses = new List<ActivationAddress>();

            foreach (var grain in grains)
            {
                // retrieve activation
                ActivationAddress address;
                int version;
                var mcstate = router.DirectoryPartition.TryGetActivation(grain, out address, out version);

                // work on the doubtful ones only
                if (mcstate == GrainDirectoryEntryStatus.Doubtful)
                {
                    // try to start retry by moving into requested_ownership state
                    if (router.DirectoryPartition.UpdateClusterRegistrationStatus(grain, address.Activation, GrainDirectoryEntryStatus.RequestedOwnership, GrainDirectoryEntryStatus.Doubtful))
                    {
                        addresses.Add(address);
                    }
                }
            }

            if (addresses.Count == 0)
                return;

            var batchResponses = new List<RemoteClusterActivationResponse[]>();

            var tasks = remoteClusters.Select(async remotecluster =>
            {
                // find gateway
                var gossiporacle = Silo.CurrentSilo.LocalMultiClusterOracle;

                // send batched request
                try
                {
                    var clusterGatewayAddress = gossiporacle.GetRandomClusterGateway(remotecluster);
                    var clusterGrainDir = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget<IClusterGrainDirectory>(Constants.ClusterDirectoryServiceId, clusterGatewayAddress);
                    var r = await clusterGrainDir.ProcessActivationRequestBatch(addresses.Select(a => a.Grain).ToArray(), Silo.CurrentSilo.ClusterId);
                    batchResponses.Add(r);
                }
                catch (Exception e)
                {
                    batchResponses.Add(
                        Enumerable.Repeat<RemoteClusterActivationResponse>(
                           new RemoteClusterActivationResponse(ActivationResponseStatus.Faulted)
                           {
                               ResponseException = e
                           }, addresses.Count).ToArray());
                }

            }).ToList();
 
            // wait for all the responses to arrive or fail
            await Task.WhenAll(tasks);

            if (logger.IsVerbose)
            { 
                foreach (var br in batchResponses)
                {
                    var summary = br.Aggregate(new { Pass = 0, Failed = 0, FailedA = 0, FailedOwned = 0, Faulted = 0 }, (agg, r) =>
                    {
                        switch (r.ResponseStatus)
                        {
                            case ActivationResponseStatus.Pass:
                                return new { Pass = agg.Pass + 1, agg.Failed, agg.FailedA, agg.FailedOwned, agg.Faulted };
                            case ActivationResponseStatus.Failed:
                                if (!r.Owned)
                                {
                                    return r.ExistingActivationAddress.Address == null
                                        ? new { agg.Pass, Failed = agg.Failed + 1, agg.FailedA, agg.FailedOwned, agg.Faulted }
                                        : new { agg.Pass, agg.Failed, FailedA = agg.FailedA + 1, agg.FailedOwned, agg.Faulted };
                                }
                                else
                                {
                                    return new { agg.Pass, agg.Failed, agg.FailedA, FailedOwned = agg.FailedOwned + 1, agg.Faulted };
                                }
                            default:
                                return new { agg.Pass, agg.Failed, agg.FailedA, agg.FailedOwned, Faulted = agg.Faulted + 1 };
                        }
                    });
                    logger.Verbose("GSIP:M batchresponse PASS:{0} FAILED:{1} FAILED(a){2}: FAILED(o){3}: FAULTED:{4}",
                        summary.Pass,
                        summary.Failed,
                        summary.FailedA,
                        summary.FailedOwned,
                        summary.Faulted);
                }
            }

            // process each address

            var loser_activations_per_silo = new Dictionary<SiloAddress, List<ActivationAddress>>();

            for (int i = 0; i < addresses.Count; i++)
            {
                var address = addresses[i];

                // array that holds the responses
                var responses = new RemoteClusterActivationResponse[remoteClusters.Count];

                for (int j = 0; j < batchResponses.Count; j++)
                    responses[j] = batchResponses[j][i];

                // response processor
                var outcomeDetails = GlobalSingleInstanceResponseTracker.GetOutcome(responses, address.Grain, logger);
                var outcome = outcomeDetails.State;

                if (logger.IsVerbose2)
                    logger.Verbose2("GSIP:M {0} Result={1}", address.Grain, outcomeDetails);

                switch (outcome)
                {
                    case OutcomeState.RemoteOwner:
                    case OutcomeState.RemoteOwnerLikely:
                    {
                        // record activations that lost and need to be deactivated
                        List<ActivationAddress> losers;
                        if (!loser_activations_per_silo.TryGetValue(address.Silo, out losers))
                            loser_activations_per_silo[address.Silo] = losers = new List<ActivationAddress>();
                        losers.Add(address);

                        router.DirectoryPartition.CacheOrUpdateRemoteClusterRegistration(address.Grain, address.Activation, outcomeDetails.RemoteOwnerAddress.Address);
                        continue;
                    }
                    case OutcomeState.Succeed:
                    {
                        var ok = (router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Owned, GrainDirectoryEntryStatus.RequestedOwnership));
                        if (ok)
                            continue;
                        else
                            break;
                    }
                    case OutcomeState.Inconclusive:
                    {
                        break;
                    }
                }

                // we were not successful, reread state to determine what is going on
                int version;
                var mcstatus = router.DirectoryPartition.TryGetActivation(address.Grain, out address, out version);

                // in each case, go back to DOUBTFUL
                if (mcstatus == GrainDirectoryEntryStatus.RequestedOwnership)
                {
                    // we failed because of inconclusive answers
                    var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RequestedOwnership);
                    if (!success) ProtocolError(address, "unable to transition from REQUESTED_OWNERSHIP to DOUBTFUL");
                }
                else if (mcstatus == GrainDirectoryEntryStatus.RaceLoser)
                {
                    // we failed because an external request moved us to RACE_LOSER
                    var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RaceLoser);
                    if (!success) ProtocolError(address, "unable to transition from RACE_LOSER to DOUBTFUL");
                }
                else
                {
                    ProtocolError(address, "unhandled protocol state");
                }

                TrackDoubtfulGrain(address.Grain);
            }

            // remove loser activations
            foreach (var kvp in loser_activations_per_silo)
            {
                var catalog = InsideRuntimeClient.Current.InternalGrainFactory.GetSystemTarget<ICatalog>(Constants.CatalogId, kvp.Key);
                catalog.DeleteActivations(kvp.Value).Ignore();
            }
        }
        private RemoteClusterActivationResponse ProcessRequestLocal(GrainId grain, string requestClusterId)
        {
            RemoteClusterActivationResponse response;

            //This function will be called only on the Owner silo.
            try
            {
                ActivationAddress address;
                int version;
                GrainDirectoryEntryStatus existingActivationStatus = router.DirectoryPartition.TryGetActivation(grain, out address, out version);


                //Return appropriate protocol response, given current mc status   
                switch (existingActivationStatus)
                {
                    case GrainDirectoryEntryStatus.Invalid:
                        response = RemoteClusterActivationResponse.Pass;
                        break;

                    case GrainDirectoryEntryStatus.Owned:
                        response = new RemoteClusterActivationResponse(ActivationResponseStatus.Failed)
                        {
                            ExistingActivationAddress = new AddressAndTag()
                            {
                                Address = address,
                                VersionTag = version
                            },
                            ClusterId = clusterId,
                            Owned = true
                        };
                        break;

                    case GrainDirectoryEntryStatus.Cached:
                    case GrainDirectoryEntryStatus.RaceLoser:
                        response = RemoteClusterActivationResponse.Pass;
                        break;

                    case GrainDirectoryEntryStatus.RequestedOwnership:
                    case GrainDirectoryEntryStatus.Doubtful:
                        var iWin = MultiClusterUtils.ActivationPrecedenceFunc(grain, clusterId, requestClusterId);
                        if (iWin)
                        {
                            response = new RemoteClusterActivationResponse(ActivationResponseStatus.Failed)
                            {
                                ExistingActivationAddress = new AddressAndTag()
                                {
                                    Address = address,
                                    VersionTag = version
                                },
                                ClusterId = clusterId,
                                Owned = false
                            };
                        }
                        else
                        {
                            response = RemoteClusterActivationResponse.Pass;
                            //update own activation status to race loser.
                            if (existingActivationStatus == GrainDirectoryEntryStatus.RequestedOwnership)
                            {
                                logger.Verbose2("GSIP:Rsp {0} Origin={1} RaceLoser", grain.ToString(), requestClusterId);
                                var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(grain, address.Activation, GrainDirectoryEntryStatus.RaceLoser, GrainDirectoryEntryStatus.RequestedOwnership);
                                if (!success)
                                {
                                    // there was a race. retry.
                                    logger.Verbose2("GSIP:Rsp {0} Origin={1} Retry", grain.ToString(), requestClusterId);
                                    return ProcessRequestLocal(grain, requestClusterId);
                                }
                            }
                        }
                        break;
                    default:
                        throw new InvalidOperationException("Invalid MultiClusterStatus value");

                }
            }
            catch (Exception ex)
            {
                //LOG exception
                response = new RemoteClusterActivationResponse(ActivationResponseStatus.Faulted)
                {
                    ResponseException = ex
                };
            }

            if (logger.IsVerbose)
                logger.Verbose("GSIP:Rsp {0} Origin={1} Result={2}", grain.ToString(), requestClusterId, response);

            return response;
        }
 /// <summary>
 /// Gets the outcome for a full round of responses from all the clusters.
 /// </summary>
 /// <param name="responses">Responses for a particular grain from all of the clusters in the multi-cluster network</param>
 /// <param name="grainId">The ID of the grain that we want to know its owner status</param>
 /// <param name="logger">The logger in case there is useful information to log.</param>
 /// <returns>The outcome of aggregating all of the responses.</returns>
 public static GlobalSingleInstanceResponseOutcome GetOutcome(RemoteClusterActivationResponse[] responses, GrainId grainId, Logger logger)
 {
     if (responses.Any(t => t == null)) throw new ArgumentException("All responses should have a value", nameof(responses));
     return GetOutcome(responses, grainId, logger, hasPendingResponses: false).Value;
 }