private async Task RunBatchedActivationRequests(List <string> remoteClusters, List <GrainId> grains)
{
var addresses = new List <ActivationAddress>();
foreach (var grain in grains)
{
// retrieve activation
ActivationAddress address;
int version;
var mcstate = router.DirectoryPartition.TryGetActivation(grain, out address, out version);
// work on the doubtful ones only
if (mcstate == GrainDirectoryEntryStatus.Doubtful)
{
// try to start retry by moving into requested_ownership state
if (router.DirectoryPartition.UpdateClusterRegistrationStatus(grain, address.Activation, GrainDirectoryEntryStatus.RequestedOwnership, GrainDirectoryEntryStatus.Doubtful))
{
addresses.Add(address);
}
}
}
if (addresses.Count == 0)
{
return;
}
var batchResponses = new List <RemoteClusterActivationResponse[]>();
var tasks = remoteClusters.Select(async remotecluster =>
{
// find gateway and send batched request
try
{
var clusterGatewayAddress = this.multiClusterOracle.GetRandomClusterGateway(remotecluster);
var clusterGrainDir = this.grainFactory.GetSystemTarget <IClusterGrainDirectory>(Constants.ClusterDirectoryServiceId, clusterGatewayAddress);
var r = await clusterGrainDir.ProcessActivationRequestBatch(addresses.Select(a => a.Grain).ToArray(), this.config.ClusterId);
batchResponses.Add(r);
}
catch (Exception e)
{
batchResponses.Add(
Enumerable.Repeat <RemoteClusterActivationResponse>(
new RemoteClusterActivationResponse(ActivationResponseStatus.Faulted)
{
ResponseException = e
}, addresses.Count).ToArray());
}
}).ToList();
// wait for all the responses to arrive or fail
await Task.WhenAll(tasks);
if (logger.IsVerbose)
{
foreach (var br in batchResponses)
{
var summary = br.Aggregate(new { Pass = 0, Failed = 0, FailedA = 0, FailedOwned = 0, Faulted = 0 }, (agg, r) =>
{
switch (r.ResponseStatus)
{
case ActivationResponseStatus.Pass:
return(new { Pass = agg.Pass + 1, agg.Failed, agg.FailedA, agg.FailedOwned, agg.Faulted });
case ActivationResponseStatus.Failed:
if (!r.Owned)
{
return(r.ExistingActivationAddress.Address == null
? new { agg.Pass, Failed = agg.Failed + 1, agg.FailedA, agg.FailedOwned, agg.Faulted }
: new { agg.Pass, agg.Failed, FailedA = agg.FailedA + 1, agg.FailedOwned, agg.Faulted });
}
else
{
return(new { agg.Pass, agg.Failed, agg.FailedA, FailedOwned = agg.FailedOwned + 1, agg.Faulted });
}
default:
return(new { agg.Pass, agg.Failed, agg.FailedA, agg.FailedOwned, Faulted = agg.Faulted + 1 });
}
});
logger.Verbose("GSIP:M batchresponse PASS:{0} FAILED:{1} FAILED(a){2}: FAILED(o){3}: FAULTED:{4}",
summary.Pass,
summary.Failed,
summary.FailedA,
summary.FailedOwned,
summary.Faulted);
}
}
// process each address
var loser_activations_per_silo = new Dictionary <SiloAddress, List <ActivationAddress> >();
for (int i = 0; i < addresses.Count; i++)
{
var address = addresses[i];
// array that holds the responses
var responses = new RemoteClusterActivationResponse[remoteClusters.Count];
for (int j = 0; j < batchResponses.Count; j++)
{
responses[j] = batchResponses[j][i];
}
// response processor
var outcomeDetails = GlobalSingleInstanceResponseTracker.GetOutcome(responses, address.Grain, logger);
var outcome = outcomeDetails.State;
if (logger.IsVerbose2)
{
logger.Verbose2("GSIP:M {0} Result={1}", address.Grain, outcomeDetails);
}
switch (outcome)
{
case OutcomeState.RemoteOwner:
case OutcomeState.RemoteOwnerLikely:
{
// record activations that lost and need to be deactivated
List <ActivationAddress> losers;
if (!loser_activations_per_silo.TryGetValue(address.Silo, out losers))
{
loser_activations_per_silo[address.Silo] = losers = new List <ActivationAddress>();
}
losers.Add(address);
router.DirectoryPartition.CacheOrUpdateRemoteClusterRegistration(address.Grain, address.Activation, outcomeDetails.RemoteOwnerAddress.Address);
continue;
}
case OutcomeState.Succeed:
{
var ok = (router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Owned, GrainDirectoryEntryStatus.RequestedOwnership));
if (ok)
{
continue;
}
else
{
break;
}
}
case OutcomeState.Inconclusive:
{
break;
}
}
// we were not successful, reread state to determine what is going on
int version;
var mcstatus = router.DirectoryPartition.TryGetActivation(address.Grain, out address, out version);
// in each case, go back to DOUBTFUL
if (mcstatus == GrainDirectoryEntryStatus.RequestedOwnership)
{
// we failed because of inconclusive answers
var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RequestedOwnership);
if (!success)
{
ProtocolError(address, "unable to transition from REQUESTED_OWNERSHIP to DOUBTFUL");
}
}
else if (mcstatus == GrainDirectoryEntryStatus.RaceLoser)
{
// we failed because an external request moved us to RACE_LOSER
var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(address.Grain, address.Activation, GrainDirectoryEntryStatus.Doubtful, GrainDirectoryEntryStatus.RaceLoser);
if (!success)
{
ProtocolError(address, "unable to transition from RACE_LOSER to DOUBTFUL");
}
}
else
{
ProtocolError(address, "unhandled protocol state");
}
TrackDoubtfulGrain(address.Grain);
}
// remove loser activations
foreach (var kvp in loser_activations_per_silo)
{
var catalog = this.grainFactory.GetSystemTarget <ICatalog>(Constants.CatalogId, kvp.Key);
catalog.DeleteActivations(kvp.Value).Ignore();
}
}
private RemoteClusterActivationResponse ProcessRequestLocal(GrainId grain, string requestClusterId)
{
RemoteClusterActivationResponse response;
//This function will be called only on the Owner silo.
try
{
ActivationAddress address;
int version;
GrainDirectoryEntryStatus existingActivationStatus = router.DirectoryPartition.TryGetActivation(grain, out address, out version);
//Return appropriate protocol response, given current mc status
switch (existingActivationStatus)
{
case GrainDirectoryEntryStatus.Invalid:
response = RemoteClusterActivationResponse.Pass;
break;
case GrainDirectoryEntryStatus.Owned:
response = new RemoteClusterActivationResponse(ActivationResponseStatus.Failed)
{
ExistingActivationAddress = new AddressAndTag()
{
Address = address,
VersionTag = version
},
ClusterId = clusterId,
Owned = true
};
break;
case GrainDirectoryEntryStatus.Cached:
case GrainDirectoryEntryStatus.RaceLoser:
response = RemoteClusterActivationResponse.Pass;
break;
case GrainDirectoryEntryStatus.RequestedOwnership:
case GrainDirectoryEntryStatus.Doubtful:
var iWin = MultiClusterUtils.ActivationPrecedenceFunc(grain, clusterId, requestClusterId);
if (iWin)
{
response = new RemoteClusterActivationResponse(ActivationResponseStatus.Failed)
{
ExistingActivationAddress = new AddressAndTag()
{
Address = address,
VersionTag = version
},
ClusterId = clusterId,
Owned = false
};
}
else
{
response = RemoteClusterActivationResponse.Pass;
//update own activation status to race loser.
if (existingActivationStatus == GrainDirectoryEntryStatus.RequestedOwnership)
{
logger.Trace("GSIP:Rsp {0} Origin={1} RaceLoser", grain.ToString(), requestClusterId);
var success = router.DirectoryPartition.UpdateClusterRegistrationStatus(grain, address.Activation, GrainDirectoryEntryStatus.RaceLoser, GrainDirectoryEntryStatus.RequestedOwnership);
if (!success)
{
// there was a race. retry.
logger.Trace("GSIP:Rsp {0} Origin={1} Retry", grain.ToString(), requestClusterId);
return(ProcessRequestLocal(grain, requestClusterId));
}
}
}
break;
default:
throw new InvalidOperationException("Invalid MultiClusterStatus value");
}
}
catch (Exception ex)
{
//LOG exception
response = new RemoteClusterActivationResponse(ActivationResponseStatus.Faulted)
{
ResponseException = ex
};
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("GSIP:Rsp {0} Origin={1} Result={2}", grain.ToString(), requestClusterId, response);
}
return(response);
}