private async void FinishDestroyActivations(List<ActivationData> list, int number, MultiTaskCompletionSource tcs)
{
try
{
//logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Starting FinishDestroyActivations #{0} - with {1} Activations.", number, list.Count);
// step 3 - UnregisterManyAsync
try
{
await scheduler.RunOrQueueTask(() =>
directory.UnregisterManyAsync(list.Select(d => ActivationAddress.GetAddress(LocalSilo, d.Grain, d.ActivationId)).ToList()),
SchedulingContext);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_UnregisterManyAsync, String.Format("UnregisterManyAsync {0} failed.", list.Count), exc);
}
// step 4 - UnregisterMessageTarget and OnFinishedGrainDeactivate
foreach (var activationData in list)
{
try
{
lock (activationData)
{
activationData.SetState(ActivationState.Invalid); // Deactivate calls on this activation are finished
}
UnregisterMessageTarget(activationData);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_UnregisterMessageTarget2, String.Format("UnregisterMessageTarget failed on {0}.", activationData), exc);
}
try
{
// IMPORTANT: no more awaits and .Ignore after that point.
// Just use this opportunity to invalidate local Cache Entry as well.
// If this silo is not the grain directory partition for this grain, it may have it in its cache.
directory.InvalidateCacheEntry(activationData.Address);
RerouteAllQueuedMessages(activationData, null, "Finished Destroy Activation");
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_UnregisterMessageTarget3, String.Format("Last stage of DestroyActivations failed on {0}.", activationData), exc);
}
}
// step 5 - Resolve any waiting TaskCompletionSource
if (tcs != null)
{
tcs.SetMultipleResults(list.Count);
}
logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Done FinishDestroyActivations #{0} - Destroyed {1} Activations.", number, list.Count);
}catch (Exception exc)
{
logger.Error(ErrorCode.Catalog_FinishDeactivateActivation_Exception, String.Format("FinishDestroyActivations #{0} failed with {1} Activations.", number, list.Count), exc);
}
}
private async void StartDestroyActivations(List<ActivationData> list, MultiTaskCompletionSource tcs = null)
{
int number = destroyActivationsNumber;
destroyActivationsNumber++;
try
{
logger.Info(ErrorCode.Catalog_DestroyActivations, "Starting DestroyActivations #{0} of {1} activations", number, list.Count);
// step 1 - WaitForAllTimersToFinish
var tasks1 = new List<Task>();
foreach (var activation in list)
{
tasks1.Add(activation.WaitForAllTimersToFinish());
}
try
{
await Task.WhenAll(tasks1);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_WaitForAllTimersToFinish_Exception, String.Format("WaitForAllTimersToFinish {0} failed.", list.Count), exc);
}
// step 2 - CallGrainDeactivate
var tasks2 = new List<Tuple<Task, ActivationData>>();
foreach (var activation in list)
{
var activationData = activation; // Capture loop variable
var task = scheduler.RunOrQueueTask(() => CallGrainDeactivateAndCleanupStreams(activationData), new SchedulingContext(activationData));
tasks2.Add(new Tuple<Task, ActivationData>(task, activationData));
}
var asyncQueue = new AsyncBatchedContinuationQueue<ActivationData>();
asyncQueue.Queue(tasks2, tupleList =>
{
FinishDestroyActivations(tupleList.Select(t => t.Item2).ToList(), number, tcs);
GC.KeepAlive(asyncQueue); // not sure about GC not collecting the asyncQueue local var prematuraly, so just want to capture it here to make sure. Just to be safe.
});
}
catch (Exception exc)
{
logger.Warn(ErrorCode.Catalog_DeactivateActivation_Exception, String.Format("StartDestroyActivations #{0} failed with {1} Activations.", number, list.Count), exc);
}
}
private void DestroyActivationAsync(ActivationData activation, MultiTaskCompletionSource tcs)
{
StartDestroyActivations(new List<ActivationData> { activation }, tcs);
}
/// <summary>
/// Forcibly deletes activations now, without waiting for any outstanding transactions to complete.
/// Deletes activation immediately regardless of active transactions etc.
/// For use by grain delete, transaction abort, etc.
/// </summary>
/// <param name="list"></param>
/// <returns></returns>
// Overall code flow:
// Deactivating state was already set before, in the correct context under lock.
// that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued)
// Wait for all already scheduled ticks to finish
// CallGrainDeactivate
// when AsyncDeactivate promise is resolved (NOT when all Deactivate turns are done, which may be orphan tasks):
// Unregister in the directory
// when all AsyncDeactivate turns are done (Dispatcher.OnActivationCompletedRequest):
// Set Invalid state
// UnregisterMessageTarget -> no new tasks will be enqueue (if an orphan task get enqueud, it is ignored and dropped on the floor).
// InvalidateCacheEntry
// Reroute pending
private Task DestroyActivations(List<ActivationData> list)
{
var tcs = new MultiTaskCompletionSource(list.Count);
StartDestroyActivations(list, tcs);
return tcs.Task;
}
/// <summary>
/// Gracefully deletes activations, putting it into a shutdown state to
/// complete and commit outstanding transactions before deleting it.
/// To be called not from within Activation context, so can be awaited.
/// </summary>
/// <param name="list"></param>
/// <returns></returns>
internal async Task DeactivateActivations(List<ActivationData> list)
{
if (list == null || list.Count == 0) return;
if (logger.IsVerbose) logger.Verbose("DeactivateActivations: {0} activations.", list.Count);
List<ActivationData> destroyNow = null;
List<MultiTaskCompletionSource> destroyLater = null;
int alreadyBeingDestroyed = 0;
foreach (var d in list)
{
var activationData = d; // capture
lock (activationData)
{
if (activationData.State == ActivationState.Valid)
{
// Change the ActivationData state here, since we're about to give up the lock.
activationData.PrepareForDeactivation(); // Don't accept any new messages
if (!activationData.IsCurrentlyExecuting)
{
if (destroyNow == null)
{
destroyNow = new List<ActivationData>();
}
destroyNow.Add(activationData);
}
else // busy, so destroy later.
{
if (destroyLater == null)
{
destroyLater = new List<MultiTaskCompletionSource>();
}
var tcs = new MultiTaskCompletionSource(1);
destroyLater.Add(tcs);
activationData.AddOnInactive(() => DestroyActivationAsync(activationData, tcs));
}
}
else
{
alreadyBeingDestroyed++;
}
}
}
int numDestroyNow = destroyNow == null ? 0 : destroyNow.Count;
int numDestroyLater = destroyLater == null ? 0 : destroyLater.Count;
logger.Info(ErrorCode.Catalog_ShutdownActivations_3,
"DeactivateActivations: total {0} to shutdown, out of them {1} promptly, {2} later when become idle and {3} are already being destroyed or invalid.",
list.Count, numDestroyNow, numDestroyLater, alreadyBeingDestroyed);
CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_SHUTDOWN_VIA_DIRECT_SHUTDOWN).IncrementBy(list.Count);
if (destroyNow != null && destroyNow.Count > 0)
{
await DestroyActivations(destroyNow);
}
if (destroyLater != null && destroyLater.Count > 0)
{
await Task.WhenAll(destroyLater.Select(t => t.Task).ToArray());
}
}