private async void FinishDestroyActivations(List<ActivationData> list, int number, MultiTaskCompletionSource tcs) { try { //logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Starting FinishDestroyActivations #{0} - with {1} Activations.", number, list.Count); // step 3 - UnregisterManyAsync try { await scheduler.RunOrQueueTask(() => directory.UnregisterManyAsync(list.Select(d => ActivationAddress.GetAddress(LocalSilo, d.Grain, d.ActivationId)).ToList()), SchedulingContext); } catch (Exception exc) { logger.Warn(ErrorCode.Catalog_UnregisterManyAsync, String.Format("UnregisterManyAsync {0} failed.", list.Count), exc); } // step 4 - UnregisterMessageTarget and OnFinishedGrainDeactivate foreach (var activationData in list) { try { lock (activationData) { activationData.SetState(ActivationState.Invalid); // Deactivate calls on this activation are finished } UnregisterMessageTarget(activationData); } catch (Exception exc) { logger.Warn(ErrorCode.Catalog_UnregisterMessageTarget2, String.Format("UnregisterMessageTarget failed on {0}.", activationData), exc); } try { // IMPORTANT: no more awaits and .Ignore after that point. // Just use this opportunity to invalidate local Cache Entry as well. // If this silo is not the grain directory partition for this grain, it may have it in its cache. directory.InvalidateCacheEntry(activationData.Address); RerouteAllQueuedMessages(activationData, null, "Finished Destroy Activation"); } catch (Exception exc) { logger.Warn(ErrorCode.Catalog_UnregisterMessageTarget3, String.Format("Last stage of DestroyActivations failed on {0}.", activationData), exc); } } // step 5 - Resolve any waiting TaskCompletionSource if (tcs != null) { tcs.SetMultipleResults(list.Count); } logger.Info(ErrorCode.Catalog_DestroyActivations_Done, "Done FinishDestroyActivations #{0} - Destroyed {1} Activations.", number, list.Count); }catch (Exception exc) { logger.Error(ErrorCode.Catalog_FinishDeactivateActivation_Exception, String.Format("FinishDestroyActivations #{0} failed with {1} Activations.", number, list.Count), exc); } }
private async void StartDestroyActivations(List<ActivationData> list, MultiTaskCompletionSource tcs = null) { int number = destroyActivationsNumber; destroyActivationsNumber++; try { logger.Info(ErrorCode.Catalog_DestroyActivations, "Starting DestroyActivations #{0} of {1} activations", number, list.Count); // step 1 - WaitForAllTimersToFinish var tasks1 = new List<Task>(); foreach (var activation in list) { tasks1.Add(activation.WaitForAllTimersToFinish()); } try { await Task.WhenAll(tasks1); } catch (Exception exc) { logger.Warn(ErrorCode.Catalog_WaitForAllTimersToFinish_Exception, String.Format("WaitForAllTimersToFinish {0} failed.", list.Count), exc); } // step 2 - CallGrainDeactivate var tasks2 = new List<Tuple<Task, ActivationData>>(); foreach (var activation in list) { var activationData = activation; // Capture loop variable var task = scheduler.RunOrQueueTask(() => CallGrainDeactivateAndCleanupStreams(activationData), new SchedulingContext(activationData)); tasks2.Add(new Tuple<Task, ActivationData>(task, activationData)); } var asyncQueue = new AsyncBatchedContinuationQueue<ActivationData>(); asyncQueue.Queue(tasks2, tupleList => { FinishDestroyActivations(tupleList.Select(t => t.Item2).ToList(), number, tcs); GC.KeepAlive(asyncQueue); // not sure about GC not collecting the asyncQueue local var prematuraly, so just want to capture it here to make sure. Just to be safe. }); } catch (Exception exc) { logger.Warn(ErrorCode.Catalog_DeactivateActivation_Exception, String.Format("StartDestroyActivations #{0} failed with {1} Activations.", number, list.Count), exc); } }
private void DestroyActivationAsync(ActivationData activation, MultiTaskCompletionSource tcs) { StartDestroyActivations(new List<ActivationData> { activation }, tcs); }
/// <summary> /// Forcibly deletes activations now, without waiting for any outstanding transactions to complete. /// Deletes activation immediately regardless of active transactions etc. /// For use by grain delete, transaction abort, etc. /// </summary> /// <param name="list"></param> /// <returns></returns> // Overall code flow: // Deactivating state was already set before, in the correct context under lock. // that means no more new requests will be accepted into this activation and all timer were stopped (no new ticks will be delivered or enqueued) // Wait for all already scheduled ticks to finish // CallGrainDeactivate // when AsyncDeactivate promise is resolved (NOT when all Deactivate turns are done, which may be orphan tasks): // Unregister in the directory // when all AsyncDeactivate turns are done (Dispatcher.OnActivationCompletedRequest): // Set Invalid state // UnregisterMessageTarget -> no new tasks will be enqueue (if an orphan task get enqueud, it is ignored and dropped on the floor). // InvalidateCacheEntry // Reroute pending private Task DestroyActivations(List<ActivationData> list) { var tcs = new MultiTaskCompletionSource(list.Count); StartDestroyActivations(list, tcs); return tcs.Task; }
/// <summary> /// Gracefully deletes activations, putting it into a shutdown state to /// complete and commit outstanding transactions before deleting it. /// To be called not from within Activation context, so can be awaited. /// </summary> /// <param name="list"></param> /// <returns></returns> internal async Task DeactivateActivations(List<ActivationData> list) { if (list == null || list.Count == 0) return; if (logger.IsVerbose) logger.Verbose("DeactivateActivations: {0} activations.", list.Count); List<ActivationData> destroyNow = null; List<MultiTaskCompletionSource> destroyLater = null; int alreadyBeingDestroyed = 0; foreach (var d in list) { var activationData = d; // capture lock (activationData) { if (activationData.State == ActivationState.Valid) { // Change the ActivationData state here, since we're about to give up the lock. activationData.PrepareForDeactivation(); // Don't accept any new messages if (!activationData.IsCurrentlyExecuting) { if (destroyNow == null) { destroyNow = new List<ActivationData>(); } destroyNow.Add(activationData); } else // busy, so destroy later. { if (destroyLater == null) { destroyLater = new List<MultiTaskCompletionSource>(); } var tcs = new MultiTaskCompletionSource(1); destroyLater.Add(tcs); activationData.AddOnInactive(() => DestroyActivationAsync(activationData, tcs)); } } else { alreadyBeingDestroyed++; } } } int numDestroyNow = destroyNow == null ? 0 : destroyNow.Count; int numDestroyLater = destroyLater == null ? 0 : destroyLater.Count; logger.Info(ErrorCode.Catalog_ShutdownActivations_3, "DeactivateActivations: total {0} to shutdown, out of them {1} promptly, {2} later when become idle and {3} are already being destroyed or invalid.", list.Count, numDestroyNow, numDestroyLater, alreadyBeingDestroyed); CounterStatistic.FindOrCreate(StatisticNames.CATALOG_ACTIVATION_SHUTDOWN_VIA_DIRECT_SHUTDOWN).IncrementBy(list.Count); if (destroyNow != null && destroyNow.Count > 0) { await DestroyActivations(destroyNow); } if (destroyLater != null && destroyLater.Count > 0) { await Task.WhenAll(destroyLater.Select(t => t.Task).ToArray()); } }