/// <summary> /// Called from generated code. /// </summary> protected Task <T> InvokeMethodAsync <T>(int methodId, object[] arguments, InvokeMethodOptions options = InvokeMethodOptions.None, SiloAddress silo = null) { object[] argsDeepCopy = null; if (arguments != null) { CheckForGrainArguments(arguments); SetGrainCancellationTokensTarget(arguments, this); argsDeepCopy = (object[])SerializationManager.DeepCopy(arguments); } var request = new InvokeMethodRequest(this.InterfaceId, methodId, argsDeepCopy); if (IsUnordered) { options |= InvokeMethodOptions.Unordered; } Task <object> resultTask = InvokeMethod_Impl(request, null, options); if (resultTask == null) { if (typeof(T) == typeof(object)) { // optimize for most common case when using one way calls. return(PublicOrleansTaskExtensions.CompletedTask as Task <T>); } return(Task.FromResult(default(T))); } resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask); return(resultTask.Unbox <T>()); }
private async Task RemoveQueues(List <QueueId> queuesToRemove) { if (queuesToRemove.Count == 0) { return; } // Stop the agents that for queues that are not in my range anymore. var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count); Log(ErrorCode.PersistentStreamPullingManager_10, "About to remove {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToString())); var removeTasks = new List <Task>(); foreach (var queueId in queuesToRemove) { PersistentStreamPullingAgent agent; if (!queuesToAgentsMap.TryGetValue(queueId, out agent)) { continue; } agents.Add(agent); queuesToAgentsMap.Remove(queueId); var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>(); var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown); task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11, String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId)); removeTasks.Add(task); } try { await Task.WhenAll(removeTasks); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } foreach (var agent in agents) { try { providerRuntime.UnregisterSystemTarget(agent); } catch (Exception exc) { Log(ErrorCode.PersistentStreamPullingManager_12, "Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", ((ISystemTargetBase)agent).GrainId, exc.Message); } } if (agents.Count > 0) { Log(ErrorCode.PersistentStreamPullingManager_10, "Removed {0} queues: {1}. Now own total of {2} queues: {3}", agents.Count, Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()), NumberRunningAgents, PrintQueues(queuesToAgentsMap.Keys)); } }
private void InitializeInternal() { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), Silo, QueueId.ToStringWithHashCode()); lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(this.options.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = ThreadSafeRandom.NextTimeSpan(this.options.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
private void InternalUnobservedTaskExceptionHandler(object sender, UnobservedTaskExceptionEventArgs e) { var aggrException = e.Exception; var baseException = aggrException.GetBaseException(); var tplTask = (Task)sender; var contextObj = tplTask.AsyncState; var context = contextObj as ISchedulingContext; try { UnobservedExceptionHandler(context, baseException); } finally { if (e.Observed) { logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an UnobservedTaskException which was successfully observed and recovered from. BaseException = {0}. Exception = {1}", baseException.Message, LogFormatter.PrintException(aggrException)); } else { var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}", OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException)); logger.Error(ErrorCode.Runtime_Error_100005, errorStr); logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version."); } } }
/// <summary> /// Called from generated code. /// </summary> protected async Task <T> InvokeMethodAsync <T>(int methodId, object[] arguments, InvokeMethodOptions options = InvokeMethodOptions.None, SiloAddress silo = null) { object[] argsDeepCopy = null; if (arguments != null) { CheckForGrainArguments(arguments); argsDeepCopy = (object[])SerializationManager.DeepCopy(arguments); } var request = new InvokeMethodRequest(this.InterfaceId, methodId, argsDeepCopy); if (IsUnordered) { options |= InvokeMethodOptions.Unordered; } Task <object> resultTask = InvokeMethod_Impl(request, null, options); if (resultTask == null) { return(default(T)); } resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask); return((T)await resultTask); }
public async Task LocalhostSiloTest() { using var portAllocator = new TestClusterPortAllocator(); var(siloPort, gatewayPort) = portAllocator.AllocateConsecutivePortPairs(1); var silo = new SiloHostBuilder() .AddMemoryGrainStorage("MemoryStore") .UseLocalhostClustering(siloPort, gatewayPort) .Build(); var client = new ClientBuilder() .UseLocalhostClustering(gatewayPort) .Build(); try { await silo.StartAsync(); await client.Connect(); var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid()); var result = await grain.Echo("test"); Assert.Equal("test", result); } finally { await OrleansTaskExtentions.SafeExecute(() => silo.StopAsync()); await OrleansTaskExtentions.SafeExecute(() => client.Close()); Utils.SafeExecute(() => silo.Dispose()); Utils.SafeExecute(() => client.Close()); } }
/// <inheritdoc /> public Task <T> InvokeMethodAsync <T>(GrainReference reference, int methodId, object[] arguments, InvokeMethodOptions options, SiloAddress silo) { if (arguments != null) { CheckForGrainArguments(arguments); SetGrainCancellationTokensTarget(arguments, reference); this.serializationManager.DeepCopyElementsInPlace(arguments); } var request = new InvokeMethodRequest(reference.InterfaceId, reference.InterfaceVersion, methodId, arguments); if (IsUnordered(reference)) { options |= InvokeMethodOptions.Unordered; } Task <object> resultTask = InvokeMethod_Impl(reference, request, options); if (resultTask == null) { if (typeof(T) == typeof(object)) { // optimize for most common case when using one way calls. return(OrleansTaskExtentions.CompletedTask as Task <T>); } return(Task.FromResult(default(T))); } #if !NETCOREAPP resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask); #endif return(resultTask.ToTypedTask <T>()); }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } if (failureHandler.Value == null) { throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); } logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter.Value; streamFailureHandler = failureHandler.Value; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc); return; } try { if (queueAdapterCache.Value != null) { queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc); return; } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode())); await task; } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod); timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
public async Task LocalhostSiloTest() { var opts = TestSiloSpecificOptions.Create(new TestClusterOptions(), 1, true); var silo = new SiloHostBuilder() .AddMemoryGrainStorage("MemoryStore") .UseLocalhostClustering(opts.SiloPort, opts.GatewayPort) .Build(); var client = new ClientBuilder() .UseLocalhostClustering(opts.GatewayPort) .Build(); try { await silo.StartAsync(); await client.Connect(); var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid()); var result = await grain.Echo("test"); Assert.Equal("test", result); } finally { await OrleansTaskExtentions.SafeExecute(() => silo.StopAsync()); await OrleansTaskExtentions.SafeExecute(() => client.Close()); Utils.SafeExecute(() => silo.Dispose()); Utils.SafeExecute(() => client.Close()); } }
private void InternalUnobservedTaskExceptionHandler(object sender, System.Threading.Tasks.UnobservedTaskExceptionEventArgs e) { var aggrException = e.Exception; var baseException = aggrException.GetBaseException(); var context = (sender as Task)?.AsyncState; try { this.logger.LogError((int)ErrorCode.Runtime_Error_100104, baseException, "Silo caught an unobserved exception thrown from context {Context}: {Exception}", context, baseException); } finally { if (e.Observed) { logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an unobserved exception which was successfully observed and recovered from. BaseException = {0}. Exception = {1}", baseException.Message, LogFormatter.PrintException(aggrException)); } else { var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}", OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException)); logger.Error(ErrorCode.Runtime_Error_100005, errorStr); logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version."); } } }
private async Task <bool> ErrorProtocol(StreamConsumerData consumerData, Exception exceptionOccured, bool isDeliveryError, IBatchContainer batch, StreamSequenceToken token) { // for loss of client, we just remove the subscription if (exceptionOccured is ClientNotAvailableException) { logger.LogWarning( (int)ErrorCode.Stream_ConsumerIsDead, "Consumer {Consumer} on stream {StreamId} is no longer active - permanently removing Consumer.", consumerData.StreamConsumer, consumerData.StreamId); pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId).Ignore(); return(true); } // notify consumer about the error or that the data is not available. await OrleansTaskExtentions.ExecuteAndIgnoreException( () => DeliverErrorToConsumer( consumerData, exceptionOccured, batch)); // record that there was a delivery failure if (isDeliveryError) { await OrleansTaskExtentions.ExecuteAndIgnoreException( () => streamFailureHandler.OnDeliveryFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token)); } else { await OrleansTaskExtentions.ExecuteAndIgnoreException( () => streamFailureHandler.OnSubscriptionFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token)); } // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { try { // notify consumer of faulted subscription, if we can. await OrleansTaskExtentions.ExecuteAndIgnoreException( () => DeliverErrorToConsumer( consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch)); // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId); } finally { // remove subscription RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); } return(true); } return(false); }
/// <summary> /// Take responsibility for a set of new queues that were assigned to me via a new range. /// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents. /// ERROR HANDLING: /// The responsibility to handle initialization and shutdown failures is inside the Agents code. /// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded. /// Same applies to shutdown. /// </summary> /// <param name="myQueues"></param> /// <param name="failOnInit"></param> /// <returns></returns> private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit) { // Create agents for queues in range that we don't yet have. // First create them and store in local queuesToAgentsMap. // Only after that Initialize them all. var agents = new List <PersistentStreamPullingAgent>(); foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId))) { try { var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE); var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout); providerRuntime.RegisterSystemTarget(agent); queuesToAgentsMap.Add(queueId, agent); agents.Add(agent); } catch (Exception exc) { logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc); // What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down. // If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers. if (failOnInit) { throw; } } } try { var initTasks = new List <Task>(); foreach (var agent in agents) { // Init the agent only after it was registered locally. var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>(); var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null); // Need to call it as a grain reference. var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId)); initTasks.Add(task); } await Task.WhenAll(initTasks); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count, Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode())); }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initiliaze again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <param name="qAdapter"></param> /// <param name="queueAdapterCache"></param> /// <param name="failureHandler"></param> /// <returns></returns> public Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler) { if (qAdapter.Value == null) { throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null"); } if (failureHandler.Value == null) { throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null"); } return(OrleansTaskExtentions.WrapInTask(() => InitializeInternal(qAdapter.Value, queueAdapterCache.Value, failureHandler.Value))); }
private async Task InitAgent(PersistentStreamPullingAgent agent) { // Init the agent only after it was registered locally. var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>(); var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null); IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId); // Need to call it as a grain reference. var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable())); await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId)); }
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler) { logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.", GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode()); // Remove cast once we cleanup queueAdapter = qAdapter; streamFailureHandler = failureHandler; lastTimeCleanedPubSubCache = DateTime.UtcNow; try { receiver = queueAdapter.CreateReceiver(QueueId); } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc); throw; } try { if (queueAdapterCache != null) { queueCache = queueAdapterCache.CreateQueueCache(QueueId); } } catch (Exception exc) { logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc); throw; } try { receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout)) .LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize."); receiverInitTask.Ignore(); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Initialize. No need to log again. } // Setup a reader for a new receiver. // Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod); timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod); logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode()); }
private async Task RemoveQueues(IEnumerable <QueueId> myQueues) { // Stop the agents that for queues that are not in my range anymore. List <QueueId> queuesToRemove = queuesToAgentsMap.Keys.Where(queueId => !myQueues.Contains(queueId)).ToList(); var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count); logger.Info((int)ErrorCode.PersistentStreamPullingManager_10, "Removing {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToStringWithHashCode())); var removeTasks = new List <Task>(); foreach (var queueId in queuesToRemove) { PersistentStreamPullingAgent agent; if (!queuesToAgentsMap.TryGetValue(queueId, out agent)) { continue; } agents.Add(agent); queuesToAgentsMap.Remove(queueId); var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>(); var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown); task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11, String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId)); removeTasks.Add(task); } try { await Task.WhenAll(removeTasks); } catch { // Just ignore this exception and proceed as if Initialize has succeeded. // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } foreach (var agent in agents) { try { providerRuntime.UnRegisterSystemTarget(agent); } catch (Exception exc) { logger.Info((int)ErrorCode.PersistentStreamPullingManager_12, "Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", agent.GrainId, exc.Message); } } }
public async Task Shutdown() { // Stop pulling from queues that are not in my range anymore. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode()); if (timer != null) { var tmp = timer; timer = null; Utils.SafeExecute(tmp.Dispose); } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(config.InitQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07, String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId)); await task; } catch { // Just ignore this exception and proceed as if Shutdown has succeeded. // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } var unregisterTasks = new List <Task>(); var meAsStreamProducer = this.AsReference <IStreamProducerExtension>(); foreach (var tuple in pubSubCache) { tuple.Value.DisposeAll(logger); var streamId = tuple.Key; logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId); unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer)); } try { await Task.WhenAll(unregisterTasks); } catch (Exception exc) { logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08, "Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc); } pubSubCache.Clear(); IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix)); //IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix)); }
public async Task Shutdown() { // Stop pulling from queues that are not in my range anymore. logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", this.GetType().Name, QueueId.ToStringWithHashCode()); if (timer != null) { var tmp = timer; timer = null; tmp.Dispose(); } var unregisterTasks = new List <Task>(); var meAsStreamProducer = StreamProducerExtensionFactory.Cast(this.AsReference()); foreach (var streamId in pubSubCache.Keys) { logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId); unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer)); } try { var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(initQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07, String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId)); await task; } catch (Exception) { // Just ignore this exception and proceed as if Shutdown has succeeded. // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } try { await Task.WhenAll(unregisterTasks); } catch (Exception exc) { logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08, "Failed to unregister myself as stream producer to some streams taht used to be in my responsibility.", exc); } }
private async Task <bool> ErrorProtocol(StreamConsumerData consumerData, Exception exceptionOccured, bool isDeliveryError, IBatchContainer batch, StreamSequenceToken token) { // notify consumer about the error or that the data is not available. await OrleansTaskExtentions.ExecuteAndIgnoreException( () => DeliverErrorToConsumer( consumerData, exceptionOccured, batch)); // record that there was a delivery failure if (isDeliveryError) { await OrleansTaskExtentions.ExecuteAndIgnoreException( () => streamFailureHandler.OnDeliveryFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token)); } else { await OrleansTaskExtentions.ExecuteAndIgnoreException( () => streamFailureHandler.OnSubscriptionFailure( consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token)); } // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { try { // notify consumer of faulted subscription, if we can. await OrleansTaskExtentions.ExecuteAndIgnoreException( () => DeliverErrorToConsumer( consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch)); // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId); } finally { // remove subscription RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); } return(true); } return(false); }
public Task Cancel(Guid id, CancellationTokenSource tokenSource, ConcurrentDictionary <GrainId, GrainReference> grainReferences) { // propagate the exception from the _cancellationTokenSource.Cancel back to the caller // but also cancel _targetGrainReferences. Task task = OrleansTaskExtentions.WrapInTask(tokenSource.Cancel); if (grainReferences.IsEmpty) { return(task); } var cancellationTasks = grainReferences .Select(pair => pair.Value.AsReference <ICancellationSourcesExtension>()) .Select(tokenExtension => CancelTokenWithRetries(id, tokenExtension)) .ToList(); cancellationTasks.Add(task); return(Task.WhenAll(cancellationTasks)); }
// Called by rendezvous when new remote subscriber subscribes to this stream. private async Task AddSubscriber_Impl( GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, StreamSequenceToken token, IStreamFilterPredicateWrapper filter) { IQueueCacheCursor cursor = null; StreamSequenceToken requestedToken = null; // if not cache, then we can't get cursor and there is no reason to ask consumer for token. if (queueCache != null) { DataNotAvailableException errorOccured = null; try { requestedToken = await streamConsumer.GetSequenceToken(subscriptionId); // Set cursor if not cursor is set, or if subscription provides new token requestedToken = requestedToken ?? token; if (requestedToken != null) { cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, requestedToken); } } catch (DataNotAvailableException dataNotAvailableException) { errorOccured = dataNotAvailableException; } if (errorOccured != null) { // notify consumer that the data is not available, if we can. await OrleansTaskExtentions.ExecuteAndIgnoreException(() => streamConsumer.ErrorInStream(subscriptionId, errorOccured)); } } AddSubscriberToSubscriptionCache(subscriptionId, streamId, streamConsumer, cursor, requestedToken, filter); }
public async Task LocalhostSiloTest() { using var portAllocator = new TestClusterPortAllocator(); var(siloPort, gatewayPort) = portAllocator.AllocateConsecutivePortPairs(1); var host = new HostBuilder().UseOrleans(siloBuilder => { siloBuilder.AddMemoryGrainStorage("MemoryStore") .UseLocalhostClustering(siloPort, gatewayPort); }).Build(); var clientHost = new HostBuilder().UseOrleansClient(clientBuilder => { clientBuilder.UseLocalhostClustering(gatewayPort); }).Build(); var client = clientHost.Services.GetRequiredService <IClusterClient>(); try { await Task.WhenAll(host.StartAsync(), clientHost.StartAsync()); var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid()); var result = await grain.Echo("test"); Assert.Equal("test", result); } finally { await OrleansTaskExtentions.SafeExecute(() => host.StopAsync()); await OrleansTaskExtentions.SafeExecute(() => clientHost.StopAsync()); Utils.SafeExecute(() => host.Dispose()); Utils.SafeExecute(() => clientHost.Dispose()); } }
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper) { try { // double check in case of interleaving if (consumerData.State == StreamConsumerDataState.Active || consumerData.Cursor == null) { return; } consumerData.State = StreamConsumerDataState.Active; while (consumerData.Cursor != null && consumerData.Cursor.MoveNext()) { IBatchContainer batch = null; Exception ex; Task deliveryTask; bool deliveryFailed = false; try { batch = consumerData.Cursor.GetCurrent(out ex); } catch (DataNotAvailableException dataNotAvailable) { ex = dataNotAvailable; } // Apply filtering to this batch, if applicable if (filterWrapper != null && batch != null) { try { // Apply batch filter to this input batch, to see whether we should deliver it to this consumer. if (!batch.ShouldDeliver( consumerData.StreamId, filterWrapper.FilterData, filterWrapper.ShouldReceive)) { continue; // Skip this batch -- nothing to do } } catch (Exception exc) { var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId); logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc); } } if (batch != null) { deliveryTask = AsyncExecutorWithRetries.ExecuteWithRetries(i => DeliverBatchToConsumer(consumerData, batch), AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is DataNotAvailableException), config.MaxEventDeliveryTime, DefaultBackoffProvider); } else if (ex == null) { deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId); } else { // If data is not avialable, bring cursor current if (ex is DataNotAvailableException) { consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid, consumerData.StreamId.Namespace, null); } // Notify client of error. deliveryTask = DeliverErrorToConsumer(consumerData, ex, null); } try { numSentMessagesCounter.Increment(); await deliveryTask; } catch (Exception exc) { var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId); logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc); deliveryFailed = true; } // if we failed to deliver a batch if (deliveryFailed && batch != null) { // notify consumer of delivery error, if we can. await OrleansTaskExtentions.ExecuteAndIgnoreException(() => DeliverErrorToConsumer(consumerData, new StreamEventDeliveryFailureException(consumerData.StreamId), batch)); // record that there was a delivery failure await streamFailureHandler.OnDeliveryFailure(consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, batch.SequenceToken); // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { try { // notify consumer of faulted subscription, if we can. DeliverErrorToConsumer(consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch) .Ignore(); // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId); } finally { // remove subscription RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); } return; } } } consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc); throw; } }
public async Task Shutdown() { // Stop pulling from queues that are not in my range anymore. logger.Info(ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode()); if (timer != null) { var tmp = timer; timer = null; Utils.SafeExecute(tmp.Dispose, this.logger); try { await tmp.GetCurrentlyExecutingTickTask().WithTimeout(TimeSpan.FromSeconds(5)); } catch (Exception ex) { this.logger.LogWarning(ex, "Waiting for the last timer tick failed"); } } this.queueCache = null; Task localReceiverInitTask = receiverInitTask; if (localReceiverInitTask != null) { try { await localReceiverInitTask; receiverInitTask = null; } catch (Exception) { receiverInitTask = null; // squelch } } try { IQueueAdapterReceiver localReceiver = this.receiver; this.receiver = null; if (localReceiver != null) { var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(this.options.InitQueueTimeout)); task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07, $"QueueAdapterReceiver {QueueId} failed to Shutdown."); await task; } } catch { // Just ignore this exception and proceed as if Shutdown has succeeded. // We already logged individual exceptions for individual calls to Shutdown. No need to log again. } var unregisterTasks = new List <Task>(); var meAsStreamProducer = this.AsReference <IStreamProducerExtension>(); foreach (var tuple in pubSubCache) { tuple.Value.DisposeAll(logger); var streamId = tuple.Key; logger.Info(ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId); unregisterTasks.Add(pubSub.UnregisterProducer(streamId, meAsStreamProducer)); } try { await Task.WhenAll(unregisterTasks); } catch (Exception exc) { logger.Warn(ErrorCode.PersistentStreamPullingAgent_08, "Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc); } pubSubCache.Clear(); IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix)); //IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix)); }
// Execute one or more turns for this activation. // This method is always called in a single-threaded environment -- that is, no more than one // thread will be in this method at once -- but other asynch threads may still be queueing tasks, etc. public void Execute() { lock (lockable) { if (state == WorkGroupStatus.Shutdown) { if (!IsActive) { return; // Don't mind if no work has been queued to this work group yet. } ReportWorkGroupProblemWithBacktrace( "Cannot execute work items in a work item group that is in a shutdown state.", ErrorCode.SchedulerNotExecuteWhenShutdown); // Throws InvalidOperationException return; } state = WorkGroupStatus.Running; } var thread = WorkerPoolThread.CurrentWorkerThread; try { // Process multiple items -- drain the applicationMessageQueue (up to max items) for this physical activation int count = 0; var stopwatch = new Stopwatch(); stopwatch.Start(); do { lock (lockable) { if (state == WorkGroupStatus.Shutdown) { if (WorkItemCount > 0) { log.Warn(ErrorCode.SchedulerSkipWorkStopping, "Thread {0} is exiting work loop due to Shutdown state {1} while still having {2} work items in the queue.", thread.ToString(), this.ToString(), WorkItemCount); } else if (log.IsVerbose) { log.Verbose("Thread {0} is exiting work loop due to Shutdown state {1}. Has {2} work items in the queue.", thread.ToString(), this.ToString(), WorkItemCount); } break; } // Check the cancellation token (means that the silo is stopping) if (thread.CancelToken.IsCancellationRequested) { log.Warn(ErrorCode.SchedulerSkipWorkCancelled, "Thread {0} is exiting work loop due to cancellation token. WorkItemGroup: {1}, Have {2} work items in the queue.", thread.ToString(), this.ToString(), WorkItemCount); break; } } // Get the first Work Item on the list Task task; lock (lockable) { if (workItems.Count > 0) { task = workItems.Dequeue(); } else // If the list is empty, then we're done { break; } } #if TRACK_DETAILED_STATS if (StatisticsCollector.CollectGlobalShedulerStats) { SchedulerStatisticsGroup.OnWorkItemDequeue(); } #endif #if DEBUG if (log.IsVerbose2) { log.Verbose2("About to execute task {0} in SchedulingContext={1}", task, SchedulingContext); } #endif var taskStart = stopwatch.Elapsed; try { thread.CurrentTask = task; #if TRACK_DETAILED_STATS if (StatisticsCollector.CollectTurnsStats) { SchedulerStatisticsGroup.OnTurnExecutionStartsByWorkGroup(workItemGroupStatisticsNumber, thread.WorkerThreadStatisticsNumber, SchedulingContext); } #endif TaskRunner.RunTask(task); } catch (Exception ex) { log.Error(ErrorCode.SchedulerExceptionFromExecute, String.Format("Worker thread caught an exception thrown from Execute by task {0}", task), ex); throw; } finally { #if TRACK_DETAILED_STATS if (StatisticsCollector.CollectTurnsStats) { SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(thread.CurrentStateStarted)); } if (StatisticsCollector.CollectThreadTimeTrackingStats) { thread.threadTracking.IncrementNumberOfProcessed(); } #endif totalItemsProcessed++; var taskLength = stopwatch.Elapsed - taskStart; if (taskLength > OrleansTaskScheduler.TurnWarningLengthThreshold) { SchedulerStatisticsGroup.NumLongRunningTurns.Increment(); log.Warn(ErrorCode.SchedulerTurnTooLong3, "Task {0} in WorkGroup {1} took elapsed time {2:g} for execution, which is longer than {3}. Running on thread {4}", OrleansTaskExtentions.ToString(task), SchedulingContext.ToString(), taskLength, OrleansTaskScheduler.TurnWarningLengthThreshold, thread.ToString()); } thread.CurrentTask = null; } count++; }while (((MaxWorkItemsPerTurn <= 0) || (count <= MaxWorkItemsPerTurn)) && ((ActivationSchedulingQuantum <= TimeSpan.Zero) || (stopwatch.Elapsed < ActivationSchedulingQuantum))); stopwatch.Stop(); } catch (Exception ex) { log.Error(ErrorCode.Runtime_Error_100032, String.Format("Worker thread {0} caught an exception thrown from IWorkItem.Execute", thread), ex); } finally { // Now we're not Running anymore. // If we left work items on our run list, we're Runnable, and need to go back on the silo run queue; // If our run list is empty, then we're waiting. lock (lockable) { if (state != WorkGroupStatus.Shutdown) { if (WorkItemCount > 0) { state = WorkGroupStatus.Runnable; masterScheduler.RunQueue.Add(this); } else { state = WorkGroupStatus.Waiting; } } } } }
/// <summary> /// Take responsibility for a new queues that was assigned to me via a new range. /// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer. /// ERROR HANDLING: /// The responsibility to handle initialization and shutdown failures is inside the INewQueueAdapterReceiver code. /// The agent will call Initialize once and log an error. It will not call initialize again. /// The receiver itself may attempt later to recover from this error and do initialization again. /// The agent will assume initialization has succeeded and will subsequently start calling pumping receive. /// Same applies to shutdown. /// </summary> /// <returns></returns> public Task Initialize() { return(OrleansTaskExtentions.WrapInTask(() => InitializeInternal())); }