public async Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId) { if (State.Consumers.Any(c => c.IsFaulted && c.Equals(subscriptionId))) { throw new FaultedSubscriptionException(subscriptionId, streamId); } int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId)); counterConsumersRemoved.Increment(); counterConsumersTotal.DecrementBy(numRemoved); LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved); await WriteStateAsync(); await NotifyProducersOfRemovedSubscription(subscriptionId, streamId); await ClearStateWhenEmpty(); }
public async Task <ISet <PubSubSubscriptionState> > RegisterProducer(StreamId streamId, IStreamProducerExtension streamProducer) { if (!IsActiveProducer(streamProducer)) { throw new ArgumentException(String.Format("Trying to register non active IStreamProducerExtension: {0}", streamProducer.ToString()), "streamProducer"); } RemoveDeadProducers(); var publisherState = new PubSubPublisherState(streamId, streamProducer); State.Producers.Add(publisherState); counterProducersAdded.Increment(); counterProducersTotal.Increment(); LogPubSubCounts("RegisterProducer {0}", streamProducer); await WriteStateAsync(); return(State.Consumers.Where(c => !c.IsFaulted).ToSet()); }
/// <summary> /// Registers a new activation, in single activation mode, with the directory service. /// If there is already an activation registered for this grain, then the new activation will /// not be registered and the address of the existing activation will be returned. /// Otherwise, the passed-in address will be returned. /// <para>This method must be called from a scheduler thread.</para> /// </summary> /// <param name="address">The address of the potential new activation.</param> /// <returns>The address registered for the grain's single activation.</returns> public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address) { registrationsSingleActIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsSingleActLocal.Increment(); // if I am the owner, store the new activation locally Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo); return(returnedAddress == null ? null : returnedAddress.Item1); } else { RegistrationsSingleActRemoteSent.Increment(); // otherwise, notify the owner Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES); // Caching optimization: // cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! if (returnedAddress == null || returnedAddress.Item1 == null) { return(null); } if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo)) { return(returnedAddress.Item1); } var cached = new List <Tuple <SiloAddress, ActivationId> >(new [] { Tuple.Create(address.Silo, address.Activation) }); // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2); return(returnedAddress.Item1); } }
private bool Send(Message msg, ClientState client) { var connection = client.Connection; if (connection is null) { return(false); } try { connection.Send(msg); gatewaySends.Increment(); return(true); } catch (Exception exception) { gateway.RecordClosedConnection(connection); connection.CloseAsync(new ConnectionAbortedException("Exception posting a message to sender. See InnerException for details.", exception)).Ignore(); return(false); } }
public async Task <ISet <PubSubSubscriptionState> > RegisterProducer(StreamId streamId, IStreamProducerExtension streamProducer) { counterProducersAdded.Increment(); try { var publisherState = new PubSubPublisherState(streamId, streamProducer); State.Producers.Add(publisherState); LogPubSubCounts("RegisterProducer {0}", streamProducer); await WriteStateAsync(); counterProducersTotal.Increment(); } catch (Exception exc) { logger.Error(ErrorCode.Stream_RegisterProducerFailed, $"Failed to register a stream producer. Stream: {streamId}, Producer: {streamProducer}", exc); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } return(State.Consumers.Where(c => !c.IsFaulted).ToSet()); }
public async Task UnregisterConsumer(GuidId subscriptionId, InternalStreamId streamId) { counterConsumersRemoved.Increment(); try { int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId)); LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved); if (await TryClearState()) { // If state was cleared expedite Deactivation DeactivateOnIdle(); } else { if (numRemoved != 0) { await WriteStateAsync(); } await NotifyProducersOfRemovedSubscription(subscriptionId, streamId); } counterConsumersTotal.DecrementBy(numRemoved); } catch (Exception exc) { logger.LogError( (int)ErrorCode.Stream_UnregisterConsumerFailed, exc, "Failed to unregister a stream consumer. Stream: {StreamId}, SubscriptionId {SubscriptionId}", streamId, subscriptionId); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } }
public async Task RegisterAsync(ActivationAddress address) { registrationsIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsLocal.Increment(); // if I am the owner, store the new activation locally DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo); } else { RegistrationsRemoteSent.Increment(); // otherwise, notify the owner int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES); if (IsValidSilo(address.Silo)) { // Caching optimization: // cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! List <Tuple <SiloAddress, ActivationId> > cached = null; if (!DirectoryCache.LookUp(address.Grain, out cached)) { cached = new List <Tuple <SiloAddress, ActivationId> >(1); } cached.Add(Tuple.Create(address.Silo, address.Activation)); // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, eTag); } } }
public async Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId) { int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId)); counterConsumersRemoved.Increment(); counterConsumersTotal.DecrementBy(numRemoved); LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved); await State.WriteStateAsync(); int numProducers = State.Producers.Count; if (numProducers > 0) { if (logger.IsVerbose) { logger.Verbose("Notifying {0} existing producers about unregistered consumer.", numProducers); } // Notify producers about unregistered consumer. var tasks = new List <Task>(); foreach (var producerState in State.Producers.Where(producerState => IsActiveProducer(producerState.Producer))) { tasks.Add(producerState.Producer.RemoveSubscriber(subscriptionId, streamId)); } await Task.WhenAll(tasks); } else if (State.Consumers.Count == 0) // + we already know that numProducers == 0 from previous if-clause { await State.ClearStateAsync(); //State contains no producers or consumers, remove it from storage // No producers or consumers left now, so flag ourselves to expedite Deactivation DeactivateOnIdle(); } }
public async Task <List <ActivationAddress> > FullLookup(GrainId grain) { fullLookups.Increment(); SiloAddress silo = CalculateTargetSilo(grain, false); // No need to check that silo != null since we're passing excludeThisSiloIfStopping = false if (log.IsVerbose) { log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode()); } // We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()). // We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called. if (silo.Equals(MyAddress)) { LocalDirectoryLookups.Increment(); var localResult = DirectoryPartition.LookUpGrain(grain); if (localResult == null) { // it can happen that we cannot find the grain in our partition if there were // some recent changes in the membership if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}=none", grain); } return(new List <ActivationAddress>()); } var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings()); } LocalDirectorySuccesses.Increment(); return(a); } // Just a optimization. Why sending a message to someone we know is not valid. if (!IsValidSilo(silo)) { throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo)); } RemoteLookupsSent.Increment(); Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES); // update the cache List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList(); List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings()); } if (entries.Count > 0) { DirectoryCache.AddOrUpdate(grain, entries, result.Item2); } return(addresses); }
/// <summary> /// Handles an incoming (proxied) message by rerouting it immediately and unconditionally, /// after some header massaging. /// </summary> /// <param name="msg"></param> /// <param name="receivedOnSocket"></param> protected override void HandleMessage(Message msg, Socket receivedOnSocket) { // Don't process messages that have already timed out if (msg.IsExpired) { msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Receive); return; } if (Message.WriteMessagingTraces) { msg.AddTimestamp(Message.LifecycleTag.ReceiveIncoming); } gatewayTrafficCounter.Increment(); // Are we overloaded? if ((MessageCenter.Metrics != null) && MessageCenter.Metrics.IsOverloaded) { MessagingStatisticsGroup.OnRejectedMessage(msg); Message rejection = msg.CreateRejectionResponse(Message.RejectionTypes.GatewayTooBusy, "Shedding load"); MessageCenter.TryDeliverToProxy(rejection); if (Log.IsVerbose) { Log.Verbose("Rejecting a request due to overloading: {0}", msg.ToString()); } loadSheddingCounter.Increment(); return; } gateway.RecordSendingProxiedGrain(msg.SendingGrain, receivedOnSocket); SiloAddress targetAddress = gateway.TryToReroute(msg); msg.SendingSilo = MessageCenter.MyAddress; if (targetAddress == null) { // reroute via Dispatcher msg.RemoveHeader(Message.Header.TARGET_SILO); msg.RemoveHeader(Message.Header.TARGET_ACTIVATION); if (msg.TargetGrain.IsSystemTarget) { msg.TargetSilo = MessageCenter.MyAddress; msg.TargetActivation = ActivationId.GetSystemActivation(msg.TargetGrain, MessageCenter.MyAddress); } if (Message.WriteMessagingTraces) { msg.AddTimestamp(Message.LifecycleTag.RerouteIncoming); } MessagingStatisticsGroup.OnMessageReRoute(msg); MessageCenter.RerouteMessage(msg); } else { // send directly msg.TargetSilo = targetAddress; MessageCenter.SendMessage(msg); } }
public async Task RegisterConsumer( StreamId streamId, IStreamConsumerExtension streamConsumer, StreamSequenceToken token, IStreamFilterPredicateWrapper filter) { // This Where clause will return either zero or one PubSubSubscriptionState var found = State.Consumers.Where(s => s.Equals(streamId, streamConsumer)).ToArray(); PubSubSubscriptionState pubSubState; if (found.Length == 0) { pubSubState = new PubSubSubscriptionState(streamId, streamConsumer, token, filter); State.Consumers.Add(pubSubState); } else { pubSubState = found[0]; if (filter != null) pubSubState.AddFilter(filter); } counterConsumersAdded.Increment(); counterConsumersTotal.Increment(); LogPubSubCounts("RegisterConsumer {0}", streamConsumer); await State.WriteStateAsync(); int numProducers = State.Producers.Count; if (numProducers > 0) { if (logger.IsVerbose) logger.Info("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}", numProducers, streamConsumer, Utils.EnumerableToString(State.Producers)); // Notify producers about a new streamConsumer. var tasks = new List<Task>(); var producers = State.Producers.ToList(); bool someProducersRemoved = false; foreach (var producerState in producers) { PubSubPublisherState producer = producerState; // Capture loop variable if (!IsActiveProducer(producer.Producer)) { // Producer is not active (could be stopping / shutting down) so skip if (logger.IsVerbose) logger.Verbose("Producer {0} on stream {1} is not active - skipping.", producer, streamId); continue; } Task addSubscriberPromise = producer.Producer.AddSubscriber(streamId, streamConsumer, token, filter) .ContinueWith(t => { if (t.IsFaulted) { var exc = t.Exception.GetBaseException(); if (exc is GrainExtensionNotInstalledException) { logger.Warn((int) ErrorCode.Stream_ProducerIsDead, "Producer {0} on stream {1} is no longer active - discarding.", producer, streamId); // This publisher has gone away, so we should cleanup pub-sub state. bool removed = State.Producers.Remove(producer); someProducersRemoved = true; // Re-save state changes at end counterProducersRemoved.Increment(); counterProducersTotal.DecrementBy(removed ? 1 : 0); // And ignore this error } else { throw exc; } } }, TaskContinuationOptions.OnlyOnFaulted); tasks.Add(addSubscriberPromise); } Exception exception = null; try { await Task.WhenAll(tasks); } catch (Exception exc) { exception = exc; } if (someProducersRemoved) await State.WriteStateAsync(); if (exception != null) throw exception; } }
internal static void OnTransactionCommitRequest() { CommitTransactionQueueLength.Increment(); CommitTransactionRequests.Increment(); }
private bool Send(Message msg, Socket sock) { if (Cts.IsCancellationRequested) { return(false); } if (sock == null) { return(false); } // Send the message List <ArraySegment <byte> > data; int headerLength; try { int bodyLength; data = msg.Serialize(this.serializationManager, out headerLength, out bodyLength); if (headerLength + bodyLength > this.serializationManager.LargeObjectSizeThreshold) { Log.Info(ErrorCode.Messaging_LargeMsg_Outgoing, "Preparing to send large message Size={0} HeaderLength={1} BodyLength={2} #ArraySegments={3}. Msg={4}", headerLength + bodyLength + Message.LENGTH_HEADER_SIZE, headerLength, bodyLength, data.Count, this.ToString()); if (Log.IsEnabled(LogLevel.Trace)) { Log.Trace("Sending large message {0}", msg.ToLongString()); } } } catch (Exception exc) { this.OnMessageSerializationFailure(msg, exc); return(true); } int length = data.Sum(x => x.Count); int bytesSent = 0; bool exceptionSending = false; bool countMismatchSending = false; string sendErrorStr; try { bytesSent = sock.Send(data); if (bytesSent != length) { // The complete message wasn't sent, even though no error was reported; treat this as an error countMismatchSending = true; sendErrorStr = String.Format("Byte count mismatch on send: sent {0}, expected {1}", bytesSent, length); Log.Warn(ErrorCode.GatewayByteCountMismatch, sendErrorStr); } } catch (Exception exc) { exceptionSending = true; string remoteEndpoint = ""; if (!(exc is ObjectDisposedException)) { try { remoteEndpoint = sock.RemoteEndPoint.ToString(); } catch (Exception) {} } sendErrorStr = String.Format("Exception sending to client at {0}: {1}", remoteEndpoint, exc); Log.Warn(ErrorCode.GatewayExceptionSendingToClient, sendErrorStr, exc); } MessagingStatisticsGroup.OnMessageSend(msg.TargetSilo, msg.Direction, bytesSent, headerLength, SocketDirection.GatewayToClient); bool sendError = exceptionSending || countMismatchSending; if (sendError) { gateway.RecordClosedSocket(sock); SocketManager.CloseSocket(sock); } gatewaySends.Increment(); msg.ReleaseBodyAndHeaderBuffers(); return(!sendError); }
internal static void OnTransactionInDoubt() { CommitTransactionQueueLength.DecrementBy(1); TransactionsInDoubt.Increment(); }
private bool Send(Message msg, Socket sock) { if (Cts.IsCancellationRequested) { return(false); } if (sock == null) { return(false); } // Send the message List <ArraySegment <byte> > data; int headerLength; try { data = msg.Serialize(out headerLength); } catch (Exception exc) { OnMessageSerializationFailure(msg, exc); return(true); } int length = data.Sum(x => x.Count); int bytesSent = 0; bool exceptionSending = false; bool countMismatchSending = false; string sendErrorStr; try { bytesSent = sock.Send(data); if (bytesSent != length) { // The complete message wasn't sent, even though no error was reported; treat this as an error countMismatchSending = true; sendErrorStr = String.Format("Byte count mismatch on send: sent {0}, expected {1}", bytesSent, length); Log.Warn(ErrorCode.GatewayByteCountMismatch, sendErrorStr); } } catch (Exception exc) { exceptionSending = true; string remoteEndpoint = ""; if (!(exc is ObjectDisposedException)) { try { remoteEndpoint = sock.RemoteEndPoint.ToString(); } catch (Exception) {} } sendErrorStr = String.Format("Exception sending to client at {0}: {1}", remoteEndpoint, exc); Log.Warn(ErrorCode.GatewayExceptionSendingToClient, sendErrorStr, exc); } MessagingStatisticsGroup.OnMessageSend(msg.TargetSilo, msg.Direction, bytesSent, headerLength, SocketDirection.GatewayToClient); bool sendError = exceptionSending || countMismatchSending; if (sendError) { gateway.RecordClosedSocket(sock); SocketManager.CloseSocket(sock); } gatewaySends.Increment(); msg.ReleaseBodyAndHeaderBuffers(); return(!sendError); }
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper) { try { // double check in case of interleaving if (consumerData.State == StreamConsumerDataState.Active || consumerData.Cursor == null) { return; } consumerData.State = StreamConsumerDataState.Active; while (consumerData.Cursor != null && consumerData.Cursor.MoveNext()) { IBatchContainer batch = null; Exception ex; Task deliveryTask; try { batch = consumerData.Cursor.GetCurrent(out ex); } catch (DataNotAvailableException dataNotAvailable) { ex = dataNotAvailable; } // Apply filtering to this batch, if applicable if (filterWrapper != null && batch != null) { try { // Apply batch filter to this input batch, to see whether we should deliver it to this consumer. if (!batch.ShouldDeliver( consumerData.StreamId, filterWrapper.FilterData, filterWrapper.ShouldReceive)) { continue; // Skip this batch -- nothing to do } } catch (Exception exc) { var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId); logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc); } } if (batch != null) { deliveryTask = consumerData.StreamConsumer .DeliverBatch(consumerData.SubscriptionId, batch.AsImmutable()); } else if (ex == null) { deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId); } else { deliveryTask = consumerData.StreamConsumer.ErrorInStream(consumerData.SubscriptionId, ex); } try { numSentMessagesCounter.Increment(); await deliveryTask; } catch (Exception exc) { var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId); logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc); } } consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc); throw; } }
/// <summary> /// Determines whether the provided <see cref="T:System.Threading.Tasks.Task"/> can be executed synchronously in this call, and if it can, executes it. /// </summary> /// <returns> /// A Boolean value indicating whether the task was executed inline. /// </returns> /// <param name="task">The <see cref="T:System.Threading.Tasks.Task"/> to be executed.</param> /// <param name="taskWasPreviouslyQueued">A Boolean denoting whether or not task has previously been queued. If this parameter is True, then the task may have been previously queued (scheduled); if False, then the task is known not to have been queued, and this call is being made in order to execute the task inline without queuing it.</param> protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) { bool canExecuteInline = WorkerPoolThread.CurrentWorkerThread != null; RuntimeContext ctx = RuntimeContext.Current; bool canExecuteInline2 = canExecuteInline && ctx != null && object.Equals(ctx.ActivationContext, workerGroup.SchedulingContext); canExecuteInline = canExecuteInline2; #if DEBUG if (logger.IsVerbose2) { logger.Verbose2(myId + " --> TryExecuteTaskInline Task Id={0} Status={1} PreviouslyQueued={2} CanExecute={3} Queued={4}", task.Id, task.Status, taskWasPreviouslyQueued, canExecuteInline, workerGroup.ExternalWorkItemCount); } #endif if (!canExecuteInline) { return(false); } // If the task was previously queued, remove it from the queue if (taskWasPreviouslyQueued) { canExecuteInline = TryDequeue(task); } if (!canExecuteInline) { #if DEBUG if (logger.IsVerbose2) { logger.Verbose2(myId + " <-X TryExecuteTaskInline Task Id={0} Status={1} Execute=No", task.Id, task.Status); } #endif return(false); } #if EXTRA_STATS turnsExecutedStatistic.Increment(); #endif #if DEBUG if (logger.IsVerbose3) { logger.Verbose3(myId + " TryExecuteTaskInline Task Id={0} Thread={1} Execute=Yes", task.Id, Thread.CurrentThread.ManagedThreadId); } #endif // Try to run the task. bool done = TryExecuteTask(task); if (!done) { logger.Warn(ErrorCode.SchedulerTaskExecuteIncomplete3, "TryExecuteTaskInline: Incomplete base.TryExecuteTask for Task Id={0} with Status={1}", task.Id, task.Status); } #if DEBUG if (logger.IsVerbose2) { logger.Verbose2(myId + " <-- TryExecuteTaskInline Task Id={0} Thread={1} Execute=Done Ok={2}", task.Id, Thread.CurrentThread.ManagedThreadId, done); } #endif return(done); }
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper) { try { // double check in case of interleaving if (consumerData.State == StreamConsumerDataState.Active || consumerData.Cursor == null) { return; } consumerData.State = StreamConsumerDataState.Active; while (consumerData.Cursor != null && consumerData.Cursor.MoveNext()) { IBatchContainer batch = null; Exception ex; Task deliveryTask; bool deliveryFailed = false; try { batch = consumerData.Cursor.GetCurrent(out ex); } catch (DataNotAvailableException dataNotAvailable) { ex = dataNotAvailable; } // Apply filtering to this batch, if applicable if (filterWrapper != null && batch != null) { try { // Apply batch filter to this input batch, to see whether we should deliver it to this consumer. if (!batch.ShouldDeliver( consumerData.StreamId, filterWrapper.FilterData, filterWrapper.ShouldReceive)) { continue; // Skip this batch -- nothing to do } } catch (Exception exc) { var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId); logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc); } } if (batch != null) { deliveryTask = AsyncExecutorWithRetries.ExecuteWithRetries(i => DeliverBatchToConsumer(consumerData, batch), AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is DataNotAvailableException), config.MaxEventDeliveryTime, DefaultBackoffProvider); } else if (ex == null) { deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId); } else { // If data is not avialable, bring cursor current if (ex is DataNotAvailableException) { consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid, consumerData.StreamId.Namespace, null); } // Notify client of error. deliveryTask = DeliverErrorToConsumer(consumerData, ex, null); } try { numSentMessagesCounter.Increment(); await deliveryTask; } catch (Exception exc) { var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId); logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc); deliveryFailed = true; } // if we failed to deliver a batch if (deliveryFailed && batch != null) { // notify consumer of delivery error, if we can. await OrleansTaskExtentions.ExecuteAndIgnoreException(() => DeliverErrorToConsumer(consumerData, new StreamEventDeliveryFailureException(consumerData.StreamId), batch)); // record that there was a delivery failure await streamFailureHandler.OnDeliveryFailure(consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, batch.SequenceToken); // if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid)) { try { // notify consumer of faulted subscription, if we can. DeliverErrorToConsumer(consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch) .Ignore(); // mark subscription as faulted. await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId); } finally { // remove subscription RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId); } return; } } } consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc); throw; } }
internal static void OnTransactionAborted() { CommitTransactionQueueLength.DecrementBy(1); AbortedTransactionsTotal.Increment(); }
public async Task RegisterConsumer( GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { counterConsumersAdded.Increment(); PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId)); if (pubSubState != null && pubSubState.IsFaulted) { throw new FaultedSubscriptionException(subscriptionId, streamId); } try { if (pubSubState == null) { pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer); State.Consumers.Add(pubSubState); } if (filter != null) { pubSubState.AddFilter(filter); } LogPubSubCounts("RegisterConsumer {0}", streamConsumer); await WriteStateAsync(); counterConsumersTotal.Increment(); } catch (Exception exc) { logger.Error(ErrorCode.Stream_RegisterConsumerFailed, $"Failed to register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } int numProducers = State.Producers.Count; if (numProducers <= 0) { return; } if (logger.IsEnabled(LogLevel.Debug)) { logger.Debug("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}", numProducers, streamConsumer, Utils.EnumerableToString(State.Producers)); } // Notify producers about a new streamConsumer. var tasks = new List <Task>(); var producers = State.Producers.ToList(); int initialProducerCount = producers.Count; try { foreach (PubSubPublisherState producerState in producers) { tasks.Add(ExecuteProducerTask(producerState, producerState.Producer.AddSubscriber(subscriptionId, streamId, streamConsumer, filter))); } Exception exception = null; try { await Task.WhenAll(tasks); } catch (Exception exc) { exception = exc; } // if the number of producers has been changed, resave state. if (State.Producers.Count != initialProducerCount) { await WriteStateAsync(); counterConsumersTotal.DecrementBy(initialProducerCount - State.Producers.Count); } if (exception != null) { throw exception; } } catch (Exception exc) { logger.Error(ErrorCode.Stream_RegisterConsumerFailed, $"Failed to update producers while register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc); // Corrupted state, deactivate grain. DeactivateOnIdle(); throw; } }
internal static void OnTransactionStarted() { StartTransactionQueueLength.DecrementBy(1); StartTransactionCompleted.Increment(); }
/// <summary> /// Determines whether the provided <see cref="T:System.Threading.Tasks.Task"/> can be executed synchronously in this call, and if it can, executes it. /// </summary> /// <returns> /// A Boolean value indicating whether the task was executed inline. /// </returns> /// <param name="task">The <see cref="T:System.Threading.Tasks.Task"/> to be executed.</param> /// <param name="taskWasPreviouslyQueued">A Boolean denoting whether or not task has previously been queued. If this parameter is True, then the task may have been previously queued (scheduled); if False, then the task is known not to have been queued, and this call is being made in order to execute the task inline without queuing it.</param> protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) { var currentContext = RuntimeContext.Current; bool canExecuteInline = currentContext != null && object.Equals(currentContext, workerGroup.GrainContext); #if DEBUG if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace( "{TaskScheduler} TryExecuteTaskInline Task Id={TaskId} Status={Status} PreviouslyQueued={PreviouslyQueued} CanExecute={CanExecute} Queued={Queued}", myId, task.Id, task.Status, taskWasPreviouslyQueued, canExecuteInline, workerGroup.ExternalWorkItemCount); } #endif if (!canExecuteInline) { return(false); } // If the task was previously queued, remove it from the queue if (taskWasPreviouslyQueued) { canExecuteInline = TryDequeue(task); } if (!canExecuteInline) { #if DEBUG if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace("{TaskScheduler} Completed TryExecuteTaskInline Task Id={TaskId} Status={Status} Execute=No", myId, task.Id, task.Status); } #endif return(false); } #if EXTRA_STATS turnsExecutedStatistic.Increment(); #endif #if DEBUG if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace( "{TaskScheduler} TryExecuteTaskInline Task Id={TaskId} Thread={Thread} Execute=Yes", myId, task.Id, Thread.CurrentThread.ManagedThreadId); } #endif // Try to run the task. bool done = TryExecuteTask(task); if (!done) { logger.LogWarning( (int)ErrorCode.SchedulerTaskExecuteIncomplete3, "TryExecuteTaskInline: Incomplete base.TryExecuteTask for Task Id={TaskId} with Status={TaskStatus}", task.Id, task.Status); } #if DEBUG if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace( "{TaskScheduler} Completed TryExecuteTaskInline Task Id={TaskId} Thread={Thread} Execute=Done Ok={Ok}", myId, task.Id, Thread.CurrentThread.ManagedThreadId, done); } #endif return(done); }
internal static void OnTransactionCommitted() { CommitTransactionQueueLength.DecrementBy(1); CommitTransactionCompleted.Increment(); }
private async Task RunConsumerCursor(StreamConsumerData consumerData) { try { // double check in case of interleaving if (consumerData.State == StreamConsumerDataState.Active || consumerData.Cursor == null) { return; } consumerData.State = StreamConsumerDataState.Active; while (consumerData.Cursor != null) { IBatchContainer batch = null; Exception exceptionOccured = null; try { batch = GetBatchForConsumer(consumerData.Cursor, consumerData.StreamId, consumerData.FilterData); if (batch == null) { break; } } catch (Exception exc) { exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); } if (batch != null) { if (!ShouldDeliverBatch(consumerData.StreamId, batch, consumerData.FilterData)) { continue; } } try { numSentMessagesCounter.Increment(); if (batch != null) { StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( i => DeliverBatchToConsumer(consumerData, batch), AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is ClientNotAvailableException) || IsShutdown, this.options.MaxEventDeliveryTime, DeliveryBackoffProvider); if (newToken != null) { consumerData.LastToken = newToken; IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = newCursor; } } } catch (Exception exc) { consumerData.Cursor?.RecordDeliveryFailure(); var message = $"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor"; logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc); exceptionOccured = exc is ClientNotAvailableException ? exc : new StreamEventDeliveryFailureException(consumerData.StreamId); } // if we failed to deliver a batch if (exceptionOccured != null) { bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken); if (faultedSubscription) { return; } } } consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc); consumerData.State = StreamConsumerDataState.Inactive; throw; } }
public async Task RegisterConsumer( GuidId subscriptionId, StreamId streamId, IStreamConsumerExtension streamConsumer, IStreamFilterPredicateWrapper filter) { PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId)); if (pubSubState == null) { pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer); State.Consumers.Add(pubSubState); } if (pubSubState.IsFaulted) { throw new FaultedSubscriptionException(subscriptionId, streamId); } if (filter != null) { pubSubState.AddFilter(filter); } counterConsumersAdded.Increment(); counterConsumersTotal.Increment(); LogPubSubCounts("RegisterConsumer {0}", streamConsumer); await WriteStateAsync(); int numProducers = State.Producers.Count; if (numProducers <= 0) { return; } if (logger.IsVerbose) { logger.Info("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}", numProducers, streamConsumer, Utils.EnumerableToString(State.Producers)); } // Notify producers about a new streamConsumer. var tasks = new List <Task>(); var producers = State.Producers.ToList(); int initialProducerCount = producers.Count; foreach (var producerState in producers) { PubSubPublisherState producer = producerState; // Capture loop variable if (!IsActiveProducer(producer.Producer)) { // Producer is not active (could be stopping / shutting down) so skip if (logger.IsVerbose) { logger.Verbose("Producer {0} on stream {1} is not active - skipping.", producer, streamId); } continue; } tasks.Add(NotifyProducer(producer, subscriptionId, streamId, streamConsumer, filter)); } Exception exception = null; try { await Task.WhenAll(tasks); } catch (Exception exc) { exception = exc; } // if the number of producers has been changed, resave state. if (State.Producers.Count != initialProducerCount) { await WriteStateAsync(); } if (exception != null) { throw exception; } }
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper) { try { // double check in case of interleaving if (consumerData.State == StreamConsumerDataState.Active || consumerData.Cursor == null) { return; } consumerData.State = StreamConsumerDataState.Active; while (consumerData.Cursor != null) { IBatchContainer batch = null; Exception exceptionOccured = null; try { Exception ignore; if (!consumerData.Cursor.MoveNext()) { break; } batch = consumerData.Cursor.GetCurrent(out ignore); } catch (Exception exc) { exceptionOccured = exc; consumerData.SafeDisposeCursor(logger); consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null); } // Apply filtering to this batch, if applicable if (filterWrapper != null && batch != null) { try { // Apply batch filter to this input batch, to see whether we should deliver it to this consumer. if (!batch.ShouldDeliver( consumerData.StreamId, filterWrapper.FilterData, filterWrapper.ShouldReceive)) { continue; // Skip this batch -- nothing to do } } catch (Exception exc) { var message = $"Ignoring exception while trying to evaluate subscription filter function {filterWrapper} on stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor"; logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc); } } try { numSentMessagesCounter.Increment(); if (batch != null) { StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries( i => DeliverBatchToConsumer(consumerData, batch), AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is ClientNotAvailableException), config.MaxEventDeliveryTime, DeliveryBackoffProvider); if (newToken != null) { consumerData.LastToken = newToken; IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token); consumerData.SafeDisposeCursor(logger); consumerData.Cursor = newCursor; } } } catch (Exception exc) { consumerData.Cursor?.RecordDeliveryFailure(); var message = $"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor"; logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc); exceptionOccured = exc is ClientNotAvailableException ? exc : new StreamEventDeliveryFailureException(consumerData.StreamId); } // if we failed to deliver a batch if (exceptionOccured != null) { bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken); if (faultedSubscription) { return; } } } consumerData.State = StreamConsumerDataState.Inactive; } catch (Exception exc) { // RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc); consumerData.State = StreamConsumerDataState.Inactive; throw; } }
internal static void OnTransactionStartRequest() { StartTransactionQueueLength.Increment(); StartTransactionRequests.Increment(); }