public async Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId)
{
if (State.Consumers.Any(c => c.IsFaulted && c.Equals(subscriptionId)))
{
throw new FaultedSubscriptionException(subscriptionId, streamId);
}
int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId));
counterConsumersRemoved.Increment();
counterConsumersTotal.DecrementBy(numRemoved);
LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved);
await WriteStateAsync();
await NotifyProducersOfRemovedSubscription(subscriptionId, streamId);
await ClearStateWhenEmpty();
}
public async Task <ISet <PubSubSubscriptionState> > RegisterProducer(StreamId streamId, IStreamProducerExtension streamProducer)
{
if (!IsActiveProducer(streamProducer))
{
throw new ArgumentException(String.Format("Trying to register non active IStreamProducerExtension: {0}", streamProducer.ToString()), "streamProducer");
}
RemoveDeadProducers();
var publisherState = new PubSubPublisherState(streamId, streamProducer);
State.Producers.Add(publisherState);
counterProducersAdded.Increment();
counterProducersTotal.Increment();
LogPubSubCounts("RegisterProducer {0}", streamProducer);
await WriteStateAsync();
return(State.Consumers.Where(c => !c.IsFaulted).ToSet());
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address)
{
registrationsSingleActIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsSingleActLocal.Increment();
// if I am the owner, store the new activation locally
Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return(returnedAddress == null ? null : returnedAddress.Item1);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (returnedAddress == null || returnedAddress.Item1 == null)
{
return(null);
}
if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo))
{
return(returnedAddress.Item1);
}
var cached = new List <Tuple <SiloAddress, ActivationId> >(new [] { Tuple.Create(address.Silo, address.Activation) });
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2);
return(returnedAddress.Item1);
}
}
private bool Send(Message msg, ClientState client)
{
var connection = client.Connection;
if (connection is null)
{
return(false);
}
try
{
connection.Send(msg);
gatewaySends.Increment();
return(true);
}
catch (Exception exception)
{
gateway.RecordClosedConnection(connection);
connection.CloseAsync(new ConnectionAbortedException("Exception posting a message to sender. See InnerException for details.", exception)).Ignore();
return(false);
}
}
public async Task <ISet <PubSubSubscriptionState> > RegisterProducer(StreamId streamId, IStreamProducerExtension streamProducer)
{
counterProducersAdded.Increment();
try
{
var publisherState = new PubSubPublisherState(streamId, streamProducer);
State.Producers.Add(publisherState);
LogPubSubCounts("RegisterProducer {0}", streamProducer);
await WriteStateAsync();
counterProducersTotal.Increment();
}
catch (Exception exc)
{
logger.Error(ErrorCode.Stream_RegisterProducerFailed, $"Failed to register a stream producer. Stream: {streamId}, Producer: {streamProducer}", exc);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
return(State.Consumers.Where(c => !c.IsFaulted).ToSet());
}
public async Task UnregisterConsumer(GuidId subscriptionId, InternalStreamId streamId)
{
counterConsumersRemoved.Increment();
try
{
int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId));
LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved);
if (await TryClearState())
{
// If state was cleared expedite Deactivation
DeactivateOnIdle();
}
else
{
if (numRemoved != 0)
{
await WriteStateAsync();
}
await NotifyProducersOfRemovedSubscription(subscriptionId, streamId);
}
counterConsumersTotal.DecrementBy(numRemoved);
}
catch (Exception exc)
{
logger.LogError(
(int)ErrorCode.Stream_UnregisterConsumerFailed,
exc,
"Failed to unregister a stream consumer. Stream: {StreamId}, SubscriptionId {SubscriptionId}",
streamId,
subscriptionId);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
}
public async Task RegisterAsync(ActivationAddress address)
{
registrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsLocal.Increment();
// if I am the owner, store the new activation locally
DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
}
else
{
RegistrationsRemoteSent.Increment();
// otherwise, notify the owner
int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES);
if (IsValidSilo(address.Silo))
{
// Caching optimization:
// cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
List <Tuple <SiloAddress, ActivationId> > cached = null;
if (!DirectoryCache.LookUp(address.Grain, out cached))
{
cached = new List <Tuple <SiloAddress, ActivationId> >(1);
}
cached.Add(Tuple.Create(address.Silo, address.Activation));
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, eTag);
}
}
}
public async Task UnregisterConsumer(GuidId subscriptionId, StreamId streamId)
{
int numRemoved = State.Consumers.RemoveWhere(c => c.Equals(subscriptionId));
counterConsumersRemoved.Increment();
counterConsumersTotal.DecrementBy(numRemoved);
LogPubSubCounts("UnregisterSubscription {0} NumRemoved={1}", subscriptionId, numRemoved);
await State.WriteStateAsync();
int numProducers = State.Producers.Count;
if (numProducers > 0)
{
if (logger.IsVerbose)
{
logger.Verbose("Notifying {0} existing producers about unregistered consumer.", numProducers);
}
// Notify producers about unregistered consumer.
var tasks = new List <Task>();
foreach (var producerState in State.Producers.Where(producerState => IsActiveProducer(producerState.Producer)))
{
tasks.Add(producerState.Producer.RemoveSubscriber(subscriptionId, streamId));
}
await Task.WhenAll(tasks);
}
else if (State.Consumers.Count == 0) // + we already know that numProducers == 0 from previous if-clause
{
await State.ClearStateAsync(); //State contains no producers or consumers, remove it from storage
// No producers or consumers left now, so flag ourselves to expedite Deactivation
DeactivateOnIdle();
}
}
public async Task <List <ActivationAddress> > FullLookup(GrainId grain)
{
fullLookups.Increment();
SiloAddress silo = CalculateTargetSilo(grain, false);
// No need to check that silo != null since we're passing excludeThisSiloIfStopping = false
if (log.IsVerbose)
{
log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
// We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()).
// We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called.
if (silo.Equals(MyAddress))
{
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpGrain(grain);
if (localResult == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}=none", grain);
}
return(new List <ActivationAddress>());
}
var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings());
}
LocalDirectorySuccesses.Increment();
return(a);
}
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(silo))
{
throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo));
}
RemoteLookupsSent.Increment();
Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES);
// update the cache
List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList();
List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings());
}
if (entries.Count > 0)
{
DirectoryCache.AddOrUpdate(grain, entries, result.Item2);
}
return(addresses);
}
/// <summary>
/// Handles an incoming (proxied) message by rerouting it immediately and unconditionally,
/// after some header massaging.
/// </summary>
/// <param name="msg"></param>
/// <param name="receivedOnSocket"></param>
protected override void HandleMessage(Message msg, Socket receivedOnSocket)
{
// Don't process messages that have already timed out
if (msg.IsExpired)
{
msg.DropExpiredMessage(MessagingStatisticsGroup.Phase.Receive);
return;
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.ReceiveIncoming);
}
gatewayTrafficCounter.Increment();
// Are we overloaded?
if ((MessageCenter.Metrics != null) && MessageCenter.Metrics.IsOverloaded)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = msg.CreateRejectionResponse(Message.RejectionTypes.GatewayTooBusy, "Shedding load");
MessageCenter.TryDeliverToProxy(rejection);
if (Log.IsVerbose)
{
Log.Verbose("Rejecting a request due to overloading: {0}", msg.ToString());
}
loadSheddingCounter.Increment();
return;
}
gateway.RecordSendingProxiedGrain(msg.SendingGrain, receivedOnSocket);
SiloAddress targetAddress = gateway.TryToReroute(msg);
msg.SendingSilo = MessageCenter.MyAddress;
if (targetAddress == null)
{
// reroute via Dispatcher
msg.RemoveHeader(Message.Header.TARGET_SILO);
msg.RemoveHeader(Message.Header.TARGET_ACTIVATION);
if (msg.TargetGrain.IsSystemTarget)
{
msg.TargetSilo = MessageCenter.MyAddress;
msg.TargetActivation = ActivationId.GetSystemActivation(msg.TargetGrain, MessageCenter.MyAddress);
}
if (Message.WriteMessagingTraces)
{
msg.AddTimestamp(Message.LifecycleTag.RerouteIncoming);
}
MessagingStatisticsGroup.OnMessageReRoute(msg);
MessageCenter.RerouteMessage(msg);
}
else
{
// send directly
msg.TargetSilo = targetAddress;
MessageCenter.SendMessage(msg);
}
}
public async Task RegisterConsumer(
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
// This Where clause will return either zero or one PubSubSubscriptionState
var found = State.Consumers.Where(s => s.Equals(streamId, streamConsumer)).ToArray();
PubSubSubscriptionState pubSubState;
if (found.Length == 0)
{
pubSubState = new PubSubSubscriptionState(streamId, streamConsumer, token, filter);
State.Consumers.Add(pubSubState);
}
else
{
pubSubState = found[0];
if (filter != null)
pubSubState.AddFilter(filter);
}
counterConsumersAdded.Increment();
counterConsumersTotal.Increment();
LogPubSubCounts("RegisterConsumer {0}", streamConsumer);
await State.WriteStateAsync();
int numProducers = State.Producers.Count;
if (numProducers > 0)
{
if (logger.IsVerbose)
logger.Info("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}",
numProducers, streamConsumer, Utils.EnumerableToString(State.Producers));
// Notify producers about a new streamConsumer.
var tasks = new List<Task>();
var producers = State.Producers.ToList();
bool someProducersRemoved = false;
foreach (var producerState in producers)
{
PubSubPublisherState producer = producerState; // Capture loop variable
if (!IsActiveProducer(producer.Producer))
{
// Producer is not active (could be stopping / shutting down) so skip
if (logger.IsVerbose) logger.Verbose("Producer {0} on stream {1} is not active - skipping.", producer, streamId);
continue;
}
Task addSubscriberPromise = producer.Producer.AddSubscriber(streamId, streamConsumer, token, filter)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
var exc = t.Exception.GetBaseException();
if (exc is GrainExtensionNotInstalledException)
{
logger.Warn((int) ErrorCode.Stream_ProducerIsDead,
"Producer {0} on stream {1} is no longer active - discarding.",
producer, streamId);
// This publisher has gone away, so we should cleanup pub-sub state.
bool removed = State.Producers.Remove(producer);
someProducersRemoved = true; // Re-save state changes at end
counterProducersRemoved.Increment();
counterProducersTotal.DecrementBy(removed ? 1 : 0);
// And ignore this error
}
else
{
throw exc;
}
}
}, TaskContinuationOptions.OnlyOnFaulted);
tasks.Add(addSubscriberPromise);
}
Exception exception = null;
try
{
await Task.WhenAll(tasks);
}
catch (Exception exc)
{
exception = exc;
}
if (someProducersRemoved)
await State.WriteStateAsync();
if (exception != null)
throw exception;
}
}
internal static void OnTransactionCommitRequest()
{
CommitTransactionQueueLength.Increment();
CommitTransactionRequests.Increment();
}
private bool Send(Message msg, Socket sock)
{
if (Cts.IsCancellationRequested)
{
return(false);
}
if (sock == null)
{
return(false);
}
// Send the message
List <ArraySegment <byte> > data;
int headerLength;
try
{
int bodyLength;
data = msg.Serialize(this.serializationManager, out headerLength, out bodyLength);
if (headerLength + bodyLength > this.serializationManager.LargeObjectSizeThreshold)
{
Log.Info(ErrorCode.Messaging_LargeMsg_Outgoing, "Preparing to send large message Size={0} HeaderLength={1} BodyLength={2} #ArraySegments={3}. Msg={4}",
headerLength + bodyLength + Message.LENGTH_HEADER_SIZE, headerLength, bodyLength, data.Count, this.ToString());
if (Log.IsEnabled(LogLevel.Trace))
{
Log.Trace("Sending large message {0}", msg.ToLongString());
}
}
}
catch (Exception exc)
{
this.OnMessageSerializationFailure(msg, exc);
return(true);
}
int length = data.Sum(x => x.Count);
int bytesSent = 0;
bool exceptionSending = false;
bool countMismatchSending = false;
string sendErrorStr;
try
{
bytesSent = sock.Send(data);
if (bytesSent != length)
{
// The complete message wasn't sent, even though no error was reported; treat this as an error
countMismatchSending = true;
sendErrorStr = String.Format("Byte count mismatch on send: sent {0}, expected {1}", bytesSent, length);
Log.Warn(ErrorCode.GatewayByteCountMismatch, sendErrorStr);
}
}
catch (Exception exc)
{
exceptionSending = true;
string remoteEndpoint = "";
if (!(exc is ObjectDisposedException))
{
try
{
remoteEndpoint = sock.RemoteEndPoint.ToString();
}
catch (Exception) {}
}
sendErrorStr = String.Format("Exception sending to client at {0}: {1}", remoteEndpoint, exc);
Log.Warn(ErrorCode.GatewayExceptionSendingToClient, sendErrorStr, exc);
}
MessagingStatisticsGroup.OnMessageSend(msg.TargetSilo, msg.Direction, bytesSent, headerLength, SocketDirection.GatewayToClient);
bool sendError = exceptionSending || countMismatchSending;
if (sendError)
{
gateway.RecordClosedSocket(sock);
SocketManager.CloseSocket(sock);
}
gatewaySends.Increment();
msg.ReleaseBodyAndHeaderBuffers();
return(!sendError);
}
internal static void OnTransactionInDoubt()
{
CommitTransactionQueueLength.DecrementBy(1);
TransactionsInDoubt.Increment();
}
private bool Send(Message msg, Socket sock)
{
if (Cts.IsCancellationRequested)
{
return(false);
}
if (sock == null)
{
return(false);
}
// Send the message
List <ArraySegment <byte> > data;
int headerLength;
try
{
data = msg.Serialize(out headerLength);
}
catch (Exception exc)
{
OnMessageSerializationFailure(msg, exc);
return(true);
}
int length = data.Sum(x => x.Count);
int bytesSent = 0;
bool exceptionSending = false;
bool countMismatchSending = false;
string sendErrorStr;
try
{
bytesSent = sock.Send(data);
if (bytesSent != length)
{
// The complete message wasn't sent, even though no error was reported; treat this as an error
countMismatchSending = true;
sendErrorStr = String.Format("Byte count mismatch on send: sent {0}, expected {1}", bytesSent, length);
Log.Warn(ErrorCode.GatewayByteCountMismatch, sendErrorStr);
}
}
catch (Exception exc)
{
exceptionSending = true;
string remoteEndpoint = "";
if (!(exc is ObjectDisposedException))
{
try
{
remoteEndpoint = sock.RemoteEndPoint.ToString();
}
catch (Exception) {}
}
sendErrorStr = String.Format("Exception sending to client at {0}: {1}", remoteEndpoint, exc);
Log.Warn(ErrorCode.GatewayExceptionSendingToClient, sendErrorStr, exc);
}
MessagingStatisticsGroup.OnMessageSend(msg.TargetSilo, msg.Direction, bytesSent, headerLength, SocketDirection.GatewayToClient);
bool sendError = exceptionSending || countMismatchSending;
if (sendError)
{
gateway.RecordClosedSocket(sock);
SocketManager.CloseSocket(sock);
}
gatewaySends.Increment();
msg.ReleaseBodyAndHeaderBuffers();
return(!sendError);
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null && consumerData.Cursor.MoveNext())
{
IBatchContainer batch = null;
Exception ex;
Task deliveryTask;
try
{
batch = consumerData.Cursor.GetCurrent(out ex);
}
catch (DataNotAvailableException dataNotAvailable)
{
ex = dataNotAvailable;
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId);
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
if (batch != null)
{
deliveryTask = consumerData.StreamConsumer
.DeliverBatch(consumerData.SubscriptionId, batch.AsImmutable());
}
else if (ex == null)
{
deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId);
}
else
{
deliveryTask = consumerData.StreamConsumer.ErrorInStream(consumerData.SubscriptionId, ex);
}
try
{
numSentMessagesCounter.Increment();
await deliveryTask;
}
catch (Exception exc)
{
var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId);
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc);
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
throw;
}
}
/// <summary>
/// Determines whether the provided <see cref="T:System.Threading.Tasks.Task"/> can be executed synchronously in this call, and if it can, executes it.
/// </summary>
/// <returns>
/// A Boolean value indicating whether the task was executed inline.
/// </returns>
/// <param name="task">The <see cref="T:System.Threading.Tasks.Task"/> to be executed.</param>
/// <param name="taskWasPreviouslyQueued">A Boolean denoting whether or not task has previously been queued. If this parameter is True, then the task may have been previously queued (scheduled); if False, then the task is known not to have been queued, and this call is being made in order to execute the task inline without queuing it.</param>
protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
{
bool canExecuteInline = WorkerPoolThread.CurrentWorkerThread != null;
RuntimeContext ctx = RuntimeContext.Current;
bool canExecuteInline2 = canExecuteInline && ctx != null && object.Equals(ctx.ActivationContext, workerGroup.SchedulingContext);
canExecuteInline = canExecuteInline2;
#if DEBUG
if (logger.IsVerbose2)
{
logger.Verbose2(myId + " --> TryExecuteTaskInline Task Id={0} Status={1} PreviouslyQueued={2} CanExecute={3} Queued={4}",
task.Id, task.Status, taskWasPreviouslyQueued, canExecuteInline, workerGroup.ExternalWorkItemCount);
}
#endif
if (!canExecuteInline)
{
return(false);
}
// If the task was previously queued, remove it from the queue
if (taskWasPreviouslyQueued)
{
canExecuteInline = TryDequeue(task);
}
if (!canExecuteInline)
{
#if DEBUG
if (logger.IsVerbose2)
{
logger.Verbose2(myId + " <-X TryExecuteTaskInline Task Id={0} Status={1} Execute=No", task.Id, task.Status);
}
#endif
return(false);
}
#if EXTRA_STATS
turnsExecutedStatistic.Increment();
#endif
#if DEBUG
if (logger.IsVerbose3)
{
logger.Verbose3(myId + " TryExecuteTaskInline Task Id={0} Thread={1} Execute=Yes", task.Id, Thread.CurrentThread.ManagedThreadId);
}
#endif
// Try to run the task.
bool done = TryExecuteTask(task);
if (!done)
{
logger.Warn(ErrorCode.SchedulerTaskExecuteIncomplete3, "TryExecuteTaskInline: Incomplete base.TryExecuteTask for Task Id={0} with Status={1}",
task.Id, task.Status);
}
#if DEBUG
if (logger.IsVerbose2)
{
logger.Verbose2(myId + " <-- TryExecuteTaskInline Task Id={0} Thread={1} Execute=Done Ok={2}", task.Id, Thread.CurrentThread.ManagedThreadId, done);
}
#endif
return(done);
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null && consumerData.Cursor.MoveNext())
{
IBatchContainer batch = null;
Exception ex;
Task deliveryTask;
bool deliveryFailed = false;
try
{
batch = consumerData.Cursor.GetCurrent(out ex);
}
catch (DataNotAvailableException dataNotAvailable)
{
ex = dataNotAvailable;
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId);
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
if (batch != null)
{
deliveryTask = AsyncExecutorWithRetries.ExecuteWithRetries(i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is DataNotAvailableException), config.MaxEventDeliveryTime, DefaultBackoffProvider);
}
else if (ex == null)
{
deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId);
}
else
{
// If data is not avialable, bring cursor current
if (ex is DataNotAvailableException)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid,
consumerData.StreamId.Namespace, null);
}
// Notify client of error.
deliveryTask = DeliverErrorToConsumer(consumerData, ex, null);
}
try
{
numSentMessagesCounter.Increment();
await deliveryTask;
}
catch (Exception exc)
{
var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId);
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc);
deliveryFailed = true;
}
// if we failed to deliver a batch
if (deliveryFailed && batch != null)
{
// notify consumer of delivery error, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(() => DeliverErrorToConsumer(consumerData, new StreamEventDeliveryFailureException(consumerData.StreamId), batch));
// record that there was a delivery failure
await streamFailureHandler.OnDeliveryFailure(consumerData.SubscriptionId, streamProviderName,
consumerData.StreamId, batch.SequenceToken);
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
DeliverErrorToConsumer(consumerData,
new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch)
.Ignore();
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
throw;
}
}
internal static void OnTransactionAborted()
{
CommitTransactionQueueLength.DecrementBy(1);
AbortedTransactionsTotal.Increment();
}
public async Task RegisterConsumer(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
counterConsumersAdded.Increment();
PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId));
if (pubSubState != null && pubSubState.IsFaulted)
{
throw new FaultedSubscriptionException(subscriptionId, streamId);
}
try
{
if (pubSubState == null)
{
pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer);
State.Consumers.Add(pubSubState);
}
if (filter != null)
{
pubSubState.AddFilter(filter);
}
LogPubSubCounts("RegisterConsumer {0}", streamConsumer);
await WriteStateAsync();
counterConsumersTotal.Increment();
}
catch (Exception exc)
{
logger.Error(ErrorCode.Stream_RegisterConsumerFailed,
$"Failed to register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
int numProducers = State.Producers.Count;
if (numProducers <= 0)
{
return;
}
if (logger.IsEnabled(LogLevel.Debug))
{
logger.Debug("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}",
numProducers, streamConsumer, Utils.EnumerableToString(State.Producers));
}
// Notify producers about a new streamConsumer.
var tasks = new List <Task>();
var producers = State.Producers.ToList();
int initialProducerCount = producers.Count;
try
{
foreach (PubSubPublisherState producerState in producers)
{
tasks.Add(ExecuteProducerTask(producerState, producerState.Producer.AddSubscriber(subscriptionId, streamId, streamConsumer, filter)));
}
Exception exception = null;
try
{
await Task.WhenAll(tasks);
}
catch (Exception exc)
{
exception = exc;
}
// if the number of producers has been changed, resave state.
if (State.Producers.Count != initialProducerCount)
{
await WriteStateAsync();
counterConsumersTotal.DecrementBy(initialProducerCount - State.Producers.Count);
}
if (exception != null)
{
throw exception;
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.Stream_RegisterConsumerFailed,
$"Failed to update producers while register a stream consumer. Stream: {streamId}, SubscriptionId {subscriptionId}, Consumer: {streamConsumer}", exc);
// Corrupted state, deactivate grain.
DeactivateOnIdle();
throw;
}
}
internal static void OnTransactionStarted()
{
StartTransactionQueueLength.DecrementBy(1);
StartTransactionCompleted.Increment();
}
/// <summary>
/// Determines whether the provided <see cref="T:System.Threading.Tasks.Task"/> can be executed synchronously in this call, and if it can, executes it.
/// </summary>
/// <returns>
/// A Boolean value indicating whether the task was executed inline.
/// </returns>
/// <param name="task">The <see cref="T:System.Threading.Tasks.Task"/> to be executed.</param>
/// <param name="taskWasPreviouslyQueued">A Boolean denoting whether or not task has previously been queued. If this parameter is True, then the task may have been previously queued (scheduled); if False, then the task is known not to have been queued, and this call is being made in order to execute the task inline without queuing it.</param>
protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued)
{
var currentContext = RuntimeContext.Current;
bool canExecuteInline = currentContext != null && object.Equals(currentContext, workerGroup.GrainContext);
#if DEBUG
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace(
"{TaskScheduler} TryExecuteTaskInline Task Id={TaskId} Status={Status} PreviouslyQueued={PreviouslyQueued} CanExecute={CanExecute} Queued={Queued}",
myId,
task.Id,
task.Status,
taskWasPreviouslyQueued,
canExecuteInline,
workerGroup.ExternalWorkItemCount);
}
#endif
if (!canExecuteInline)
{
return(false);
}
// If the task was previously queued, remove it from the queue
if (taskWasPreviouslyQueued)
{
canExecuteInline = TryDequeue(task);
}
if (!canExecuteInline)
{
#if DEBUG
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace("{TaskScheduler} Completed TryExecuteTaskInline Task Id={TaskId} Status={Status} Execute=No", myId, task.Id, task.Status);
}
#endif
return(false);
}
#if EXTRA_STATS
turnsExecutedStatistic.Increment();
#endif
#if DEBUG
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace(
"{TaskScheduler} TryExecuteTaskInline Task Id={TaskId} Thread={Thread} Execute=Yes",
myId,
task.Id,
Thread.CurrentThread.ManagedThreadId);
}
#endif
// Try to run the task.
bool done = TryExecuteTask(task);
if (!done)
{
logger.LogWarning(
(int)ErrorCode.SchedulerTaskExecuteIncomplete3,
"TryExecuteTaskInline: Incomplete base.TryExecuteTask for Task Id={TaskId} with Status={TaskStatus}",
task.Id,
task.Status);
}
#if DEBUG
if (logger.IsEnabled(LogLevel.Trace))
{
logger.LogTrace(
"{TaskScheduler} Completed TryExecuteTaskInline Task Id={TaskId} Thread={Thread} Execute=Done Ok={Ok}",
myId,
task.Id,
Thread.CurrentThread.ManagedThreadId,
done);
}
#endif
return(done);
}
internal static void OnTransactionCommitted()
{
CommitTransactionQueueLength.DecrementBy(1);
CommitTransactionCompleted.Increment();
}
private async Task RunConsumerCursor(StreamConsumerData consumerData)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
batch = GetBatchForConsumer(consumerData.Cursor, consumerData.StreamId, consumerData.FilterData);
if (batch == null)
{
break;
}
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
if (batch != null)
{
if (!ShouldDeliverBatch(consumerData.StreamId, batch, consumerData.FilterData))
{
continue;
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException) || IsShutdown,
this.options.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
public async Task RegisterConsumer(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
IStreamFilterPredicateWrapper filter)
{
PubSubSubscriptionState pubSubState = State.Consumers.FirstOrDefault(s => s.Equals(subscriptionId));
if (pubSubState == null)
{
pubSubState = new PubSubSubscriptionState(subscriptionId, streamId, streamConsumer);
State.Consumers.Add(pubSubState);
}
if (pubSubState.IsFaulted)
{
throw new FaultedSubscriptionException(subscriptionId, streamId);
}
if (filter != null)
{
pubSubState.AddFilter(filter);
}
counterConsumersAdded.Increment();
counterConsumersTotal.Increment();
LogPubSubCounts("RegisterConsumer {0}", streamConsumer);
await WriteStateAsync();
int numProducers = State.Producers.Count;
if (numProducers <= 0)
{
return;
}
if (logger.IsVerbose)
{
logger.Info("Notifying {0} existing producer(s) about new consumer {1}. Producers={2}",
numProducers, streamConsumer, Utils.EnumerableToString(State.Producers));
}
// Notify producers about a new streamConsumer.
var tasks = new List <Task>();
var producers = State.Producers.ToList();
int initialProducerCount = producers.Count;
foreach (var producerState in producers)
{
PubSubPublisherState producer = producerState; // Capture loop variable
if (!IsActiveProducer(producer.Producer))
{
// Producer is not active (could be stopping / shutting down) so skip
if (logger.IsVerbose)
{
logger.Verbose("Producer {0} on stream {1} is not active - skipping.", producer, streamId);
}
continue;
}
tasks.Add(NotifyProducer(producer, subscriptionId, streamId, streamConsumer, filter));
}
Exception exception = null;
try
{
await Task.WhenAll(tasks);
}
catch (Exception exc)
{
exception = exc;
}
// if the number of producers has been changed, resave state.
if (State.Producers.Count != initialProducerCount)
{
await WriteStateAsync();
}
if (exception != null)
{
throw exception;
}
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null)
{
IBatchContainer batch = null;
Exception exceptionOccured = null;
try
{
Exception ignore;
if (!consumerData.Cursor.MoveNext())
{
break;
}
batch = consumerData.Cursor.GetCurrent(out ignore);
}
catch (Exception exc)
{
exceptionOccured = exc;
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId, null);
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message =
$"Ignoring exception while trying to evaluate subscription filter function {filterWrapper} on stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
try
{
numSentMessagesCounter.Increment();
if (batch != null)
{
StreamHandshakeToken newToken = await AsyncExecutorWithRetries.ExecuteWithRetries(
i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES,
(exception, i) => !(exception is ClientNotAvailableException),
config.MaxEventDeliveryTime,
DeliveryBackoffProvider);
if (newToken != null)
{
consumerData.LastToken = newToken;
IQueueCacheCursor newCursor = queueCache.GetCacheCursor(consumerData.StreamId, newToken.Token);
consumerData.SafeDisposeCursor(logger);
consumerData.Cursor = newCursor;
}
}
}
catch (Exception exc)
{
consumerData.Cursor?.RecordDeliveryFailure();
var message =
$"Exception while trying to deliver msgs to stream {consumerData.StreamId} in PersistentStreamPullingAgentGrain.RunConsumerCursor";
logger.Error(ErrorCode.PersistentStreamPullingAgent_14, message, exc);
exceptionOccured = exc is ClientNotAvailableException
? exc
: new StreamEventDeliveryFailureException(consumerData.StreamId);
}
// if we failed to deliver a batch
if (exceptionOccured != null)
{
bool faultedSubscription = await ErrorProtocol(consumerData, exceptionOccured, true, batch, batch?.SequenceToken);
if (faultedSubscription)
{
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error(ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
consumerData.State = StreamConsumerDataState.Inactive;
throw;
}
}
internal static void OnTransactionStartRequest()
{
StartTransactionQueueLength.Increment();
StartTransactionRequests.Increment();
}
