/// <summary>
/// Called from generated code.
/// </summary>
protected Task <T> InvokeMethodAsync <T>(int methodId, object[] arguments, InvokeMethodOptions options = InvokeMethodOptions.None, SiloAddress silo = null)
{
object[] argsDeepCopy = null;
if (arguments != null)
{
CheckForGrainArguments(arguments);
SetGrainCancellationTokensTarget(arguments, this);
argsDeepCopy = (object[])SerializationManager.DeepCopy(arguments);
}
var request = new InvokeMethodRequest(this.InterfaceId, methodId, argsDeepCopy);
if (IsUnordered)
{
options |= InvokeMethodOptions.Unordered;
}
Task <object> resultTask = InvokeMethod_Impl(request, null, options);
if (resultTask == null)
{
if (typeof(T) == typeof(object))
{
// optimize for most common case when using one way calls.
return(PublicOrleansTaskExtensions.CompletedTask as Task <T>);
}
return(Task.FromResult(default(T)));
}
resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask);
return(resultTask.Unbox <T>());
}
private async Task RemoveQueues(List <QueueId> queuesToRemove)
{
if (queuesToRemove.Count == 0)
{
return;
}
// Stop the agents that for queues that are not in my range anymore.
var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count);
Log(ErrorCode.PersistentStreamPullingManager_10, "About to remove {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToString()));
var removeTasks = new List <Task>();
foreach (var queueId in queuesToRemove)
{
PersistentStreamPullingAgent agent;
if (!queuesToAgentsMap.TryGetValue(queueId, out agent))
{
continue;
}
agents.Add(agent);
queuesToAgentsMap.Remove(queueId);
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown);
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11,
String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId));
removeTasks.Add(task);
}
try
{
await Task.WhenAll(removeTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
foreach (var agent in agents)
{
try
{
providerRuntime.UnregisterSystemTarget(agent);
}
catch (Exception exc)
{
Log(ErrorCode.PersistentStreamPullingManager_12,
"Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", ((ISystemTargetBase)agent).GrainId, exc.Message);
}
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_10, "Removed {0} queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
private void InitializeInternal()
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), Silo, QueueId.ToStringWithHashCode());
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(this.options.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = ThreadSafeRandom.NextTimeSpan(this.options.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private void InternalUnobservedTaskExceptionHandler(object sender, UnobservedTaskExceptionEventArgs e)
{
var aggrException = e.Exception;
var baseException = aggrException.GetBaseException();
var tplTask = (Task)sender;
var contextObj = tplTask.AsyncState;
var context = contextObj as ISchedulingContext;
try
{
UnobservedExceptionHandler(context, baseException);
}
finally
{
if (e.Observed)
{
logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an UnobservedTaskException which was successfully observed and recovered from. BaseException = {0}. Exception = {1}",
baseException.Message, LogFormatter.PrintException(aggrException));
}
else
{
var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}",
OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException));
logger.Error(ErrorCode.Runtime_Error_100005, errorStr);
logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version.");
}
}
}
/// <summary>
/// Called from generated code.
/// </summary>
protected async Task <T> InvokeMethodAsync <T>(int methodId, object[] arguments, InvokeMethodOptions options = InvokeMethodOptions.None, SiloAddress silo = null)
{
object[] argsDeepCopy = null;
if (arguments != null)
{
CheckForGrainArguments(arguments);
argsDeepCopy = (object[])SerializationManager.DeepCopy(arguments);
}
var request = new InvokeMethodRequest(this.InterfaceId, methodId, argsDeepCopy);
if (IsUnordered)
{
options |= InvokeMethodOptions.Unordered;
}
Task <object> resultTask = InvokeMethod_Impl(request, null, options);
if (resultTask == null)
{
return(default(T));
}
resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask);
return((T)await resultTask);
}
public async Task LocalhostSiloTest()
{
using var portAllocator = new TestClusterPortAllocator();
var(siloPort, gatewayPort) = portAllocator.AllocateConsecutivePortPairs(1);
var silo = new SiloHostBuilder()
.AddMemoryGrainStorage("MemoryStore")
.UseLocalhostClustering(siloPort, gatewayPort)
.Build();
var client = new ClientBuilder()
.UseLocalhostClustering(gatewayPort)
.Build();
try
{
await silo.StartAsync();
await client.Connect();
var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid());
var result = await grain.Echo("test");
Assert.Equal("test", result);
}
finally
{
await OrleansTaskExtentions.SafeExecute(() => silo.StopAsync());
await OrleansTaskExtentions.SafeExecute(() => client.Close());
Utils.SafeExecute(() => silo.Dispose());
Utils.SafeExecute(() => client.Close());
}
}
/// <inheritdoc />
public Task <T> InvokeMethodAsync <T>(GrainReference reference, int methodId, object[] arguments, InvokeMethodOptions options, SiloAddress silo)
{
if (arguments != null)
{
CheckForGrainArguments(arguments);
SetGrainCancellationTokensTarget(arguments, reference);
this.serializationManager.DeepCopyElementsInPlace(arguments);
}
var request = new InvokeMethodRequest(reference.InterfaceId, reference.InterfaceVersion, methodId, arguments);
if (IsUnordered(reference))
{
options |= InvokeMethodOptions.Unordered;
}
Task <object> resultTask = InvokeMethod_Impl(reference, request, options);
if (resultTask == null)
{
if (typeof(T) == typeof(object))
{
// optimize for most common case when using one way calls.
return(OrleansTaskExtentions.CompletedTask as Task <T>);
}
return(Task.FromResult(default(T)));
}
#if !NETCOREAPP
resultTask = OrleansTaskExtentions.ConvertTaskViaTcs(resultTask);
#endif
return(resultTask.ToTypedTask <T>());
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
if (failureHandler.Value == null)
{
throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod);
timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
public async Task LocalhostSiloTest()
{
var opts = TestSiloSpecificOptions.Create(new TestClusterOptions(), 1, true);
var silo = new SiloHostBuilder()
.AddMemoryGrainStorage("MemoryStore")
.UseLocalhostClustering(opts.SiloPort, opts.GatewayPort)
.Build();
var client = new ClientBuilder()
.UseLocalhostClustering(opts.GatewayPort)
.Build();
try
{
await silo.StartAsync();
await client.Connect();
var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid());
var result = await grain.Echo("test");
Assert.Equal("test", result);
}
finally
{
await OrleansTaskExtentions.SafeExecute(() => silo.StopAsync());
await OrleansTaskExtentions.SafeExecute(() => client.Close());
Utils.SafeExecute(() => silo.Dispose());
Utils.SafeExecute(() => client.Close());
}
}
private void InternalUnobservedTaskExceptionHandler(object sender, System.Threading.Tasks.UnobservedTaskExceptionEventArgs e)
{
var aggrException = e.Exception;
var baseException = aggrException.GetBaseException();
var context = (sender as Task)?.AsyncState;
try
{
this.logger.LogError((int)ErrorCode.Runtime_Error_100104, baseException, "Silo caught an unobserved exception thrown from context {Context}: {Exception}", context, baseException);
}
finally
{
if (e.Observed)
{
logger.Info(ErrorCode.Runtime_Error_100311, "Silo caught an unobserved exception which was successfully observed and recovered from. BaseException = {0}. Exception = {1}",
baseException.Message, LogFormatter.PrintException(aggrException));
}
else
{
var errorStr = String.Format("Silo Caught an UnobservedTaskException event sent by {0}. Exception = {1}",
OrleansTaskExtentions.ToString((Task)sender), LogFormatter.PrintException(aggrException));
logger.Error(ErrorCode.Runtime_Error_100005, errorStr);
logger.Error(ErrorCode.Runtime_Error_100006, "Exception remained UnObserved!!! The subsequent behavior depends on the ThrowUnobservedTaskExceptions setting in app config and .NET version.");
}
}
}
private async Task <bool> ErrorProtocol(StreamConsumerData consumerData, Exception exceptionOccured, bool isDeliveryError, IBatchContainer batch, StreamSequenceToken token)
{
// for loss of client, we just remove the subscription
if (exceptionOccured is ClientNotAvailableException)
{
logger.LogWarning(
(int)ErrorCode.Stream_ConsumerIsDead,
"Consumer {Consumer} on stream {StreamId} is no longer active - permanently removing Consumer.",
consumerData.StreamConsumer,
consumerData.StreamId);
pubSub.UnregisterConsumer(consumerData.SubscriptionId, consumerData.StreamId).Ignore();
return(true);
}
// notify consumer about the error or that the data is not available.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, exceptionOccured, batch));
// record that there was a delivery failure
if (isDeliveryError)
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnDeliveryFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
else
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnSubscriptionFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch));
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return(true);
}
return(false);
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
initTasks.Add(task);
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
if (failureHandler.Value == null)
{
throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
}
return(OrleansTaskExtentions.WrapInTask(() => InitializeInternal(qAdapter.Value, queueAdapterCache.Value, failureHandler.Value)));
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private async Task RemoveQueues(IEnumerable <QueueId> myQueues)
{
// Stop the agents that for queues that are not in my range anymore.
List <QueueId> queuesToRemove = queuesToAgentsMap.Keys.Where(queueId => !myQueues.Contains(queueId)).ToList();
var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_10, "Removing {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToStringWithHashCode()));
var removeTasks = new List <Task>();
foreach (var queueId in queuesToRemove)
{
PersistentStreamPullingAgent agent;
if (!queuesToAgentsMap.TryGetValue(queueId, out agent))
{
continue;
}
agents.Add(agent);
queuesToAgentsMap.Remove(queueId);
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown);
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11,
String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId));
removeTasks.Add(task);
}
try
{
await Task.WhenAll(removeTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
foreach (var agent in agents)
{
try
{
providerRuntime.UnRegisterSystemTarget(agent);
}
catch (Exception exc)
{
logger.Info((int)ErrorCode.PersistentStreamPullingManager_12,
"Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", agent.GrainId, exc.Message);
}
}
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
Utils.SafeExecute(tmp.Dispose);
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(config.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId));
await task;
}
catch
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
foreach (var tuple in pubSubCache)
{
tuple.Value.DisposeAll(logger);
var streamId = tuple.Key;
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer));
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc);
}
pubSubCache.Clear();
IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix));
//IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix));
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", this.GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
tmp.Dispose();
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = StreamProducerExtensionFactory.Cast(this.AsReference());
foreach (var streamId in pubSubCache.Keys)
{
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer));
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId));
await task;
}
catch (Exception)
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams taht used to be in my responsibility.", exc);
}
}
private async Task <bool> ErrorProtocol(StreamConsumerData consumerData, Exception exceptionOccured, bool isDeliveryError, IBatchContainer batch, StreamSequenceToken token)
{
// notify consumer about the error or that the data is not available.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, exceptionOccured, batch));
// record that there was a delivery failure
if (isDeliveryError)
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnDeliveryFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
else
{
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => streamFailureHandler.OnSubscriptionFailure(
consumerData.SubscriptionId, streamProviderName, consumerData.StreamId, token));
}
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(
() => DeliverErrorToConsumer(
consumerData, new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch));
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return(true);
}
return(false);
}
public Task Cancel(Guid id, CancellationTokenSource tokenSource, ConcurrentDictionary <GrainId, GrainReference> grainReferences)
{
// propagate the exception from the _cancellationTokenSource.Cancel back to the caller
// but also cancel _targetGrainReferences.
Task task = OrleansTaskExtentions.WrapInTask(tokenSource.Cancel);
if (grainReferences.IsEmpty)
{
return(task);
}
var cancellationTasks = grainReferences
.Select(pair => pair.Value.AsReference <ICancellationSourcesExtension>())
.Select(tokenExtension => CancelTokenWithRetries(id, tokenExtension))
.ToList();
cancellationTasks.Add(task);
return(Task.WhenAll(cancellationTasks));
}
// Called by rendezvous when new remote subscriber subscribes to this stream.
private async Task AddSubscriber_Impl(
GuidId subscriptionId,
StreamId streamId,
IStreamConsumerExtension streamConsumer,
StreamSequenceToken token,
IStreamFilterPredicateWrapper filter)
{
IQueueCacheCursor cursor = null;
StreamSequenceToken requestedToken = null;
// if not cache, then we can't get cursor and there is no reason to ask consumer for token.
if (queueCache != null)
{
DataNotAvailableException errorOccured = null;
try
{
requestedToken = await streamConsumer.GetSequenceToken(subscriptionId);
// Set cursor if not cursor is set, or if subscription provides new token
requestedToken = requestedToken ?? token;
if (requestedToken != null)
{
cursor = queueCache.GetCacheCursor(streamId.Guid, streamId.Namespace, requestedToken);
}
}
catch (DataNotAvailableException dataNotAvailableException)
{
errorOccured = dataNotAvailableException;
}
if (errorOccured != null)
{
// notify consumer that the data is not available, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(() => streamConsumer.ErrorInStream(subscriptionId, errorOccured));
}
}
AddSubscriberToSubscriptionCache(subscriptionId, streamId, streamConsumer, cursor, requestedToken, filter);
}
public async Task LocalhostSiloTest()
{
using var portAllocator = new TestClusterPortAllocator();
var(siloPort, gatewayPort) = portAllocator.AllocateConsecutivePortPairs(1);
var host = new HostBuilder().UseOrleans(siloBuilder =>
{
siloBuilder.AddMemoryGrainStorage("MemoryStore")
.UseLocalhostClustering(siloPort, gatewayPort);
}).Build();
var clientHost = new HostBuilder().UseOrleansClient(clientBuilder =>
{
clientBuilder.UseLocalhostClustering(gatewayPort);
}).Build();
var client = clientHost.Services.GetRequiredService <IClusterClient>();
try
{
await Task.WhenAll(host.StartAsync(), clientHost.StartAsync());
var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid());
var result = await grain.Echo("test");
Assert.Equal("test", result);
}
finally
{
await OrleansTaskExtentions.SafeExecute(() => host.StopAsync());
await OrleansTaskExtentions.SafeExecute(() => clientHost.StopAsync());
Utils.SafeExecute(() => host.Dispose());
Utils.SafeExecute(() => clientHost.Dispose());
}
}
private async Task RunConsumerCursor(StreamConsumerData consumerData, IStreamFilterPredicateWrapper filterWrapper)
{
try
{
// double check in case of interleaving
if (consumerData.State == StreamConsumerDataState.Active ||
consumerData.Cursor == null)
{
return;
}
consumerData.State = StreamConsumerDataState.Active;
while (consumerData.Cursor != null && consumerData.Cursor.MoveNext())
{
IBatchContainer batch = null;
Exception ex;
Task deliveryTask;
bool deliveryFailed = false;
try
{
batch = consumerData.Cursor.GetCurrent(out ex);
}
catch (DataNotAvailableException dataNotAvailable)
{
ex = dataNotAvailable;
}
// Apply filtering to this batch, if applicable
if (filterWrapper != null && batch != null)
{
try
{
// Apply batch filter to this input batch, to see whether we should deliver it to this consumer.
if (!batch.ShouldDeliver(
consumerData.StreamId,
filterWrapper.FilterData,
filterWrapper.ShouldReceive))
{
continue; // Skip this batch -- nothing to do
}
}
catch (Exception exc)
{
var message = string.Format("Ignoring exception while trying to evaluate subscription filter function {0} on stream {1} in PersistentStreamPullingAgentGrain.RunConsumerCursor", filterWrapper, consumerData.StreamId);
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_13, message, exc);
}
}
if (batch != null)
{
deliveryTask = AsyncExecutorWithRetries.ExecuteWithRetries(i => DeliverBatchToConsumer(consumerData, batch),
AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !(exception is DataNotAvailableException), config.MaxEventDeliveryTime, DefaultBackoffProvider);
}
else if (ex == null)
{
deliveryTask = consumerData.StreamConsumer.CompleteStream(consumerData.SubscriptionId);
}
else
{
// If data is not avialable, bring cursor current
if (ex is DataNotAvailableException)
{
consumerData.Cursor = queueCache.GetCacheCursor(consumerData.StreamId.Guid,
consumerData.StreamId.Namespace, null);
}
// Notify client of error.
deliveryTask = DeliverErrorToConsumer(consumerData, ex, null);
}
try
{
numSentMessagesCounter.Increment();
await deliveryTask;
}
catch (Exception exc)
{
var message = string.Format("Exception while trying to deliver msgs to stream {0} in PersistentStreamPullingAgentGrain.RunConsumerCursor", consumerData.StreamId);
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_14, message, exc);
deliveryFailed = true;
}
// if we failed to deliver a batch
if (deliveryFailed && batch != null)
{
// notify consumer of delivery error, if we can.
await OrleansTaskExtentions.ExecuteAndIgnoreException(() => DeliverErrorToConsumer(consumerData, new StreamEventDeliveryFailureException(consumerData.StreamId), batch));
// record that there was a delivery failure
await streamFailureHandler.OnDeliveryFailure(consumerData.SubscriptionId, streamProviderName,
consumerData.StreamId, batch.SequenceToken);
// if configured to fault on delivery failure and this is not an implicit subscription, fault and remove the subscription
if (streamFailureHandler.ShouldFaultSubsriptionOnError && !SubscriptionMarker.IsImplicitSubscription(consumerData.SubscriptionId.Guid))
{
try
{
// notify consumer of faulted subscription, if we can.
DeliverErrorToConsumer(consumerData,
new FaultedSubscriptionException(consumerData.SubscriptionId, consumerData.StreamId), batch)
.Ignore();
// mark subscription as faulted.
await pubSub.FaultSubscription(consumerData.StreamId, consumerData.SubscriptionId);
}
finally
{
// remove subscription
RemoveSubscriber_Impl(consumerData.SubscriptionId, consumerData.StreamId);
}
return;
}
}
}
consumerData.State = StreamConsumerDataState.Inactive;
}
catch (Exception exc)
{
// RunConsumerCursor is fired with .Ignore so we should log if anything goes wrong, because there is no one to catch the exception
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_15, "Ignored RunConsumerCursor Error", exc);
throw;
}
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info(ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
Utils.SafeExecute(tmp.Dispose, this.logger);
try
{
await tmp.GetCurrentlyExecutingTickTask().WithTimeout(TimeSpan.FromSeconds(5));
}
catch (Exception ex)
{
this.logger.LogWarning(ex, "Waiting for the last timer tick failed");
}
}
this.queueCache = null;
Task localReceiverInitTask = receiverInitTask;
if (localReceiverInitTask != null)
{
try
{
await localReceiverInitTask;
receiverInitTask = null;
}
catch (Exception)
{
receiverInitTask = null;
// squelch
}
}
try
{
IQueueAdapterReceiver localReceiver = this.receiver;
this.receiver = null;
if (localReceiver != null)
{
var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(this.options.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
$"QueueAdapterReceiver {QueueId} failed to Shutdown.");
await task;
}
}
catch
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
foreach (var tuple in pubSubCache)
{
tuple.Value.DisposeAll(logger);
var streamId = tuple.Key;
logger.Info(ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, meAsStreamProducer));
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc);
}
pubSubCache.Clear();
IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix));
//IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix));
}
// Execute one or more turns for this activation.
// This method is always called in a single-threaded environment -- that is, no more than one
// thread will be in this method at once -- but other asynch threads may still be queueing tasks, etc.
public void Execute()
{
lock (lockable)
{
if (state == WorkGroupStatus.Shutdown)
{
if (!IsActive)
{
return; // Don't mind if no work has been queued to this work group yet.
}
ReportWorkGroupProblemWithBacktrace(
"Cannot execute work items in a work item group that is in a shutdown state.",
ErrorCode.SchedulerNotExecuteWhenShutdown); // Throws InvalidOperationException
return;
}
state = WorkGroupStatus.Running;
}
var thread = WorkerPoolThread.CurrentWorkerThread;
try
{
// Process multiple items -- drain the applicationMessageQueue (up to max items) for this physical activation
int count = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
do
{
lock (lockable)
{
if (state == WorkGroupStatus.Shutdown)
{
if (WorkItemCount > 0)
{
log.Warn(ErrorCode.SchedulerSkipWorkStopping, "Thread {0} is exiting work loop due to Shutdown state {1} while still having {2} work items in the queue.",
thread.ToString(), this.ToString(), WorkItemCount);
}
else
if (log.IsVerbose)
{
log.Verbose("Thread {0} is exiting work loop due to Shutdown state {1}. Has {2} work items in the queue.",
thread.ToString(), this.ToString(), WorkItemCount);
}
break;
}
// Check the cancellation token (means that the silo is stopping)
if (thread.CancelToken.IsCancellationRequested)
{
log.Warn(ErrorCode.SchedulerSkipWorkCancelled, "Thread {0} is exiting work loop due to cancellation token. WorkItemGroup: {1}, Have {2} work items in the queue.",
thread.ToString(), this.ToString(), WorkItemCount);
break;
}
}
// Get the first Work Item on the list
Task task;
lock (lockable)
{
if (workItems.Count > 0)
{
task = workItems.Dequeue();
}
else // If the list is empty, then we're done
{
break;
}
}
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectGlobalShedulerStats)
{
SchedulerStatisticsGroup.OnWorkItemDequeue();
}
#endif
#if DEBUG
if (log.IsVerbose2)
{
log.Verbose2("About to execute task {0} in SchedulingContext={1}", task, SchedulingContext);
}
#endif
var taskStart = stopwatch.Elapsed;
try
{
thread.CurrentTask = task;
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectTurnsStats)
{
SchedulerStatisticsGroup.OnTurnExecutionStartsByWorkGroup(workItemGroupStatisticsNumber, thread.WorkerThreadStatisticsNumber, SchedulingContext);
}
#endif
TaskRunner.RunTask(task);
}
catch (Exception ex)
{
log.Error(ErrorCode.SchedulerExceptionFromExecute, String.Format("Worker thread caught an exception thrown from Execute by task {0}", task), ex);
throw;
}
finally
{
#if TRACK_DETAILED_STATS
if (StatisticsCollector.CollectTurnsStats)
{
SchedulerStatisticsGroup.OnTurnExecutionEnd(Utils.Since(thread.CurrentStateStarted));
}
if (StatisticsCollector.CollectThreadTimeTrackingStats)
{
thread.threadTracking.IncrementNumberOfProcessed();
}
#endif
totalItemsProcessed++;
var taskLength = stopwatch.Elapsed - taskStart;
if (taskLength > OrleansTaskScheduler.TurnWarningLengthThreshold)
{
SchedulerStatisticsGroup.NumLongRunningTurns.Increment();
log.Warn(ErrorCode.SchedulerTurnTooLong3, "Task {0} in WorkGroup {1} took elapsed time {2:g} for execution, which is longer than {3}. Running on thread {4}",
OrleansTaskExtentions.ToString(task), SchedulingContext.ToString(), taskLength, OrleansTaskScheduler.TurnWarningLengthThreshold, thread.ToString());
}
thread.CurrentTask = null;
}
count++;
}while (((MaxWorkItemsPerTurn <= 0) || (count <= MaxWorkItemsPerTurn)) &&
((ActivationSchedulingQuantum <= TimeSpan.Zero) || (stopwatch.Elapsed < ActivationSchedulingQuantum)));
stopwatch.Stop();
}
catch (Exception ex)
{
log.Error(ErrorCode.Runtime_Error_100032, String.Format("Worker thread {0} caught an exception thrown from IWorkItem.Execute", thread), ex);
}
finally
{
// Now we're not Running anymore.
// If we left work items on our run list, we're Runnable, and need to go back on the silo run queue;
// If our run list is empty, then we're waiting.
lock (lockable)
{
if (state != WorkGroupStatus.Shutdown)
{
if (WorkItemCount > 0)
{
state = WorkGroupStatus.Runnable;
masterScheduler.RunQueue.Add(this);
}
else
{
state = WorkGroupStatus.Waiting;
}
}
}
}
}
/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initialize again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <returns></returns>
public Task Initialize()
{
return(OrleansTaskExtentions.WrapInTask(() => InitializeInternal()));
}
