internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.Info(ErrorCode.PersistentStreamPullingAgent_01,
"Created {0} {1} for Stream Provider {2} on silo {3} for Queue {4}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToString(), streamProviderName, Silo, QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
// TODO: move queue cache size statistics tracking into queue cache implementation once Telemetry APIs and LogStatistics have been reconciled.
//IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, statUniquePostfix), () => queueCache != null ? queueCache.Size : 0);
}
public async Task <IPersistentStreamPullingManager> InitializePullingAgents(
string streamProviderName,
IQueueAdapterFactory adapterFactory,
IQueueAdapter queueAdapter)
{
IStreamQueueBalancer queueBalancer = CreateQueueBalancer(streamProviderName);
var managerId = SystemTargetGrainId.Create(Constants.StreamPullingAgentManagerType, this.siloDetails.SiloAddress, streamProviderName);
var pubsubOptions = this.ServiceProvider.GetOptionsByName <StreamPubSubOptions>(streamProviderName);
var pullingAgentOptions = this.ServiceProvider.GetOptionsByName <StreamPullingAgentOptions>(streamProviderName);
var filter = this.ServiceProvider.GetServiceByName <IStreamFilter>(streamProviderName) ?? new NoOpStreamFilter();
var manager = new PersistentStreamPullingManager(
managerId,
streamProviderName,
this.PubSub(pubsubOptions.PubSubType),
adapterFactory,
queueBalancer,
filter,
pullingAgentOptions,
this.loggerFactory,
this.siloDetails.SiloAddress,
queueAdapter);
var catalog = this.ServiceProvider.GetRequiredService <Catalog>();
catalog.RegisterSystemTarget(manager);
// Init the manager only after it was registered locally.
var pullingAgentManager = manager.AsReference <IPersistentStreamPullingManager>();
// Need to call it as a grain reference though.
await pullingAgentManager.Initialize();
return(pullingAgentManager);
}
public TGrainInterface GetSystemTarget <TGrainInterface>(GrainType grainType, SiloAddress destination)
where TGrainInterface : ISystemTarget
{
var grainId = SystemTargetGrainId.Create(grainType, destination);
return(this.GetSystemTarget <TGrainInterface>(grainId.GrainId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues)
{
if (queuesToAgentsMap.ContainsKey(queueId))
{
continue;
}
try
{
var agentIdNumber = Interlocked.Increment(ref nextAgentId);
var agentId = SystemTargetGrainId.Create(Constants.StreamPullingAgentType, this.Silo, $"{streamProviderName}_{agentIdNumber}_{queueId.ToStringWithHashCode()}");
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(queueId);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, this.loggerFactory, pubSub, streamFilter, queueId, this.options, this.Silo, queueAdapter, queueAdapterCache, deliveryFailureHandler);
this.ActivationServices.GetRequiredService <Catalog>().RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingManager_07, "Exception while creating PersistentStreamPullingAgent.", exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
initTasks.Add(InitAgent(agent));
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_08, "Added {0} new queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
protected override void OnReceivedMessage(Message msg)
{
// Don't process messages that have already timed out
if (msg.IsExpired)
{
this.MessagingTrace.OnDropExpiredMessage(msg, MessagingStatisticsGroup.Phase.Receive);
return;
}
// Are we overloaded?
if (this.overloadDetector.Overloaded)
{
MessagingStatisticsGroup.OnRejectedMessage(msg);
Message rejection = this.MessageFactory.CreateRejectionResponse(msg, Message.RejectionTypes.GatewayTooBusy, "Shedding load");
this.messageCenter.TryDeliverToProxy(rejection);
if (this.Log.IsEnabled(LogLevel.Debug))
{
this.Log.LogDebug("Rejecting a request due to overloading: {Message}", msg.ToString());
}
loadSheddingCounter.Increment();
return;
}
SiloAddress targetAddress = this.gateway.TryToReroute(msg);
msg.SendingSilo = this.myAddress;
if (targetAddress is null)
{
// reroute via Dispatcher
msg.TargetSilo = null;
msg.TargetActivation = default;
msg.ClearTargetAddress();
if (SystemTargetGrainId.TryParse(msg.TargetGrain, out var systemTargetId))
{
msg.TargetSilo = this.myAddress;
msg.TargetGrain = systemTargetId.WithSiloAddress(this.myAddress).GrainId;
msg.TargetActivation = ActivationId.GetDeterministic(msg.TargetGrain);
}
MessagingStatisticsGroup.OnMessageReRoute(msg);
this.messageCenter.RerouteMessage(msg);
}
else
{
// send directly
msg.TargetSilo = targetAddress;
if (SystemTargetGrainId.TryParse(msg.TargetGrain, out var systemTargetId))
{
msg.TargetGrain = systemTargetId.WithSiloAddress(targetAddress).GrainId;
msg.TargetActivation = ActivationId.GetDeterministic(msg.TargetGrain);
}
this.messageCenter.SendMessage(msg);
}
}
/// <summary>
/// Get a reference to the <see cref="GrainService"/> responsible for actioning the request based on the <paramref name="callingGrainId"/>.
/// </summary>
protected TGrainService GetGrainService(GrainId callingGrainId)
{
var hashCode = callingGrainId.GetUniformHashCode();
var destination = ringProvider.GetPrimaryTargetSilo(hashCode);
var grainId = SystemTargetGrainId.CreateGrainServiceGrainId(grainType, destination);
var grainService = grainFactory.GetSystemTarget <TGrainService>(grainId);
return(grainService);
}
/// <summary>
/// Currently we only support a single GrainService per Silo, when multiple are supported we will request the number of GrainServices to partition per silo here.
/// </summary>
protected GrainServiceClient(IServiceProvider serviceProvider)
{
grainFactory = serviceProvider.GetRequiredService <IInternalGrainFactory>();
ringProvider = serviceProvider.GetRequiredService <IConsistentRingProvider>();
// GrainInterfaceMap only holds IGrain types, not ISystemTarget types, so resolved via Orleans.CodeGeneration.
// Resolve this before merge.
var grainTypeCode = GrainInterfaceUtils.GetGrainClassTypeCode(typeof(TGrainService));
grainType = SystemTargetGrainId.CreateGrainServiceGrainType(grainTypeCode, null);
}
private static IGrainService GrainServiceFactory(Type serviceType, IServiceProvider services)
{
var grainServiceInterfaceType = serviceType.GetInterfaces().FirstOrDefault(x => x.GetInterfaces().Contains(typeof(IGrainService)));
if (grainServiceInterfaceType is null)
{
throw new InvalidOperationException(String.Format($"Cannot find an interface on {serviceType.FullName} which implements IGrainService"));
}
var typeCode = GrainInterfaceUtils.GetGrainClassTypeCode(grainServiceInterfaceType);
var grainId = SystemTargetGrainId.CreateGrainServiceGrainId(typeCode, null, SiloAddress.Zero);
var grainService = (IGrainService)ActivatorUtilities.CreateInstance(services, serviceType, grainId);
return(grainService);
}
internal PersistentStreamPullingManager(
SystemTargetGrainId managerId,
string strProviderName,
IStreamProviderRuntime runtime,
IStreamPubSub streamPubSub,
IQueueAdapterFactory adapterFactory,
IStreamQueueBalancer streamQueueBalancer,
IStreamFilter streamFilter,
StreamPullingAgentOptions options,
ILoggerFactory loggerFactory,
SiloAddress siloAddress)
: base(managerId, siloAddress, lowPriority: false, loggerFactory)
{
if (string.IsNullOrWhiteSpace(strProviderName))
{
throw new ArgumentNullException("strProviderName");
}
if (runtime == null)
{
throw new ArgumentNullException("runtime", "IStreamProviderRuntime runtime reference should not be null");
}
if (streamPubSub == null)
{
throw new ArgumentNullException("streamPubSub", "StreamPubSub reference should not be null");
}
if (streamQueueBalancer == null)
{
throw new ArgumentNullException("streamQueueBalancer", "IStreamQueueBalancer streamQueueBalancer reference should not be null");
}
queuesToAgentsMap = new Dictionary <QueueId, PersistentStreamPullingAgent>();
streamProviderName = strProviderName;
providerRuntime = runtime;
pubSub = streamPubSub;
this.options = options;
nonReentrancyGuarantor = new AsyncSerialExecutor();
latestRingNotificationSequenceNumber = 0;
latestCommandNumber = 0;
queueBalancer = streamQueueBalancer;
this.streamFilter = streamFilter;
this.adapterFactory = adapterFactory;
queueAdapterCache = adapterFactory.GetQueueAdapterCache();
logger = loggerFactory.CreateLogger($"{GetType().FullName}.{streamProviderName}");
Log(ErrorCode.PersistentStreamPullingManager_01, "Created {0} for Stream Provider {1}.", GetType().Name, streamProviderName);
this.loggerFactory = loggerFactory;
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS, strProviderName), () => queuesToAgentsMap.Count);
}
internal PersistentStreamPullingAgent(
SystemTargetGrainId id,
string strProviderName,
ILoggerFactory loggerFactory,
IStreamPubSub streamPubSub,
IStreamFilter streamFilter,
QueueId queueId,
StreamPullingAgentOptions options,
SiloAddress siloAddress,
IQueueAdapter queueAdapter,
IQueueAdapterCache queueAdapterCache,
IStreamFailureHandler streamFailureHandler)
: base(id, siloAddress, true, loggerFactory)
{
if (strProviderName == null)
{
throw new ArgumentNullException("runtime", "PersistentStreamPullingAgent: strProviderName should not be null");
}
QueueId = queueId;
streamProviderName = strProviderName;
pubSub = streamPubSub;
this.streamFilter = streamFilter;
pubSubCache = new Dictionary <InternalStreamId, StreamConsumerCollection>();
this.options = options;
this.queueAdapter = queueAdapter ?? throw new ArgumentNullException(nameof(queueAdapter));
this.streamFailureHandler = streamFailureHandler ?? throw new ArgumentNullException(nameof(streamFailureHandler));;
this.queueAdapterCache = queueAdapterCache;
numMessages = 0;
logger = loggerFactory.CreateLogger($"{this.GetType().Namespace}.{streamProviderName}");
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_01,
"Created {Name} {Id} for Stream Provider {StreamProvider} on silo {Silo} for Queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
streamProviderName,
Silo,
QueueId.ToStringWithHashCode());
numReadMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES, StatisticUniquePostfix));
numSentMessagesCounter = CounterStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES, StatisticUniquePostfix));
}
internal LocalReminderService(
Silo silo,
IReminderTable reminderTable,
TimeSpan initTimeout,
ILoggerFactory loggerFactory,
IAsyncTimerFactory asyncTimerFactory)
: base(SystemTargetGrainId.CreateGrainServiceGrainId(GrainInterfaceUtils.GetGrainClassTypeCode(typeof(IReminderService)), null, silo.SiloAddress), silo, loggerFactory)
{
localReminders = new Dictionary <ReminderIdentity, LocalReminderData>();
this.reminderTable = reminderTable;
this.initTimeout = initTimeout;
this.asyncTimerFactory = asyncTimerFactory;
localTableSequence = 0;
tardinessStat = AverageTimeSpanStatistic.FindOrCreate(StatisticNames.REMINDERS_AVERAGE_TARDINESS_SECONDS);
IntValueStatistic.FindOrCreate(StatisticNames.REMINDERS_NUMBER_ACTIVE_REMINDERS, () => localReminders.Count);
ticksDeliveredStat = CounterStatistic.FindOrCreate(StatisticNames.REMINDERS_COUNTERS_TICKS_DELIVERED);
startedTask = new TaskCompletionSource <bool>(TaskCreationOptions.RunContinuationsAsynchronously);
this.logger = loggerFactory.CreateLogger <LocalReminderService>();
this.listRefreshTimer = asyncTimerFactory.Create(Constants.RefreshReminderList, "ReminderService.ReminderListRefresher");
}
private void SendRequestMessage(GrainReference target, Message message, IResponseCompletionSource context, InvokeMethodOptions options)
{
message.InterfaceType = target.InterfaceType;
message.InterfaceVersion = target.InterfaceVersion;
var targetGrainId = target.GrainId;
var oneWay = (options & InvokeMethodOptions.OneWay) != 0;
message.SendingGrain = CurrentActivationAddress.Grain;
message.SendingActivation = CurrentActivationAddress.Activation;
message.TargetGrain = targetGrainId;
if (SystemTargetGrainId.TryParse(targetGrainId, out var systemTargetGrainId))
{
// If the silo isn't be supplied, it will be filled in by the sender to be the gateway silo
message.TargetSilo = systemTargetGrainId.GetSiloAddress();
message.TargetActivation = ActivationId.GetDeterministic(targetGrainId);
}
if (message.IsExpirableMessage(this.clientMessagingOptions.DropExpiredMessages))
{
// don't set expiration for system target messages.
message.TimeToLive = this.clientMessagingOptions.ResponseTimeout;
}
if (!oneWay)
{
var callbackData = new CallbackData(this.sharedCallbackData, context, message);
callbacks.TryAdd(message.Id, callbackData);
}
else
{
context?.Complete();
}
if (logger.IsEnabled(LogLevel.Trace))
{
logger.Trace("Send {0}", message);
}
MessageCenter.SendMessage(message);
}
private static SystemTargetGrainId CreateId(ILocalSiloDetails localSiloDetails)
{
return(SystemTargetGrainId.Create(Constants.SystemMembershipTableType, SiloAddress.New(localSiloDetails.SiloAddress.Endpoint, 0)));
}