/// <summary>
/// Take responsibility for a new queues that was assigned to me via a new range.
/// We first store the new queue in our internal data structure, try to initialize it and start a pumping timer.
/// ERROR HANDLING:
/// The resposibility to handle initializatoion and shutdown failures is inside the INewQueueAdapterReceiver code.
/// The agent will call Initialize once and log an error. It will not call initiliaze again.
/// The receiver itself may attempt later to recover from this error and do initialization again.
/// The agent will assume initialization has succeeded and will subsequently start calling pumping receive.
/// Same applies to shutdown.
/// </summary>
/// <param name="qAdapter"></param>
/// <param name="queueAdapterCache"></param>
/// <param name="failureHandler"></param>
/// <returns></returns>
public async Task Initialize(Immutable <IQueueAdapter> qAdapter, Immutable <IQueueAdapterCache> queueAdapterCache, Immutable <IStreamFailureHandler> failureHandler)
{
if (qAdapter.Value == null)
{
throw new ArgumentNullException("qAdapter", "Init: queueAdapter should not be null");
}
if (failureHandler.Value == null)
{
throw new ArgumentNullException("failureHandler", "Init: streamDeliveryFailureHandler should not be null");
}
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter.Value;
streamFailureHandler = failureHandler.Value;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_02, String.Format("Exception while calling IQueueAdapter.CreateNewReceiver."), exc);
return;
}
try
{
if (queueAdapterCache.Value != null)
{
queueCache = queueAdapterCache.Value.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingAgent_23, String.Format("Exception while calling IQueueAdapterCache.CreateQueueCache."), exc);
return;
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, String.Format("QueueAdapterReceiver {0} failed to Initialize.", QueueId.ToStringWithHashCode()));
await task;
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(queueGetPeriod);
timer = providerRuntime.RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, queueGetPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private async Task RemoveQueues(List <QueueId> queuesToRemove)
{
if (queuesToRemove.Count == 0)
{
return;
}
// Stop the agents that for queues that are not in my range anymore.
var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count);
Log(ErrorCode.PersistentStreamPullingManager_10, "About to remove {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToString()));
var removeTasks = new List <Task>();
foreach (var queueId in queuesToRemove)
{
PersistentStreamPullingAgent agent;
if (!queuesToAgentsMap.TryGetValue(queueId, out agent))
{
continue;
}
agents.Add(agent);
queuesToAgentsMap.Remove(queueId);
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown);
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11,
String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId));
removeTasks.Add(task);
}
try
{
await Task.WhenAll(removeTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
foreach (var agent in agents)
{
try
{
providerRuntime.UnRegisterSystemTarget(agent);
}
catch (Exception exc)
{
Log(ErrorCode.PersistentStreamPullingManager_12,
"Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", ((ISystemTargetBase)agent).GrainId, exc.Message);
}
}
if (agents.Count > 0)
{
Log(ErrorCode.PersistentStreamPullingManager_10, "Removed {0} queues: {1}. Now own total of {2} queues: {3}",
agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToString()),
NumberRunningAgents,
PrintQueues(queuesToAgentsMap.Keys));
}
}
public async Task LocalhostSiloTest()
{
var silo = new SiloHostBuilder()
.AddMemoryGrainStorage("MemoryStore")
.UseLocalhostClustering()
.Build();
var client = new ClientBuilder()
.UseLocalhostClustering()
.Build();
try
{
await silo.StartAsync();
await client.Connect();
var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid());
var result = await grain.Echo("test");
Assert.Equal("test", result);
}
finally
{
await OrleansTaskExtentions.SafeExecute(() => silo.StopAsync());
await OrleansTaskExtentions.SafeExecute(() => client.Close());
Utils.SafeExecute(() => silo.Dispose());
Utils.SafeExecute(() => client.Close());
}
}
public async Task LocalhostSiloTest()
{
using var portAllocator = new TestClusterPortAllocator();
var(siloPort, gatewayPort) = portAllocator.AllocateConsecutivePortPairs(1);
var host = new HostBuilder().UseOrleans(siloBuilder =>
{
siloBuilder.AddMemoryGrainStorage("MemoryStore")
.UseLocalhostClustering(siloPort, gatewayPort);
}).Build();
var client = new ClientBuilder()
.UseLocalhostClustering(gatewayPort)
.Build();
try
{
await host.StartAsync();
await client.Connect();
var grain = client.GetGrain <IEchoGrain>(Guid.NewGuid());
var result = await grain.Echo("test");
Assert.Equal("test", result);
}
finally
{
await OrleansTaskExtentions.SafeExecute(() => host.StopAsync());
await OrleansTaskExtentions.SafeExecute(() => client.Close());
Utils.SafeExecute(() => host.Dispose());
Utils.SafeExecute(() => client.Close());
}
}
private void InitializeInternal()
{
logger.LogInformation(
(int)ErrorCode.PersistentStreamPullingAgent_02,
"Init of {Name} {Id} on silo {Silo} for queue {Queue}.",
GetType().Name,
((ISystemTargetBase)this).GrainId.ToString(),
Silo,
QueueId.ToStringWithHashCode());
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.LogError((int)ErrorCode.PersistentStreamPullingAgent_23, exc, "Exception while calling IQueueAdapterCache.CreateQueueCache.");
throw;
}
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.LogError((int)ErrorCode.PersistentStreamPullingAgent_02, exc, "Exception while calling IQueueAdapter.CreateNewReceiver.");
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(this.options.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = ThreadSafeRandom.NextTimeSpan(this.options.GetQueueMsgsTimerPeriod);
timer = RegisterGrainTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod);
IntValueStatistic.FindOrCreate(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix), () => pubSubCache.Count);
logger.LogInformation((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {Queue} under my responsibility.", QueueId.ToStringWithHashCode());
}
private async Task InitAgent(PersistentStreamPullingAgent agent)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
IStreamFailureHandler deliveryFailureHandler = await adapterFactory.GetDeliveryFailureHandler(agent.QueueId);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable, deliveryFailureHandler.AsImmutable()));
await task.LogException(logger, ErrorCode.PersistentStreamPullingManager_09, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
}
/// <summary>
/// Take responsibility for a set of new queues that were assigned to me via a new range.
/// We first create one pulling agent for every new queue and store them in our internal data structure, then try to initialize the agents.
/// ERROR HANDLING:
/// The responsibility to handle initialization and shutdown failures is inside the Agents code.
/// The manager will call Initialize once and log an error. It will not call initialize again and will assume initialization has succeeded.
/// Same applies to shutdown.
/// </summary>
/// <param name="myQueues"></param>
/// <param name="failOnInit"></param>
/// <returns></returns>
private async Task AddNewQueues(IEnumerable <QueueId> myQueues, bool failOnInit)
{
// Create agents for queues in range that we don't yet have.
// First create them and store in local queuesToAgentsMap.
// Only after that Initialize them all.
var agents = new List <PersistentStreamPullingAgent>();
foreach (var queueId in myQueues.Where(queueId => !queuesToAgentsMap.ContainsKey(queueId)))
{
try
{
var agentId = GrainId.NewSystemTargetGrainIdByTypeCode(Constants.PULLING_AGENT_SYSTEM_TARGET_TYPE_CODE);
var agent = new PersistentStreamPullingAgent(agentId, streamProviderName, providerRuntime, queueId, queueGetPeriod, initQueueTimeout);
providerRuntime.RegisterSystemTarget(agent);
queuesToAgentsMap.Add(queueId, agent);
agents.Add(agent);
}
catch (Exception exc)
{
logger.Error((int)ErrorCode.PersistentStreamPullingManager_07, String.Format("Exception while creating PersistentStreamPullingAgent."), exc);
// What should we do? This error is not recoverable and considered a bug. But we don't want to bring the silo down.
// If this is when silo is starting and agent is initializing, fail the silo startup. Otherwise, just swallow to limit impact on other receivers.
if (failOnInit)
{
throw;
}
}
}
try
{
var initTasks = new List <Task>();
foreach (var agent in agents)
{
// Init the agent only after it was registered locally.
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var queueAdapterCacheAsImmutable = queueAdapterCache != null?queueAdapterCache.AsImmutable() : new Immutable <IQueueAdapterCache>(null);
// Need to call it as a grain reference.
var task = OrleansTaskExtentions.SafeExecute(() => agentGrainRef.Initialize(queueAdapter.AsImmutable(), queueAdapterCacheAsImmutable));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_08, String.Format("PersistentStreamPullingAgent {0} failed to Initialize.", agent.QueueId));
initTasks.Add(task);
}
await Task.WhenAll(initTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
logger.Info((int)ErrorCode.PersistentStreamPullingManager_09, "Took {0} new queues under my responsibility: {1}", agents.Count,
Utils.EnumerableToString(agents, agent => agent.QueueId.ToStringWithHashCode()));
}
private void InitializeInternal(IQueueAdapter qAdapter, IQueueAdapterCache queueAdapterCache, IStreamFailureHandler failureHandler)
{
logger.Info(ErrorCode.PersistentStreamPullingAgent_02, "Init of {0} {1} on silo {2} for queue {3}.",
GetType().Name, ((ISystemTargetBase)this).GrainId.ToDetailedString(), Silo, QueueId.ToStringWithHashCode());
// Remove cast once we cleanup
queueAdapter = qAdapter;
streamFailureHandler = failureHandler;
lastTimeCleanedPubSubCache = DateTime.UtcNow;
try
{
receiver = queueAdapter.CreateReceiver(QueueId);
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_02, "Exception while calling IQueueAdapter.CreateNewReceiver.", exc);
throw;
}
try
{
if (queueAdapterCache != null)
{
queueCache = queueAdapterCache.CreateQueueCache(QueueId);
}
}
catch (Exception exc)
{
logger.Error(ErrorCode.PersistentStreamPullingAgent_23, "Exception while calling IQueueAdapterCache.CreateQueueCache.", exc);
throw;
}
try
{
receiverInitTask = OrleansTaskExtentions.SafeExecute(() => receiver.Initialize(config.InitQueueTimeout))
.LogException(logger, ErrorCode.PersistentStreamPullingAgent_03, $"QueueAdapterReceiver {QueueId.ToStringWithHashCode()} failed to Initialize.");
receiverInitTask.Ignore();
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Initialize. No need to log again.
}
// Setup a reader for a new receiver.
// Even if the receiver failed to initialise, treat it as OK and start pumping it. It's receiver responsibility to retry initialization.
var randomTimerOffset = safeRandom.NextTimeSpan(config.GetQueueMsgsTimerPeriod);
timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, config.GetQueueMsgsTimerPeriod);
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_04, "Taking queue {0} under my responsibility.", QueueId.ToStringWithHashCode());
}
private async Task RemoveQueues(IEnumerable <QueueId> myQueues)
{
// Stop the agents that for queues that are not in my range anymore.
List <QueueId> queuesToRemove = queuesToAgentsMap.Keys.Where(queueId => !myQueues.Contains(queueId)).ToList();
var agents = new List <PersistentStreamPullingAgent>(queuesToRemove.Count);
logger.Info((int)ErrorCode.PersistentStreamPullingManager_10, "Removing {0} agents from my responsibility: {1}", queuesToRemove.Count, Utils.EnumerableToString(queuesToRemove, q => q.ToStringWithHashCode()));
var removeTasks = new List <Task>();
foreach (var queueId in queuesToRemove)
{
PersistentStreamPullingAgent agent;
if (!queuesToAgentsMap.TryGetValue(queueId, out agent))
{
continue;
}
agents.Add(agent);
queuesToAgentsMap.Remove(queueId);
var agentGrainRef = agent.AsReference <IPersistentStreamPullingAgent>();
var task = OrleansTaskExtentions.SafeExecute(agentGrainRef.Shutdown);
task = task.LogException(logger, ErrorCode.PersistentStreamPullingManager_11,
String.Format("PersistentStreamPullingAgent {0} failed to Shutdown.", agent.QueueId));
removeTasks.Add(task);
}
try
{
await Task.WhenAll(removeTasks);
}
catch
{
// Just ignore this exception and proceed as if Initialize has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
foreach (var agent in agents)
{
try
{
providerRuntime.UnRegisterSystemTarget(agent);
}
catch (Exception exc)
{
logger.Info((int)ErrorCode.PersistentStreamPullingManager_12,
"Exception while UnRegisterSystemTarget of PersistentStreamPullingAgent {0}. Ignoring. Exc.Message = {1}.", agent.GrainId, exc.Message);
}
}
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
Utils.SafeExecute(tmp.Dispose);
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(config.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId));
await task;
}
catch
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
foreach (var tuple in pubSubCache)
{
tuple.Value.DisposeAll(logger);
var streamId = tuple.Key;
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer));
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc);
}
pubSubCache.Clear();
IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix));
//IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix));
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", this.GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
tmp.Dispose();
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = StreamProducerExtensionFactory.Cast(this.AsReference());
foreach (var streamId in pubSubCache.Keys)
{
logger.Info((int)ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, streamProviderName, meAsStreamProducer));
}
try
{
var task = OrleansTaskExtentions.SafeExecute(() => receiver.Shutdown(initQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
String.Format("QueueAdapterReceiver {0} failed to Shutdown.", QueueId));
await task;
}
catch (Exception)
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn((int)ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams taht used to be in my responsibility.", exc);
}
}
public async Task Shutdown()
{
// Stop pulling from queues that are not in my range anymore.
logger.Info(ErrorCode.PersistentStreamPullingAgent_05, "Shutdown of {0} responsible for queue: {1}", GetType().Name, QueueId.ToStringWithHashCode());
if (timer != null)
{
var tmp = timer;
timer = null;
Utils.SafeExecute(tmp.Dispose, this.logger);
try
{
await tmp.GetCurrentlyExecutingTickTask().WithTimeout(TimeSpan.FromSeconds(5));
}
catch (Exception ex)
{
this.logger.LogWarning(ex, "Waiting for the last timer tick failed");
}
}
this.queueCache = null;
Task localReceiverInitTask = receiverInitTask;
if (localReceiverInitTask != null)
{
try
{
await localReceiverInitTask;
receiverInitTask = null;
}
catch (Exception)
{
receiverInitTask = null;
// squelch
}
}
try
{
IQueueAdapterReceiver localReceiver = this.receiver;
this.receiver = null;
if (localReceiver != null)
{
var task = OrleansTaskExtentions.SafeExecute(() => localReceiver.Shutdown(this.options.InitQueueTimeout));
task = task.LogException(logger, ErrorCode.PersistentStreamPullingAgent_07,
$"QueueAdapterReceiver {QueueId} failed to Shutdown.");
await task;
}
}
catch
{
// Just ignore this exception and proceed as if Shutdown has succeeded.
// We already logged individual exceptions for individual calls to Shutdown. No need to log again.
}
var unregisterTasks = new List <Task>();
var meAsStreamProducer = this.AsReference <IStreamProducerExtension>();
foreach (var tuple in pubSubCache)
{
tuple.Value.DisposeAll(logger);
var streamId = tuple.Key;
logger.Info(ErrorCode.PersistentStreamPullingAgent_06, "Unregister PersistentStreamPullingAgent Producer for stream {0}.", streamId);
unregisterTasks.Add(pubSub.UnregisterProducer(streamId, meAsStreamProducer));
}
try
{
await Task.WhenAll(unregisterTasks);
}
catch (Exception exc)
{
logger.Warn(ErrorCode.PersistentStreamPullingAgent_08,
"Failed to unregister myself as stream producer to some streams that used to be in my responsibility.", exc);
}
pubSubCache.Clear();
IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE, StatisticUniquePostfix));
//IntValueStatistic.Delete(new StatisticName(StatisticNames.STREAMS_PERSISTENT_STREAM_QUEUE_CACHE_SIZE, StatisticUniquePostfix));
}
