public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
var hasLock = await storage.Session.TryGetGlobalLock(TransportConstants.OutgoingMessageLockId);
if (!hasLock)
{
return;
}
try
{
var destinations = await storage.Outgoing.FindAllDestinations();
var count = 0;
foreach (var destination in destinations)
{
var found = await recoverFrom(destination, storage);
count += found;
}
var wasMaxedOut = count >= _settings.RecoveryBatchSize;
if (wasMaxedOut)
{
agent.RescheduleOutgoingRecovery();
}
}
finally
{
await storage.Session.ReleaseGlobalLock(TransportConstants.OutgoingMessageLockId);
}
}
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
if (_workers.QueuedCount > _options.MaximumLocalEnqueuedBackPressureThreshold)
{
return;
}
await storage.Session.Begin();
Envelope[] incoming = null;
try
{
var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.IncomingMessageLockId);
if (!gotLock)
{
await storage.Session.Rollback();
return;
}
incoming = await storage.Incoming.LoadPageOfLocallyOwned();
if (!incoming.Any())
{
await storage.Session.Rollback();
return;
}
await storage.Incoming.Reassign(_options.UniqueNodeId, incoming);
await storage.Session.Commit();
}
catch (Exception)
{
await storage.Session.Rollback();
throw;
}
finally
{
await storage.Session.ReleaseGlobalLock(TransportConstants.IncomingMessageLockId);
}
_logger.RecoveredIncoming(incoming);
foreach (var envelope in incoming)
{
envelope.OwnerId = _options.UniqueNodeId;
await _workers.Enqueue(envelope);
}
if (incoming.Length == _options.Retries.RecoveryBatchSize &&
_workers.QueuedCount < _options.MaximumLocalEnqueuedBackPressureThreshold)
{
agent.RescheduleIncomingRecovery();
}
}
public DurabilityAgent(ITransportLogger logger,
ILogger <DurabilityAgent> trace,
IWorkerQueue workers,
IEnvelopePersistence persistence,
ITransportRuntime runtime,
AdvancedSettings settings)
{
if (persistence is NulloEnvelopePersistence)
{
_disabled = true;
return;
}
Logger = logger;
_trace = trace;
_workers = workers;
_persistence = persistence;
_settings = settings;
_storage = _persistence.AgentStorage;
_worker = new ActionBlock <IMessagingAction>(processAction, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
CancellationToken = _settings.Cancellation
});
NodeId = _settings.UniqueNodeId;
IncomingMessages = new RecoverIncomingMessages(persistence, workers, settings, logger);
OutgoingMessages = new RecoverOutgoingMessages(runtime, settings, logger);
NodeReassignment = new NodeReassignment(settings);
ScheduledJobs = new RunScheduledJobs(settings, logger);
}
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
// TODO -- enforce back pressure here on the retries listener!
await storage.Session.Begin();
Envelope[] incoming = null;
try
{
var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.IncomingMessageLockId);
if (!gotLock)
{
await storage.Session.Rollback();
return;
}
incoming = await storage.Incoming.LoadPageOfLocallyOwned();
if (!incoming.Any())
{
await storage.Session.Rollback();
return;
}
await storage.Incoming.Reassign(_settings.UniqueNodeId, incoming);
await storage.Session.Commit();
}
catch (Exception)
{
await storage.Session.Rollback();
throw;
}
finally
{
await storage.Session.ReleaseGlobalLock(TransportConstants.IncomingMessageLockId);
}
_logger.RecoveredIncoming(incoming);
foreach (var envelope in incoming)
{
envelope.OwnerId = _settings.UniqueNodeId;
envelope.Callback = new DurableCallback(envelope, _workers, _persistence, _logger);
await _workers.Enqueue(envelope);
}
// TODO -- this should be smart enough later to check for back pressure before rescheduling
if (incoming.Length == _settings.RecoveryBatchSize)
{
agent.RescheduleIncomingRecovery();
}
}
public async Task <Envelope[]> ExecuteAtTime(IDurabilityAgentStorage storage, IDurabilityAgent agent, DateTimeOffset utcNow)
{
var hasLock = await storage.Session.TryGetGlobalLock(TransportConstants.ScheduledJobLockId);
if (!hasLock)
{
return(null);
}
await storage.Session.Begin();
try
{
Envelope[] readyToExecute = null;
try
{
// TODO -- this needs to be paged to keep it from being too big
readyToExecute = await storage.LoadScheduledToExecute(utcNow);
if (!readyToExecute.Any())
{
await storage.Session.Rollback();
return(readyToExecute);
}
await storage.Incoming.Reassign(_options.UniqueNodeId, readyToExecute);
await storage.Session.Commit();
}
catch (Exception)
{
await storage.Session.Rollback();
throw;
}
_logger.ScheduledJobsQueuedForExecution(readyToExecute);
foreach (var envelope in readyToExecute)
{
await agent.EnqueueLocally(envelope);
}
return(readyToExecute);
}
finally
{
await storage.Session.ReleaseGlobalLock(TransportConstants.ScheduledJobLockId);
}
}
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
try
{
await _inner.Execute(storage, agent);
_completion.SetResult(true);
}
catch (Exception e)
{
_completion.SetException(e);
}
}
protected DatabaseBackedEnvelopePersistence(DatabaseSettings settings, JasperOptions options, IEnvelopeStorageAdmin admin, IDurabilityAgentStorage agentStorage)
{
this.settings = settings;
Admin = admin;
AgentStorage = agentStorage;
_options = options;
_cancellation = options.Cancellation;
_incrementIncomingAttempts = $"update {this.settings.SchemaName}.{IncomingTable} set attempts = @attempts where id = @id";
_storeIncoming = $@"
insert into {this.settings.SchemaName}.{IncomingTable}
(id, status, owner_id, execution_time, attempts, body)
values
(@id, @status, @owner, @time, @attempts, @body);
";
_insertOutgoingSql = $"insert into {this.settings.SchemaName}.{OutgoingTable} (id, owner_id, destination, deliver_by, body) values (@id, @owner, @destination, @deliverBy, @body)";
}
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
await storage.Session.Begin();
var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.ReassignmentLockId);
if (!gotLock)
{
await storage.Session.Rollback();
return;
}
try
{
var owners = await storage.Nodes.FindUniqueOwners(_settings.UniqueNodeId);
foreach (var owner in owners.Where(x => x != TransportConstants.AnyNode))
{
if (owner == _settings.UniqueNodeId)
{
continue;
}
if (await storage.Session.TryGetGlobalTxLock(owner))
{
await storage.Nodes.ReassignDormantNodeToAnyNode(owner);
}
}
}
catch (Exception)
{
await storage.Session.Rollback();
throw;
}
finally
{
await storage.Session.ReleaseGlobalLock(TransportConstants.ReassignmentLockId);
}
await storage.Session.Commit();
}
public DurabilityAgent(JasperOptions options,
ITransportLogger logger,
ILogger <DurabilityAgent> trace,
IWorkerQueue workers,
IEnvelopePersistence persistence,
ISubscriberGraph subscribers)
{
if (persistence is NulloEnvelopePersistence)
{
_disabled = true;
return;
}
options.DurabilityAgent = this;
_options = options;
Logger = logger;
_trace = trace;
_workers = workers;
_persistence = persistence;
_storage = _persistence.AgentStorage;
_worker = new ActionBlock <IMessagingAction>(processAction, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
CancellationToken = _options.Cancellation
});
NodeId = _options.UniqueNodeId;
IncomingMessages = new RecoverIncomingMessages(workers, options, logger);
OutgoingMessages = new RecoverOutgoingMessages(subscribers, options, logger);
NodeReassignment = new NodeReassignment(options, logger);
ScheduledJobs = new RunScheduledJobs(options, logger);
}
private async Task <int> recoverFrom(Uri destination, IDurabilityAgentStorage storage)
{
try
{
Envelope[] filtered = null;
Envelope[] outgoing = null;
if (_runtime.GetOrBuildSendingAgent(destination).Latched)
{
return(0);
}
await storage.Session.Begin();
try
{
outgoing = await storage.Outgoing.Load(destination : destination);
var expiredMessages = outgoing.Where(x => x.IsExpired()).ToArray();
_logger.DiscardedExpired(expiredMessages);
await storage.Outgoing.Delete(expiredMessages.ToArray());
filtered = outgoing.Where(x => !expiredMessages.Contains(x)).ToArray();
// Might easily try to do this in the time between starting
// and having the data fetched. Was able to make that happen in
// (contrived) testing
if (_runtime.GetOrBuildSendingAgent(destination).Latched || !filtered.Any())
{
await storage.Session.Rollback();
return(0);
}
await storage.Outgoing.Reassign(_settings.UniqueNodeId, filtered);
await storage.Session.Commit();
}
catch (Exception)
{
await storage.Session.Rollback();
throw;
}
_logger.RecoveredOutgoing(filtered);
foreach (var envelope in filtered)
{
try
{
await _runtime.GetOrBuildSendingAgent(destination).EnqueueOutgoing(envelope);
}
catch (Exception e)
{
_logger.LogException(e, message: $"Unable to enqueue {envelope} for sending");
}
}
return(outgoing.Count());
}
catch (UnknownTransportException e)
{
_logger.LogException(e, message: $"Could not resolve a channel for {destination}");
await storage.Session.Begin();
await storage.Outgoing.DeleteByDestination(destination);
await storage.Session.Commit();
return(0);
}
}
public Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent)
{
var utcNow = DateTimeOffset.UtcNow;
return(ExecuteAtTime(storage, agent, utcNow));
}
