public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { var hasLock = await storage.Session.TryGetGlobalLock(TransportConstants.OutgoingMessageLockId); if (!hasLock) { return; } try { var destinations = await storage.Outgoing.FindAllDestinations(); var count = 0; foreach (var destination in destinations) { var found = await recoverFrom(destination, storage); count += found; } var wasMaxedOut = count >= _settings.RecoveryBatchSize; if (wasMaxedOut) { agent.RescheduleOutgoingRecovery(); } } finally { await storage.Session.ReleaseGlobalLock(TransportConstants.OutgoingMessageLockId); } }
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { if (_workers.QueuedCount > _options.MaximumLocalEnqueuedBackPressureThreshold) { return; } await storage.Session.Begin(); Envelope[] incoming = null; try { var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.IncomingMessageLockId); if (!gotLock) { await storage.Session.Rollback(); return; } incoming = await storage.Incoming.LoadPageOfLocallyOwned(); if (!incoming.Any()) { await storage.Session.Rollback(); return; } await storage.Incoming.Reassign(_options.UniqueNodeId, incoming); await storage.Session.Commit(); } catch (Exception) { await storage.Session.Rollback(); throw; } finally { await storage.Session.ReleaseGlobalLock(TransportConstants.IncomingMessageLockId); } _logger.RecoveredIncoming(incoming); foreach (var envelope in incoming) { envelope.OwnerId = _options.UniqueNodeId; await _workers.Enqueue(envelope); } if (incoming.Length == _options.Retries.RecoveryBatchSize && _workers.QueuedCount < _options.MaximumLocalEnqueuedBackPressureThreshold) { agent.RescheduleIncomingRecovery(); } }
public DurabilityAgent(ITransportLogger logger, ILogger <DurabilityAgent> trace, IWorkerQueue workers, IEnvelopePersistence persistence, ITransportRuntime runtime, AdvancedSettings settings) { if (persistence is NulloEnvelopePersistence) { _disabled = true; return; } Logger = logger; _trace = trace; _workers = workers; _persistence = persistence; _settings = settings; _storage = _persistence.AgentStorage; _worker = new ActionBlock <IMessagingAction>(processAction, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, CancellationToken = _settings.Cancellation }); NodeId = _settings.UniqueNodeId; IncomingMessages = new RecoverIncomingMessages(persistence, workers, settings, logger); OutgoingMessages = new RecoverOutgoingMessages(runtime, settings, logger); NodeReassignment = new NodeReassignment(settings); ScheduledJobs = new RunScheduledJobs(settings, logger); }
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { // TODO -- enforce back pressure here on the retries listener! await storage.Session.Begin(); Envelope[] incoming = null; try { var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.IncomingMessageLockId); if (!gotLock) { await storage.Session.Rollback(); return; } incoming = await storage.Incoming.LoadPageOfLocallyOwned(); if (!incoming.Any()) { await storage.Session.Rollback(); return; } await storage.Incoming.Reassign(_settings.UniqueNodeId, incoming); await storage.Session.Commit(); } catch (Exception) { await storage.Session.Rollback(); throw; } finally { await storage.Session.ReleaseGlobalLock(TransportConstants.IncomingMessageLockId); } _logger.RecoveredIncoming(incoming); foreach (var envelope in incoming) { envelope.OwnerId = _settings.UniqueNodeId; envelope.Callback = new DurableCallback(envelope, _workers, _persistence, _logger); await _workers.Enqueue(envelope); } // TODO -- this should be smart enough later to check for back pressure before rescheduling if (incoming.Length == _settings.RecoveryBatchSize) { agent.RescheduleIncomingRecovery(); } }
public async Task <Envelope[]> ExecuteAtTime(IDurabilityAgentStorage storage, IDurabilityAgent agent, DateTimeOffset utcNow) { var hasLock = await storage.Session.TryGetGlobalLock(TransportConstants.ScheduledJobLockId); if (!hasLock) { return(null); } await storage.Session.Begin(); try { Envelope[] readyToExecute = null; try { // TODO -- this needs to be paged to keep it from being too big readyToExecute = await storage.LoadScheduledToExecute(utcNow); if (!readyToExecute.Any()) { await storage.Session.Rollback(); return(readyToExecute); } await storage.Incoming.Reassign(_options.UniqueNodeId, readyToExecute); await storage.Session.Commit(); } catch (Exception) { await storage.Session.Rollback(); throw; } _logger.ScheduledJobsQueuedForExecution(readyToExecute); foreach (var envelope in readyToExecute) { await agent.EnqueueLocally(envelope); } return(readyToExecute); } finally { await storage.Session.ReleaseGlobalLock(TransportConstants.ScheduledJobLockId); } }
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { try { await _inner.Execute(storage, agent); _completion.SetResult(true); } catch (Exception e) { _completion.SetException(e); } }
protected DatabaseBackedEnvelopePersistence(DatabaseSettings settings, JasperOptions options, IEnvelopeStorageAdmin admin, IDurabilityAgentStorage agentStorage) { this.settings = settings; Admin = admin; AgentStorage = agentStorage; _options = options; _cancellation = options.Cancellation; _incrementIncomingAttempts = $"update {this.settings.SchemaName}.{IncomingTable} set attempts = @attempts where id = @id"; _storeIncoming = $@" insert into {this.settings.SchemaName}.{IncomingTable} (id, status, owner_id, execution_time, attempts, body) values (@id, @status, @owner, @time, @attempts, @body); "; _insertOutgoingSql = $"insert into {this.settings.SchemaName}.{OutgoingTable} (id, owner_id, destination, deliver_by, body) values (@id, @owner, @destination, @deliverBy, @body)"; }
public async Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { await storage.Session.Begin(); var gotLock = await storage.Session.TryGetGlobalLock(TransportConstants.ReassignmentLockId); if (!gotLock) { await storage.Session.Rollback(); return; } try { var owners = await storage.Nodes.FindUniqueOwners(_settings.UniqueNodeId); foreach (var owner in owners.Where(x => x != TransportConstants.AnyNode)) { if (owner == _settings.UniqueNodeId) { continue; } if (await storage.Session.TryGetGlobalTxLock(owner)) { await storage.Nodes.ReassignDormantNodeToAnyNode(owner); } } } catch (Exception) { await storage.Session.Rollback(); throw; } finally { await storage.Session.ReleaseGlobalLock(TransportConstants.ReassignmentLockId); } await storage.Session.Commit(); }
public DurabilityAgent(JasperOptions options, ITransportLogger logger, ILogger <DurabilityAgent> trace, IWorkerQueue workers, IEnvelopePersistence persistence, ISubscriberGraph subscribers) { if (persistence is NulloEnvelopePersistence) { _disabled = true; return; } options.DurabilityAgent = this; _options = options; Logger = logger; _trace = trace; _workers = workers; _persistence = persistence; _storage = _persistence.AgentStorage; _worker = new ActionBlock <IMessagingAction>(processAction, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, CancellationToken = _options.Cancellation }); NodeId = _options.UniqueNodeId; IncomingMessages = new RecoverIncomingMessages(workers, options, logger); OutgoingMessages = new RecoverOutgoingMessages(subscribers, options, logger); NodeReassignment = new NodeReassignment(options, logger); ScheduledJobs = new RunScheduledJobs(options, logger); }
private async Task <int> recoverFrom(Uri destination, IDurabilityAgentStorage storage) { try { Envelope[] filtered = null; Envelope[] outgoing = null; if (_runtime.GetOrBuildSendingAgent(destination).Latched) { return(0); } await storage.Session.Begin(); try { outgoing = await storage.Outgoing.Load(destination : destination); var expiredMessages = outgoing.Where(x => x.IsExpired()).ToArray(); _logger.DiscardedExpired(expiredMessages); await storage.Outgoing.Delete(expiredMessages.ToArray()); filtered = outgoing.Where(x => !expiredMessages.Contains(x)).ToArray(); // Might easily try to do this in the time between starting // and having the data fetched. Was able to make that happen in // (contrived) testing if (_runtime.GetOrBuildSendingAgent(destination).Latched || !filtered.Any()) { await storage.Session.Rollback(); return(0); } await storage.Outgoing.Reassign(_settings.UniqueNodeId, filtered); await storage.Session.Commit(); } catch (Exception) { await storage.Session.Rollback(); throw; } _logger.RecoveredOutgoing(filtered); foreach (var envelope in filtered) { try { await _runtime.GetOrBuildSendingAgent(destination).EnqueueOutgoing(envelope); } catch (Exception e) { _logger.LogException(e, message: $"Unable to enqueue {envelope} for sending"); } } return(outgoing.Count()); } catch (UnknownTransportException e) { _logger.LogException(e, message: $"Could not resolve a channel for {destination}"); await storage.Session.Begin(); await storage.Outgoing.DeleteByDestination(destination); await storage.Session.Commit(); return(0); } }
public Task Execute(IDurabilityAgentStorage storage, IDurabilityAgent agent) { var utcNow = DateTimeOffset.UtcNow; return(ExecuteAtTime(storage, agent, utcNow)); }