public async Task Execute(SqlConnection conn, ISchedulingAgent agent)
{
if (!await conn.TryGetGlobalLock(OutgoingMessageLockId))
{
return;
}
try
{
var destinations = await FindAllOutgoingDestinations(conn);
var count = 0;
foreach (var destination in destinations)
{
var found = await recoverFrom(destination, conn);
count += found;
}
var wasMaxedOut = count >= _settings.Retries.RecoveryBatchSize;
if (wasMaxedOut)
{
agent.RescheduleOutgoingRecovery();
}
}
finally
{
await conn.ReleaseGlobalLock(OutgoingMessageLockId);
}
}
public async Task Execute(IDocumentSession session, ISchedulingAgent agent)
{
var utcNow = DateTimeOffset.UtcNow;
;
await ExecuteAtTime(session, utcNow);
}
public async Task Execute(SqlConnection conn, ISchedulingAgent agent)
{
var utcNow = DateTimeOffset.UtcNow;
;
await ExecuteAtTime(conn, utcNow);
}
public RecoverIncomingMessages(IWorkerQueue workers, BusSettings settings, EnvelopeTables marker,
ISchedulingAgent schedulingAgent, CompositeTransportLogger logger)
{
_workers = workers;
_settings = settings;
_marker = marker;
_schedulingAgent = schedulingAgent;
_logger = logger;
_findAtLargeEnvelopesSql = $"select body from {marker.Incoming} where owner_id = {TransportConstants.AnyNode} and status = '{TransportConstants.Incoming}' limit {settings.Retries.RecoveryBatchSize}";
}
public async Task Execute(IDocumentSession session, ISchedulingAgent agent)
{
if (!await session.TryGetGlobalTxLock(ReassignmentLockId))
{
return;
}
session.QueueOperation(new SqlCommandOperation(_reassignDormantNodeIncomingSql));
session.QueueOperation(new SqlCommandOperation(_reassignDormantNodeOutgoingSql));
await session.SaveChangesAsync();
}
public RecoverOutgoingMessages(IChannelGraph channels, BusSettings settings, EnvelopeTables marker, ISchedulingAgent schedulingAgent, CompositeTransportLogger logger)
{
_channels = channels;
_settings = settings;
_marker = marker;
_schedulingAgent = schedulingAgent;
_logger = logger;
_findUniqueDestinations = $"select distinct destination from {_marker.Outgoing}";
_findOutgoingEnvelopesSql = $"select body from {marker.Outgoing} where owner_id = {TransportConstants.AnyNode} and destination = :destination limit {settings.Retries.RecoveryBatchSize}";
_deleteOutgoingSql = $"delete from {marker.Outgoing} where owner_id = :owner and destination = :destination";
}
public async Task Execute(SqlConnection conn, ISchedulingAgent agent)
{
if (_workers.QueuedCount > _settings.MaximumLocalEnqueuedBackPressureThreshold)
{
return;
}
var tx = conn.BeginTransaction();
List <Envelope> incoming = null;
try
{
if (!await conn.TryGetGlobalTxLock(tx, IncomingMessageLockId))
{
tx.Rollback();
return;
}
incoming = await conn.CreateCommand(tx, _findAtLargeEnvelopesSql)
.ExecuteToEnvelopes();
if (!incoming.Any())
{
tx.Rollback();
return;
}
await markOwnership(conn, tx, incoming);
tx.Commit();
}
catch (Exception)
{
tx.Rollback();
throw;
}
_logger.RecoveredIncoming(incoming);
foreach (var envelope in incoming)
{
envelope.OwnerId = _settings.UniqueNodeId;
await _workers.Enqueue(envelope);
}
if (incoming.Count == _settings.Retries.RecoveryBatchSize &&
_workers.QueuedCount < _settings.MaximumLocalEnqueuedBackPressureThreshold)
{
agent.RescheduleIncomingRecovery();
}
}
public async Task Execute(IDocumentSession session, ISchedulingAgent agent)
{
if (!await session.TryGetGlobalTxLock(ReassignmentLockId))
{
return;
}
await session.Connection.CreateCommand()
.Sql(_reassignDormantNodeSql).ExecuteNonQueryAsync();
await session.SaveChangesAsync();
}
public async Task Execute(IDocumentSession session, ISchedulingAgent agent)
{
if (_workers.QueuedCount > _settings.MaximumLocalEnqueuedBackPressureThreshold)
{
return;
}
if (!await session.TryGetGlobalTxLock(IncomingMessageLockId))
{
return;
}
var incoming = await session.Connection.CreateCommand(_findAtLargeEnvelopesSql)
.ExecuteToEnvelopes();
if (!incoming.Any())
{
return;
}
session.MarkOwnership(_marker.Incoming, _settings.UniqueNodeId, incoming);
await session.SaveChangesAsync();
_logger.RecoveredIncoming(incoming);
foreach (var envelope in incoming)
{
envelope.OwnerId = _settings.UniqueNodeId;
await _workers.Enqueue(envelope);
}
if (incoming.Count == _settings.Retries.RecoveryBatchSize &&
_workers.QueuedCount < _settings.MaximumLocalEnqueuedBackPressureThreshold)
{
agent.RescheduleIncomingRecovery();
}
}
public async Task Execute(SqlConnection conn, ISchedulingAgent agent)
{
var tx = conn.BeginTransaction();
if (!await conn.TryGetGlobalTxLock(tx, ReassignmentLockId))
{
tx.Rollback();
return;
}
try
{
var owners = await fetchOwners(tx);
foreach (var owner in owners.Distinct())
{
if (owner == _settings.UniqueNodeId)
{
continue;
}
if (await conn.TryGetGlobalTxLock(tx, owner))
{
await conn.CreateCommand(tx, _reassignDormantNodeSql)
.With("owner", owner, SqlDbType.Int)
.ExecuteNonQueryAsync();
}
}
}
catch (Exception)
{
tx.Rollback();
throw;
}
tx.Commit();
}
public async Task Execute(IDocumentSession session, ISchedulingAgent agent)
{
if (!await session.TryGetGlobalTxLock(OutgoingMessageLockId))
{
return;
}
var destinations = await FindAllOutgoingDestinations(session);
var count = 0;
foreach (var destination in destinations)
{
count += await recoverFrom(destination, session);
}
var wasMaxedOut = count >= _settings.Retries.RecoveryBatchSize;
if (wasMaxedOut)
{
agent.RescheduleOutgoingRecovery();
}
}
