async Task <ReceivedStatus> IReceiverCallback.Received(Uri uri, Envelope[] messages)
{
try
{
var now = DateTime.UtcNow;
foreach (var message in messages)
{
message.ReceivedAt = uri;
message.Callback = new LightweightCallback(_workerQueue);
if (message.IsDelayed(now))
{
_workerQueue.ScheduledJobs.Enqueue(message.ExecutionTime.Value, message);
}
else
{
await _workerQueue.Enqueue(message);
}
}
_logger.IncomingBatchReceived(messages);
return(ReceivedStatus.Successful);
}
catch (Exception e)
{
_logger.LogException(e);
return(ReceivedStatus.ProcessFailure);
}
}
public Task SerializationFailure(OutgoingMessageBatch outgoing)
{
_logger.OutgoingBatchFailed(outgoing);
// Can't really happen now, but what the heck.
_logger.LogException(new Exception("Serialization failure with outgoing envelopes " + outgoing.Messages.Select(x => x.ToString()).Join(", ")));
return(Task.CompletedTask);
}
private async Task <int> recoverFrom(Uri destination, IDocumentSession session)
{
try
{
var channel = _channels.GetOrBuildChannel(destination);
if (channel.Latched)
{
return(0);
}
var outgoing = await session.Connection.CreateCommand(_findOutgoingEnvelopesSql)
.With("destination", destination.ToString(), NpgsqlDbType.Varchar)
.ExecuteToEnvelopes();
var filtered = filterExpired(session, outgoing);
// Might easily try to do this in the time between starting
// and having the data fetched. Was able to make that happen in
// (contrived) testing
if (channel.Latched || !filtered.Any())
{
return(0);
}
session.MarkOwnership(_marker.Outgoing, _marker.CurrentNodeId, filtered);
await session.SaveChangesAsync();
_logger.RecoveredOutgoing(filtered);
foreach (var envelope in filtered)
{
try
{
await channel.QuickSend(envelope);
}
catch (Exception e)
{
_logger.LogException(e, message: $"Unable to enqueue {envelope} for sending");
}
}
return(outgoing.Count());
}
catch (UnknownTransportException e)
{
_logger.LogException(e, message: $"Could not resolve a channel for {destination}");
await DeleteFromOutgoingEnvelopes(session, TransportConstants.AnyNode, destination);
await session.SaveChangesAsync();
return(0);
}
}
// Separated for testing here.
public async Task <ReceivedStatus> ProcessReceivedMessages(DateTime now, Uri uri, Envelope[] messages)
{
try
{
foreach (var message in messages)
{
message.ReceivedAt = uri;
if (message.IsDelayed(now))
{
message.Status = TransportConstants.Scheduled;
message.OwnerId = TransportConstants.AnyNode;
}
else
{
message.Status = TransportConstants.Scheduled;
message.OwnerId = _settings.UniqueNodeId;
}
message.Status = message.IsDelayed(now)
? TransportConstants.Scheduled
: TransportConstants.Incoming;
}
using (var session = _store.LightweightSession())
{
session.StoreIncoming(_marker, messages);
await session.SaveChangesAsync();
}
foreach (var message in messages.Where(x => x.Status == TransportConstants.Incoming))
{
message.Callback = new MartenCallback(message, _queues, _store, _marker, _retries, _logger);
await _queues.Enqueue(message);
}
_logger.IncomingBatchReceived(messages);
return(ReceivedStatus.Successful);
}
catch (Exception e)
{
_logger.LogException(e);
return(ReceivedStatus.ProcessFailure);
}
}
private async Task processAction(IMessagingAction action)
{
await tryRestartConnection();
if (_connection == null)
{
return;
}
var tx = _connection.BeginTransaction();
var session = _store.OpenSession(new SessionOptions
{
Connection = _connection,
Transaction = tx,
Tracking = DocumentTracking.None
});
try
{
await action.Execute(session);
}
catch (Exception e)
{
_logger.LogException(e);
}
finally
{
if (!tx.IsCompleted)
{
await tx.RollbackAsync();
}
session.Dispose();
}
await tryRestartConnection();
}
public override async Task EnqueueForRetry(OutgoingMessageBatch batch)
{
var expiredInQueue = Queued.Where(x => x.IsExpired()).ToArray();
var expiredInBatch = batch.Messages.Where(x => x.IsExpired()).ToArray();
try
{
using (var session = _store.LightweightSession())
{
var expired = expiredInBatch.Concat(expiredInQueue).ToArray();
session.DeleteEnvelopes(_marker.Incoming, expired);
var all = Queued.Where(x => !expiredInQueue.Contains(x))
.Concat(batch.Messages.Where(x => !expiredInBatch.Contains(x)))
.ToList();
if (all.Count > _settings.MaximumEnvelopeRetryStorage)
{
var reassigned = all.Skip(_settings.MaximumEnvelopeRetryStorage).ToArray();
session.MarkOwnership(_marker.Incoming, TransportConstants.AnyNode, reassigned);
}
await session.SaveChangesAsync();
_logger.DiscardedExpired(expired);
Queued = all.Take(_settings.MaximumEnvelopeRetryStorage).ToList();
}
}
catch (Exception e)
{
_logger.LogException(e, message: "Failed while trying to enqueue a message batch for retries");
#pragma warning disable 4014
Task.Delay(100).ContinueWith(async _ => await EnqueueForRetry(batch));
#pragma warning restore 4014
}
}
public void Start(ISenderCallback callback)
{
_callback = callback;
_sender = new ActionBlock <OutgoingMessageBatch>(SendBatch, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation
});
_serializing = new ActionBlock <Envelope>(e =>
{
try
{
e.EnsureData();
_batching.Post(e);
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error while trying to serialize envelope {e}");
}
},
new ExecutionDataflowBlockOptions
{
CancellationToken = _cancellation
});
_batchWriting = new TransformBlock <Envelope[], OutgoingMessageBatch>(
envelopes =>
{
var batch = new OutgoingMessageBatch(Destination, envelopes);
_queued += batch.Messages.Count;
return(batch);
});
_batchWriting.LinkTo(_sender);
_batching = new BatchingBlock <Envelope>(200, _batchWriting, _cancellation);
}
public override async Task Successful(OutgoingMessageBatch outgoing)
{
try
{
using (var conn = _store.Tenancy.Default.CreateConnection())
{
await conn.OpenAsync(_cancellation);
await conn.CreateCommand($"delete from {_marker.Outgoing} where id = ANY(:idlist)")
.With("idlist", outgoing.Messages.Select(x => x.Id).ToArray(),
NpgsqlDbType.Array | NpgsqlDbType.Uuid)
.ExecuteNonQueryAsync(_cancellation);
}
}
catch (Exception e)
{
_logger.LogException(e, message: "Error trying to delete outgoing envelopes after a successful batch send");
foreach (var envelope in outgoing.Messages)
{
_retries.DeleteOutgoing(envelope);
}
}
}
