private async Task ConsumeAsync(IListeningWorkerQueue callback)
{
await foreach (Message message in _consumer.Messages(_cancellation))
{
Envelope envelope;
try
{
envelope = _protocol.ReadEnvelope(new DotPulsarMessage(message.Data, message.Properties));
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error trying to map an incoming Pulsar {_endpoint.Topic} Topic message to an Envelope. See the Dead Letter Queue");
continue;
}
try
{
await callback.Received(Address, envelope);
await _consumer.Acknowledge(message, _cancellation);
}
catch (Exception e)
{
// DotPulsar currently doesn't support Nack so will likely just keep retrying message for now.
_logger.LogException(e, envelope.Id, "Error trying to receive a message from " + Address);
}
}
}
public void Start(IListeningWorkerQueue callback)
{
_listener = new TcpListener(new IPEndPoint(_ipaddr, _port));
_listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_socketHandling = new ActionBlock <Socket>(async s =>
{
using (var stream = new NetworkStream(s, true))
{
await HandleStream(callback, stream);
}
}, new ExecutionDataflowBlockOptions {
CancellationToken = _cancellationToken
});
_receivingLoop = Task.Run(async() =>
{
_listener.Start();
while (!_cancellationToken.IsCancellationRequested)
{
var socket = await _listener.AcceptSocketAsync();
await _socketHandling.SendAsync(socket, _cancellationToken);
}
}, _cancellationToken);
}
private static async Task receive(ITransportLogger logger, Stream stream, IListeningWorkerQueue callback,
Envelope[] messages, Uri uri)
{
// Just a ping
if (messages.Any() && messages.First().IsPing())
{
await stream.SendBuffer(ReceivedBuffer);
// We aren't gonna use this in this case
var ack = await stream.ReadExpectedBuffer(AcknowledgedBuffer);
return;
}
try
{
await callback.Received(uri, messages);
await stream.SendBuffer(ReceivedBuffer);
await stream.ReadExpectedBuffer(AcknowledgedBuffer);
}
catch (Exception e)
{
logger.LogException(e);
await stream.SendBuffer(ProcessingFailureBuffer);
}
}
public void Start(IListeningWorkerQueue callback)
{
_consumer.Subscribe(new [] { _endpoint.TopicName });
_consumerTask = ConsumeAsync(callback);
_logger.ListeningStatusChange(ListeningStatus.Accepting);
}
public Task HandleStream(IListeningWorkerQueue callback, Stream stream)
{
if (Status == ListeningStatus.TooBusy)
{
return(stream.SendBuffer(WireProtocol.ProcessingFailureBuffer));
}
return(WireProtocol.Receive(_logger, stream, callback, Address));
}
private async Task ConsumeAsync(IListeningWorkerQueue callback)
{
while (!_cancellation.IsCancellationRequested)
{
ConsumeResult <byte[], byte[]> message;
try
{
message = await Task.Run(() => _consumer.Consume(), _cancellation);
}
catch (Confluent.Kafka.ConsumeException cex)
{
if (cex.Error.Code == ErrorCode.PolicyViolation)
{
throw;
}
continue;
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error consuming message from Kafka topic {_endpoint.TopicName}");
continue;
}
Envelope envelope;
try
{
envelope = _protocol.ReadEnvelope(message.Message);
}
catch (Exception ex)
{
_logger.LogException(ex, message: $"Error trying to map an incoming Kafka {_endpoint.TopicName} Topic message to an Envelope. See the Dead Letter Queue");
continue;
}
try
{
await callback.Received(Address, envelope);
_consumer.Commit(message);
}
catch (Exception e)
{
// TODO -- Got to either discard this or defer it back to the queue
_logger.LogException(e, envelope.Id, "Error trying to receive a message from " + Address);
}
}
}
public void Start(IListeningWorkerQueue callback)
{
_callback = callback;
var options = new SessionHandlerOptions(handleException);
var connectionString = _transport.ConnectionString;
var tokenProvider = _transport.TokenProvider;
var subscriptionName = _endpoint.SubscriptionName;
var queueName = _endpoint.QueueName;
var retryPolicy = _transport.RetryPolicy;
var receiveMode = _transport.ReceiveMode;
var transportType = _transport.TransportType;
var topicName = _endpoint.TopicName;
if (topicName.IsEmpty())
{
var client = tokenProvider != null
? new QueueClient(connectionString, queueName, tokenProvider, transportType, receiveMode,
retryPolicy)
: new QueueClient(connectionString, queueName, receiveMode, retryPolicy);
client.RegisterSessionHandler(handleMessage, options);
_clientEntities.Add(client);
}
else
{
var client = tokenProvider != null
? new SubscriptionClient(connectionString, topicName, subscriptionName, tokenProvider,
transportType, receiveMode, retryPolicy)
: new SubscriptionClient(connectionString, topicName, subscriptionName,
receiveMode, retryPolicy);
client.RegisterSessionHandler(handleMessage, options);
_clientEntities.Add(client);
}
}
public ListeningAgent(IListeningWorkerQueue callback, IPAddress ipaddr, int port, string protocol,
CancellationToken cancellationToken)
{
Port = port;
_callback = callback;
_cancellationToken = cancellationToken;
_listener = new TcpListener(new IPEndPoint(ipaddr, port));
_listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_socketHandling = new ActionBlock <Socket>(async s =>
{
using (var stream = new NetworkStream(s, true))
{
await WireProtocol.Receive(TransportLogger.Empty(), stream, _callback, _uri);
}
}, new ExecutionDataflowBlockOptions {
CancellationToken = _cancellationToken
});
_uri = $"{protocol}://{ipaddr}:{port}/".ToUri();
}
public static async Task Receive(ITransportLogger logger, Stream stream, IListeningWorkerQueue callback,
Uri uri)
{
Envelope[] messages = null;
try
{
var lengthBytes = await stream.ReadBytesAsync(sizeof(int));
var length = BitConverter.ToInt32(lengthBytes, 0);
if (length == 0)
{
return;
}
var bytes = await stream.ReadBytesAsync(length);
messages = Envelope.ReadMany(bytes);
}
catch (Exception e)
{
logger.LogException(new MessageFailureException(messages, e));
await stream.SendBuffer(SerializationFailureBuffer);
return;
}
try
{
await receive(logger, stream, callback, messages, uri);
}
catch (Exception ex)
{
logger.LogException(new MessageFailureException(messages, ex));
await stream.SendBuffer(ProcessingFailureBuffer);
}
}
public WorkerQueueMessageConsumer(IListeningWorkerQueue callback, ITransportLogger logger, IModel channel,
IRabbitMqProtocol mapper, Uri address) : base(logger, channel, mapper, address)
{
_callback = callback;
}
public void Start(IListeningWorkerQueue callback)
{
_consumerTask = ConsumeAsync(callback);
_logger.ListeningStatusChange(ListeningStatus.Accepting);
}
