private Action <EventStorePersistentSubscriptionBase, ResolvedEvent> EventProcessor(CancellationToken token)
{
return((subscription, e) =>
{
var @event = e.Event;
if ([email protected])
{
_toBeAcknowledged.Add(e);
return;
}
Logger.Write(LogLevel.Debug,
() => $"Event {@event.EventId} type {@event.EventType} appeared stream [{@event.EventStreamId}] number {@event.EventNumber}");
var descriptor = @event.Metadata.Deserialize(_settings);
if (token.IsCancellationRequested)
{
subscription.Stop(TimeSpan.FromSeconds(30));
return;
}
_concurrencyLimit.WaitAsync(token).ConfigureAwait(false).GetAwaiter().GetResult();
Task.Run(async() =>
{
var headers = new Dictionary <string, string>(descriptor.Headers)
{
[Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(Type.GetType(@event.EventType)),
[Headers.MessageId] = @event.EventId.ToString()
};
Logger.Write(LogLevel.Debug, () => $"Processing event {@event.EventId}");
using (var tokenSource = new CancellationTokenSource())
{
var processed = false;
var numberOfDeliveryAttempts = 0;
while (!processed)
{
try
{
var messageContext = new MessageContext(@event.EventId.ToString(), headers,
@event.Data ?? new byte[0], transportTranaction, tokenSource, contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);
processed = true;
}
catch (ObjectDisposedException)
{
// Rabbit has been disconnected
subscription.Stop(TimeSpan.FromSeconds(30));
return;
}
catch (Exception ex)
{
++numberOfDeliveryAttempts;
var errorContext = new ErrorContext(ex, headers, @event.EventId.ToString(),
@event.Data ?? new byte[0], transportTranaction, numberOfDeliveryAttempts);
if (await Bus.OnError(errorContext).ConfigureAwait(false) == ErrorHandleResult.Handled)
{
break;
}
_concurrencyLimit.Release();
await
Task.Delay(100 * (numberOfDeliveryAttempts / 2), cancellationToken: token)
.ConfigureAwait(false);
await _concurrencyLimit.WaitAsync(token).ConfigureAwait(false);
}
}
_concurrencyLimit.Release();
if (tokenSource.IsCancellationRequested)
{
return;
}
Logger.Write(LogLevel.Debug, () => $"Queueing acknowledge for event {@event.EventId}");
_toBeAcknowledged.Add(e);
//subscription.Acknowledge(e);
}
}, token);
});
}
private static async Task ProcessEvents(ThreadParam param, DelayedClient client, ResolvedEvent[] events)
{
// A fake message that will travel through the pipeline in order to bulk process messages from the context bag
var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();
var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();
Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events on stream {events.First().Event.EventStreamId}");
var contextBag = new ContextBag();
// Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
contextBag.Set(Defaults.BulkHeader, delayed);
// Run bulk process on this thread
using (var tokenSource = new CancellationTokenSource())
{
var success = false;
var retry = 0;
do
{
var transportTransaction = new TransportTransaction();
// Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>()
{
[Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.MessageId] = messageId,
[Defaults.BulkHeader] = delayed.Count().ToString(),
};
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
param.Token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new NServiceBus.Transport.MessageContext(messageId,
headers,
bulkMarker, transportTransaction, tokenSource,
contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token);
Logger.Write(LogLevel.Debug,
() => $"Scheduling acknowledge of {delayed.Count()} bulk events");
DelayedCount.Increment(delayed.Count());
client.Acknowledge(events);
success = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
break;
}
catch (Exception e)
{
DelayedErrors.Mark($"{e.GetType().Name} {e.Message}");
}
retry++;
} while (!success && retry <= param.MaxRetry);
if (!success)
{
// Dont run through NSB's error handler, it expects a single serialized message which
// is not compatible here. Just tell EventStore to retry it
// Todo: ES will park messages that fail - develop something to monitor parked messages
client.Nack(events);
}
}
}
private static async Task ProcessEvent(IMessaging messaging, string stream, long position, IFullEvent @event, CancellationToken token)
{
Logger.Write(LogLevel.Debug, () => $"Processing event from stream [{@event.Descriptor.StreamId}] bucket [{@event.Descriptor.Bucket}] entity [{@event.Descriptor.EntityType}] event id {@event.EventId}");
var contextBag = new ContextBag();
// Hack to get all the events to invoker without NSB deserializing
contextBag.Set(Defaults.EventHeader, @event.Event);
using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token))
{
var processed = false;
var numberOfDeliveryAttempts = 0;
var messageId = Guid.NewGuid().ToString();
var corrId = "";
if (@event.Descriptor?.Headers?.ContainsKey(Headers.CorrelationId) ?? false)
{
corrId = @event.Descriptor.Headers[Headers.CorrelationId];
}
while (!processed)
{
var transportTransaction = new TransportTransaction();
var headers = new Dictionary <string, string>(@event.Descriptor.Headers ?? new Dictionary <string, string>())
{
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(messaging, @event.Event.GetType()),
[Headers.MessageId] = messageId,
[Headers.CorrelationId] = corrId,
[Defaults.EventHeader] = "",
[$"{Defaults.EventPrefixHeader}.EventId"] = @event.EventId.ToString(),
[$"{Defaults.EventPrefixHeader}.EventStream"] = stream,
[$"{Defaults.EventPrefixHeader}.EventPosition"] = position.ToString()
};
using (var ctx = EventExecution.NewContext())
{
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new MessageContext(messageId,
headers,
Marker, transportTransaction, tokenSource,
contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);
EventsHandled.Increment();
processed = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
throw new OperationCanceledException();
}
catch (Exception ex)
{
EventErrors.Mark($"{ex.GetType().Name} {ex.Message}");
++numberOfDeliveryAttempts;
// Don't retry a cancelation
if (tokenSource.IsCancellationRequested)
{
numberOfDeliveryAttempts = Int32.MaxValue;
}
var errorContext = new ErrorContext(ex, headers,
messageId,
Marker, transportTransaction,
numberOfDeliveryAttempts);
if (await Bus.OnError(errorContext).ConfigureAwait(false) ==
ErrorHandleResult.Handled || tokenSource.IsCancellationRequested)
{
break;
}
}
}
}
}
}
private static async Task ProcessEvent(MessageMetadataRegistry messageMeta, JsonSerializerSettings settings, RecordedEvent @event, CancellationToken token)
{
var transportTransaction = new TransportTransaction();
var contextBag = new ContextBag();
var metadata = @event.Metadata;
var data = @event.Data;
var descriptor = metadata.Deserialize(settings);
if (descriptor.Compressed)
{
data = data.Decompress();
}
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>(descriptor.Headers)
{
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(messageMeta, Type.GetType(@event.EventType)),
[Headers.MessageId] = messageId,
["EventId"] = @event.EventId.ToString(),
["EventStreamId"] = @event.EventStreamId,
["EventNumber"] = @event.EventNumber.ToString()
};
using (var tokenSource = new CancellationTokenSource())
{
var processed = false;
var numberOfDeliveryAttempts = 0;
while (!processed)
{
using (var ctx = EventExecution.NewContext())
{
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new MessageContext(messageId,
headers,
data ?? new byte[0], transportTransaction, tokenSource,
contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);
EventCount.Increment();
processed = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
break;
}
catch (Exception ex)
{
EventErrors.Mark($"{ex.GetType().Name} {ex.Message}");
++numberOfDeliveryAttempts;
var errorContext = new ErrorContext(ex, headers,
messageId,
data ?? new byte[0], transportTransaction,
numberOfDeliveryAttempts);
if (await Bus.OnError(errorContext).ConfigureAwait(false) ==
ErrorHandleResult.Handled)
{
break;
}
}
}
}
}
}
private static async Task ProcessEvents(ThreadParam param, IFullEvent[] events)
{
var delayed = events.Select(x => x.Event as IDelayedMessage).ToArray();
Logger.Write(LogLevel.Debug, () => $"Processing {delayed.Count()} bulk events from stream [{events.First().Descriptor.StreamId}] bucket [{events.First().Descriptor.Bucket}] entity [{events.First().Descriptor.EntityType}]");
var contextBag = new ContextBag();
// Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
contextBag.Set(Defaults.BulkHeader, delayed);
// Run bulk process on this thread
using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(param.Token))
{
var success = false;
var retry = 0;
var messageId = Guid.NewGuid().ToString();
do
{
var transportTransaction = new TransportTransaction();
// Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
var headers = new Dictionary <string, string>()
{
[Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.MessageId] = messageId,
[Defaults.BulkHeader] = delayed.Count().ToString(),
};
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
param.Token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new NServiceBus.Transport.MessageContext(messageId,
headers,
Marker, transportTransaction, tokenSource,
contextBag);
await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token);
tokenSource.Token.ThrowIfCancellationRequested();
Logger.Write(LogLevel.Debug,
() => $"Scheduling acknowledge of {delayed.Count()} bulk events");
DelayedHandled.Increment(delayed.Count());
await param.Consumer.Acknowledge(events).ConfigureAwait(false);
success = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
throw new OperationCanceledException();
}
catch (Exception e)
{
// Don't retry a cancelation
if (tokenSource.IsCancellationRequested)
{
throw;
}
DelayedErrors.Mark($"{e.GetType().Name} {e.Message}");
if ((retry % param.MaxRetry) == 0)
{
Logger.Warn($"So far, we've received {retry} errors while running {delayed.Count()} bulk events from stream [{events.First().Descriptor.StreamId}] bucket [{events.First().Descriptor.Bucket}] entity [{events.First().Descriptor.EntityType}]", e);
}
// Don't burn cpu in case of non-transient errors
await Task.Delay((retry / 5) * 200, param.Token).ConfigureAwait(false);
}
retry++;
// Keep retrying forever but print warn messages once MaxRetry exceeded
} while (!success);
}
}
private static void Threaded(object state)
{
var param = (ThreadParam)state;
// A fake message that will travel through the pipeline in order to bulk process messages from the context bag
var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();
param.Clients.SelectAsync(x => x.Connect()).Wait();
var threadIdle = Metric.Timer("Delayed Events Idle", Unit.None, tags: "debug");
TimerContext?idleContext = threadIdle.NewContext();
while (true)
{
param.Token.ThrowIfCancellationRequested();
var noEvents = true;
for (var i = 0; i < param.Clients.Count(); i++)
{
var client = param.Clients.ElementAt(i);
var events = client.Flush();
if (!events.Any())
{
continue;
}
noEvents = false;
idleContext?.Dispose();
idleContext = null;
var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();
Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events");
var transportTransaction = new TransportTransaction();
var contextBag = new ContextBag();
// Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
contextBag.Set(Defaults.BulkHeader, delayed);
// Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
var messageId = Guid.NewGuid().ToString();
var headers = new Dictionary <string, string>()
{
[Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
[Headers.MessageIntent] = MessageIntentEnum.Send.ToString(),
[Headers.MessageId] = messageId,
[Defaults.BulkHeader] = delayed.Count().ToString(),
};
// Run bulk process on this thread
using (var tokenSource = new CancellationTokenSource())
{
var success = false;
var retry = 0;
do
{
using (var ctx = DelayedExecution.NewContext())
{
try
{
// If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
param.Token.ThrowIfCancellationRequested();
// Don't re-use the event id for the message id
var messageContext = new NServiceBus.Transport.MessageContext(messageId,
headers,
bulkMarker, transportTransaction, tokenSource,
contextBag);
Bus.OnMessage(messageContext).Wait(param.Token);
Logger.Write(LogLevel.Debug,
() => $"Scheduling acknowledge of {delayed.Count()} bulk events");
DelayedHandled.Update(delayed.Count());
client.Acknowledge(events);
success = true;
}
catch (ObjectDisposedException)
{
// NSB transport has been disconnected
break;
}
catch (Exception ex)
{
}
if (ctx.Elapsed > TimeSpan.FromSeconds(5))
{
SlowLogger.Warn(
$"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalSeconds} seconds!");
}
Logger.Write(LogLevel.Info,
() =>
$"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalMilliseconds} ms");
}
// Todo: use max retry setting
} while (!success && retry < 10);
if (!success)
{
// Dont run through NSB's error handler, it expects a single serialized message which
// is not compatible here. Just tell EventStore to retry it
// Todo: ES will park messages that fail - develop something to monitor parked messages
client.Nack(events);
}
}
}
if (idleContext == null)
{
idleContext = threadIdle.NewContext();
}
// Cheap hack to not burn cpu incase there are no events
if (noEvents)
{
Thread.Sleep(50);
}
}
}