private Action <EventStorePersistentSubscriptionBase, ResolvedEvent> EventProcessor(CancellationToken token) { return((subscription, e) => { var @event = e.Event; if ([email protected]) { _toBeAcknowledged.Add(e); return; } Logger.Write(LogLevel.Debug, () => $"Event {@event.EventId} type {@event.EventType} appeared stream [{@event.EventStreamId}] number {@event.EventNumber}"); var descriptor = @event.Metadata.Deserialize(_settings); if (token.IsCancellationRequested) { subscription.Stop(TimeSpan.FromSeconds(30)); return; } _concurrencyLimit.WaitAsync(token).ConfigureAwait(false).GetAwaiter().GetResult(); Task.Run(async() => { var headers = new Dictionary <string, string>(descriptor.Headers) { [Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(Type.GetType(@event.EventType)), [Headers.MessageId] = @event.EventId.ToString() }; Logger.Write(LogLevel.Debug, () => $"Processing event {@event.EventId}"); using (var tokenSource = new CancellationTokenSource()) { var processed = false; var numberOfDeliveryAttempts = 0; while (!processed) { try { var messageContext = new MessageContext(@event.EventId.ToString(), headers, @event.Data ?? new byte[0], transportTranaction, tokenSource, contextBag); await Bus.OnMessage(messageContext).ConfigureAwait(false); processed = true; } catch (ObjectDisposedException) { // Rabbit has been disconnected subscription.Stop(TimeSpan.FromSeconds(30)); return; } catch (Exception ex) { ++numberOfDeliveryAttempts; var errorContext = new ErrorContext(ex, headers, @event.EventId.ToString(), @event.Data ?? new byte[0], transportTranaction, numberOfDeliveryAttempts); if (await Bus.OnError(errorContext).ConfigureAwait(false) == ErrorHandleResult.Handled) { break; } _concurrencyLimit.Release(); await Task.Delay(100 * (numberOfDeliveryAttempts / 2), cancellationToken: token) .ConfigureAwait(false); await _concurrencyLimit.WaitAsync(token).ConfigureAwait(false); } } _concurrencyLimit.Release(); if (tokenSource.IsCancellationRequested) { return; } Logger.Write(LogLevel.Debug, () => $"Queueing acknowledge for event {@event.EventId}"); _toBeAcknowledged.Add(e); //subscription.Acknowledge(e); } }, token); }); }
private static async Task ProcessEvents(ThreadParam param, DelayedClient client, ResolvedEvent[] events) { // A fake message that will travel through the pipeline in order to bulk process messages from the context bag var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray(); var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray(); Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events on stream {events.First().Event.EventStreamId}"); var contextBag = new ContextBag(); // Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one contextBag.Set(Defaults.BulkHeader, delayed); // Run bulk process on this thread using (var tokenSource = new CancellationTokenSource()) { var success = false; var retry = 0; do { var transportTransaction = new TransportTransaction(); // Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message var messageId = Guid.NewGuid().ToString(); var headers = new Dictionary <string, string>() { [Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName, [Headers.MessageIntent] = MessageIntentEnum.Send.ToString(), [Headers.MessageId] = messageId, [Defaults.BulkHeader] = delayed.Count().ToString(), }; try { // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue param.Token.ThrowIfCancellationRequested(); // Don't re-use the event id for the message id var messageContext = new NServiceBus.Transport.MessageContext(messageId, headers, bulkMarker, transportTransaction, tokenSource, contextBag); await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token); Logger.Write(LogLevel.Debug, () => $"Scheduling acknowledge of {delayed.Count()} bulk events"); DelayedCount.Increment(delayed.Count()); client.Acknowledge(events); success = true; } catch (ObjectDisposedException) { // NSB transport has been disconnected break; } catch (Exception e) { DelayedErrors.Mark($"{e.GetType().Name} {e.Message}"); } retry++; } while (!success && retry <= param.MaxRetry); if (!success) { // Dont run through NSB's error handler, it expects a single serialized message which // is not compatible here. Just tell EventStore to retry it // Todo: ES will park messages that fail - develop something to monitor parked messages client.Nack(events); } } }
private static async Task ProcessEvent(IMessaging messaging, string stream, long position, IFullEvent @event, CancellationToken token) { Logger.Write(LogLevel.Debug, () => $"Processing event from stream [{@event.Descriptor.StreamId}] bucket [{@event.Descriptor.Bucket}] entity [{@event.Descriptor.EntityType}] event id {@event.EventId}"); var contextBag = new ContextBag(); // Hack to get all the events to invoker without NSB deserializing contextBag.Set(Defaults.EventHeader, @event.Event); using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token)) { var processed = false; var numberOfDeliveryAttempts = 0; var messageId = Guid.NewGuid().ToString(); var corrId = ""; if (@event.Descriptor?.Headers?.ContainsKey(Headers.CorrelationId) ?? false) { corrId = @event.Descriptor.Headers[Headers.CorrelationId]; } while (!processed) { var transportTransaction = new TransportTransaction(); var headers = new Dictionary <string, string>(@event.Descriptor.Headers ?? new Dictionary <string, string>()) { [Headers.MessageIntent] = MessageIntentEnum.Send.ToString(), [Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(messaging, @event.Event.GetType()), [Headers.MessageId] = messageId, [Headers.CorrelationId] = corrId, [Defaults.EventHeader] = "", [$"{Defaults.EventPrefixHeader}.EventId"] = @event.EventId.ToString(), [$"{Defaults.EventPrefixHeader}.EventStream"] = stream, [$"{Defaults.EventPrefixHeader}.EventPosition"] = position.ToString() }; using (var ctx = EventExecution.NewContext()) { try { // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue token.ThrowIfCancellationRequested(); // Don't re-use the event id for the message id var messageContext = new MessageContext(messageId, headers, Marker, transportTransaction, tokenSource, contextBag); await Bus.OnMessage(messageContext).ConfigureAwait(false); EventsHandled.Increment(); processed = true; } catch (ObjectDisposedException) { // NSB transport has been disconnected throw new OperationCanceledException(); } catch (Exception ex) { EventErrors.Mark($"{ex.GetType().Name} {ex.Message}"); ++numberOfDeliveryAttempts; // Don't retry a cancelation if (tokenSource.IsCancellationRequested) { numberOfDeliveryAttempts = Int32.MaxValue; } var errorContext = new ErrorContext(ex, headers, messageId, Marker, transportTransaction, numberOfDeliveryAttempts); if (await Bus.OnError(errorContext).ConfigureAwait(false) == ErrorHandleResult.Handled || tokenSource.IsCancellationRequested) { break; } } } } } }
private static async Task ProcessEvent(MessageMetadataRegistry messageMeta, JsonSerializerSettings settings, RecordedEvent @event, CancellationToken token) { var transportTransaction = new TransportTransaction(); var contextBag = new ContextBag(); var metadata = @event.Metadata; var data = @event.Data; var descriptor = metadata.Deserialize(settings); if (descriptor.Compressed) { data = data.Decompress(); } var messageId = Guid.NewGuid().ToString(); var headers = new Dictionary <string, string>(descriptor.Headers) { [Headers.MessageIntent] = MessageIntentEnum.Send.ToString(), [Headers.EnclosedMessageTypes] = SerializeEnclosedMessageTypes(messageMeta, Type.GetType(@event.EventType)), [Headers.MessageId] = messageId, ["EventId"] = @event.EventId.ToString(), ["EventStreamId"] = @event.EventStreamId, ["EventNumber"] = @event.EventNumber.ToString() }; using (var tokenSource = new CancellationTokenSource()) { var processed = false; var numberOfDeliveryAttempts = 0; while (!processed) { using (var ctx = EventExecution.NewContext()) { try { // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue token.ThrowIfCancellationRequested(); // Don't re-use the event id for the message id var messageContext = new MessageContext(messageId, headers, data ?? new byte[0], transportTransaction, tokenSource, contextBag); await Bus.OnMessage(messageContext).ConfigureAwait(false); EventCount.Increment(); processed = true; } catch (ObjectDisposedException) { // NSB transport has been disconnected break; } catch (Exception ex) { EventErrors.Mark($"{ex.GetType().Name} {ex.Message}"); ++numberOfDeliveryAttempts; var errorContext = new ErrorContext(ex, headers, messageId, data ?? new byte[0], transportTransaction, numberOfDeliveryAttempts); if (await Bus.OnError(errorContext).ConfigureAwait(false) == ErrorHandleResult.Handled) { break; } } } } } }
private static async Task ProcessEvents(ThreadParam param, IFullEvent[] events) { var delayed = events.Select(x => x.Event as IDelayedMessage).ToArray(); Logger.Write(LogLevel.Debug, () => $"Processing {delayed.Count()} bulk events from stream [{events.First().Descriptor.StreamId}] bucket [{events.First().Descriptor.Bucket}] entity [{events.First().Descriptor.EntityType}]"); var contextBag = new ContextBag(); // Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one contextBag.Set(Defaults.BulkHeader, delayed); // Run bulk process on this thread using (var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(param.Token)) { var success = false; var retry = 0; var messageId = Guid.NewGuid().ToString(); do { var transportTransaction = new TransportTransaction(); // Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message var headers = new Dictionary <string, string>() { [Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName, [Headers.MessageIntent] = MessageIntentEnum.Send.ToString(), [Headers.MessageId] = messageId, [Defaults.BulkHeader] = delayed.Count().ToString(), }; try { // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue param.Token.ThrowIfCancellationRequested(); // Don't re-use the event id for the message id var messageContext = new NServiceBus.Transport.MessageContext(messageId, headers, Marker, transportTransaction, tokenSource, contextBag); await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token); tokenSource.Token.ThrowIfCancellationRequested(); Logger.Write(LogLevel.Debug, () => $"Scheduling acknowledge of {delayed.Count()} bulk events"); DelayedHandled.Increment(delayed.Count()); await param.Consumer.Acknowledge(events).ConfigureAwait(false); success = true; } catch (ObjectDisposedException) { // NSB transport has been disconnected throw new OperationCanceledException(); } catch (Exception e) { // Don't retry a cancelation if (tokenSource.IsCancellationRequested) { throw; } DelayedErrors.Mark($"{e.GetType().Name} {e.Message}"); if ((retry % param.MaxRetry) == 0) { Logger.Warn($"So far, we've received {retry} errors while running {delayed.Count()} bulk events from stream [{events.First().Descriptor.StreamId}] bucket [{events.First().Descriptor.Bucket}] entity [{events.First().Descriptor.EntityType}]", e); } // Don't burn cpu in case of non-transient errors await Task.Delay((retry / 5) * 200, param.Token).ConfigureAwait(false); } retry++; // Keep retrying forever but print warn messages once MaxRetry exceeded } while (!success); } }
private static void Threaded(object state) { var param = (ThreadParam)state; // A fake message that will travel through the pipeline in order to bulk process messages from the context bag var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray(); param.Clients.SelectAsync(x => x.Connect()).Wait(); var threadIdle = Metric.Timer("Delayed Events Idle", Unit.None, tags: "debug"); TimerContext?idleContext = threadIdle.NewContext(); while (true) { param.Token.ThrowIfCancellationRequested(); var noEvents = true; for (var i = 0; i < param.Clients.Count(); i++) { var client = param.Clients.ElementAt(i); var events = client.Flush(); if (!events.Any()) { continue; } noEvents = false; idleContext?.Dispose(); idleContext = null; var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray(); Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events"); var transportTransaction = new TransportTransaction(); var contextBag = new ContextBag(); // Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one contextBag.Set(Defaults.BulkHeader, delayed); // Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message var messageId = Guid.NewGuid().ToString(); var headers = new Dictionary <string, string>() { [Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName, [Headers.MessageIntent] = MessageIntentEnum.Send.ToString(), [Headers.MessageId] = messageId, [Defaults.BulkHeader] = delayed.Count().ToString(), }; // Run bulk process on this thread using (var tokenSource = new CancellationTokenSource()) { var success = false; var retry = 0; do { using (var ctx = DelayedExecution.NewContext()) { try { // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue param.Token.ThrowIfCancellationRequested(); // Don't re-use the event id for the message id var messageContext = new NServiceBus.Transport.MessageContext(messageId, headers, bulkMarker, transportTransaction, tokenSource, contextBag); Bus.OnMessage(messageContext).Wait(param.Token); Logger.Write(LogLevel.Debug, () => $"Scheduling acknowledge of {delayed.Count()} bulk events"); DelayedHandled.Update(delayed.Count()); client.Acknowledge(events); success = true; } catch (ObjectDisposedException) { // NSB transport has been disconnected break; } catch (Exception ex) { } if (ctx.Elapsed > TimeSpan.FromSeconds(5)) { SlowLogger.Warn( $"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalSeconds} seconds!"); } Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalMilliseconds} ms"); } // Todo: use max retry setting } while (!success && retry < 10); if (!success) { // Dont run through NSB's error handler, it expects a single serialized message which // is not compatible here. Just tell EventStore to retry it // Todo: ES will park messages that fail - develop something to monitor parked messages client.Nack(events); } } } if (idleContext == null) { idleContext = threadIdle.NewContext(); } // Cheap hack to not burn cpu incase there are no events if (noEvents) { Thread.Sleep(50); } } }