Пример #1
0
        public Task Send(IFullMessage[] messages, string destination)
        {
            var options = new SendOptions();

            options.SetDestination(destination);

            var message = new BulkMessage
            {
                Messages = messages
            };

            _metrics.Mark("Dispatched Messages", Unit.Message);
            return(Bus.Instance.Send(message, options));
        }
Пример #2
0
        public Task Publish(IFullMessage[] messages)
        {
            var options = new PublishOptions();

            _metrics.Mark("Dispatched Messages", Unit.Message);

            // Todo: publish would only be called for messages on a single stream
            // we can set a routing key somehow for BulkMessage so its routed to the same sharded queue
            var message = new BulkMessage
            {
                Messages = messages
            };

            // Publishing an IMessage normally creates a warning
            options.DoNotEnforceBestPractices();

            return(Bus.Instance.Publish(message, options));
        }
Пример #3
0
        private static async Task ProcessEvents(ThreadParam param, DelayedClient client, ResolvedEvent[] events)
        {
            // A fake message that will travel through the pipeline in order to bulk process messages from the context bag
            var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();

            var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();

            Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events on stream {events.First().Event.EventStreamId}");

            var contextBag = new ContextBag();

            // Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
            contextBag.Set(Defaults.BulkHeader, delayed);

            // Run bulk process on this thread
            using (var tokenSource = new CancellationTokenSource())
            {
                var success = false;
                var retry   = 0;
                do
                {
                    var transportTransaction = new TransportTransaction();

                    // Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
                    var messageId = Guid.NewGuid().ToString();
                    var headers   = new Dictionary <string, string>()
                    {
                        [Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
                        [Headers.MessageIntent]        = MessageIntentEnum.Send.ToString(),
                        [Headers.MessageId]            = messageId,
                        [Defaults.BulkHeader]          = delayed.Count().ToString(),
                    };

                    try
                    {
                        // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
                        param.Token.ThrowIfCancellationRequested();

                        // Don't re-use the event id for the message id
                        var messageContext = new NServiceBus.Transport.MessageContext(messageId,
                                                                                      headers,
                                                                                      bulkMarker, transportTransaction, tokenSource,
                                                                                      contextBag);
                        await Bus.OnMessage(messageContext).ConfigureAwait(false);//param.Token);

                        Logger.Write(LogLevel.Debug,
                                     () => $"Scheduling acknowledge of {delayed.Count()} bulk events");
                        DelayedCount.Increment(delayed.Count());
                        client.Acknowledge(events);
                        success = true;
                    }
                    catch (ObjectDisposedException)
                    {
                        // NSB transport has been disconnected
                        break;
                    }
                    catch (Exception e)
                    {
                        DelayedErrors.Mark($"{e.GetType().Name} {e.Message}");
                    }


                    retry++;
                } while (!success && retry <= param.MaxRetry);

                if (!success)
                {
                    // Dont run through NSB's error handler, it expects a single serialized message which
                    // is not compatible here.  Just tell EventStore to retry it
                    // Todo: ES will park messages that fail - develop something to monitor parked messages
                    client.Nack(events);
                }
            }
        }
Пример #4
0
        private static void Threaded(object state)
        {
            var param = (ThreadParam)state;
            // A fake message that will travel through the pipeline in order to bulk process messages from the context bag
            var bulkMarker = new BulkMessage().Serialize(param.JsonSettings).AsByteArray();

            param.Clients.SelectAsync(x => x.Connect()).Wait();

            var threadIdle = Metric.Timer("Delayed Events Idle", Unit.None, tags: "debug");

            TimerContext?idleContext = threadIdle.NewContext();

            while (true)
            {
                param.Token.ThrowIfCancellationRequested();

                var noEvents = true;
                for (var i = 0; i < param.Clients.Count(); i++)
                {
                    var client = param.Clients.ElementAt(i);
                    var events = client.Flush();

                    if (!events.Any())
                    {
                        continue;
                    }

                    noEvents = false;
                    idleContext?.Dispose();
                    idleContext = null;

                    var delayed = events.Select(x => x.Event.Data.Deserialize <DelayedMessage>(param.JsonSettings)).ToArray();

                    Logger.Write(LogLevel.Info, () => $"Processing {delayed.Count()} bulk events");

                    var transportTransaction = new TransportTransaction();
                    var contextBag           = new ContextBag();
                    // Hack to get all the delayed messages to bulk invoker without NSB deserializing and processing each one
                    contextBag.Set(Defaults.BulkHeader, delayed);

                    // Need to supply EnclosedMessageTypes to trick NSB pipeline into processing our fake message
                    var messageId = Guid.NewGuid().ToString();
                    var headers   = new Dictionary <string, string>()
                    {
                        [Headers.EnclosedMessageTypes] = typeof(BulkMessage).AssemblyQualifiedName,
                        [Headers.MessageIntent]        = MessageIntentEnum.Send.ToString(),
                        [Headers.MessageId]            = messageId,
                        [Defaults.BulkHeader]          = delayed.Count().ToString(),
                    };

                    // Run bulk process on this thread
                    using (var tokenSource = new CancellationTokenSource())
                    {
                        var success = false;
                        var retry   = 0;
                        do
                        {
                            using (var ctx = DelayedExecution.NewContext())
                            {
                                try
                                {
                                    // If canceled, this will throw the number of time immediate retry requires to send the message to the error queue
                                    param.Token.ThrowIfCancellationRequested();

                                    // Don't re-use the event id for the message id
                                    var messageContext = new NServiceBus.Transport.MessageContext(messageId,
                                                                                                  headers,
                                                                                                  bulkMarker, transportTransaction, tokenSource,
                                                                                                  contextBag);
                                    Bus.OnMessage(messageContext).Wait(param.Token);

                                    Logger.Write(LogLevel.Debug,
                                                 () => $"Scheduling acknowledge of {delayed.Count()} bulk events");
                                    DelayedHandled.Update(delayed.Count());
                                    client.Acknowledge(events);
                                    success = true;
                                }
                                catch (ObjectDisposedException)
                                {
                                    // NSB transport has been disconnected
                                    break;
                                }
                                catch (Exception ex)
                                {
                                }

                                if (ctx.Elapsed > TimeSpan.FromSeconds(5))
                                {
                                    SlowLogger.Warn(
                                        $"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalSeconds} seconds!");
                                }
                                Logger.Write(LogLevel.Info,
                                             () =>
                                             $"Processing {delayed.Count()} bulked events took {ctx.Elapsed.TotalMilliseconds} ms");
                            }
                            // Todo: use max retry setting
                        } while (!success && retry < 10);

                        if (!success)
                        {
                            // Dont run through NSB's error handler, it expects a single serialized message which
                            // is not compatible here.  Just tell EventStore to retry it
                            // Todo: ES will park messages that fail - develop something to monitor parked messages
                            client.Nack(events);
                        }
                    }
                }
                if (idleContext == null)
                {
                    idleContext = threadIdle.NewContext();
                }
                // Cheap hack to not burn cpu incase there are no events
                if (noEvents)
                {
                    Thread.Sleep(50);
                }
            }
        }