Example #1
0
        public ProxySystem(ServerId local, QueueConfiguration queueConfiguration,
                           ConnectionConfiguration connectionConfiguration,
                           ProxyCacheConfiguration cacheConfiguration,
                           ProxyCacheConfiguration asyncCacheConfiguration,
                           NetReceiverConfiguration receiverConfiguration,
                           AsyncTasksConfiguration asyncGetData,
                           AsyncTasksConfiguration asyncPing, ConnectionTimeoutConfiguration connectionTimeoutConfiguration)
        {
            Contract.Requires(queueConfiguration != null);
            Contract.Requires(connectionConfiguration != null);
            Contract.Requires(cacheConfiguration != null);
            Contract.Requires(asyncCacheConfiguration != null);
            Contract.Requires(receiverConfiguration != null);
            Contract.Requires(asyncGetData != null);
            Contract.Requires(local != null);
            Contract.Requires(asyncPing != null);

            _local = local;
            _queueConfiguration       = queueConfiguration;
            _connectionConfiguration  = connectionConfiguration;
            _cacheConfiguration       = cacheConfiguration;
            _asyncCacheConfiguration  = asyncCacheConfiguration;
            _netReceiverConfiguration = receiverConfiguration;
            _asyncGetData             = asyncGetData;
            _asyncPing = asyncPing;
            _connectionTimeoutConfiguration = connectionTimeoutConfiguration;
        }
Example #2
0
File: JobQueue.cs Project: zxbe/abp
        protected virtual Task EnsureInitializedAsync()
        {
            if (ChannelAccessor != null)
            {
                return(Task.CompletedTask);
            }

            ChannelAccessor = ChannelPool.Acquire(
                ChannelPrefix + QueueConfiguration.QueueName,
                QueueConfiguration.ConnectionName
                );

            var result = QueueConfiguration.Declare(ChannelAccessor.Channel);

            Logger.LogDebug($"RabbitMQ Queue '{QueueConfiguration.QueueName}' has {result.MessageCount} messages and {result.ConsumerCount} consumers.");

            if (AbpBackgroundJobOptions.IsJobExecutionEnabled)
            {
                Consumer           = new EventingBasicConsumer(ChannelAccessor.Channel);
                Consumer.Received += MessageReceived;

                //TODO: What BasicConsume returns?
                ChannelAccessor.Channel.BasicConsume(
                    queue: QueueConfiguration.QueueName,
                    autoAck: false,
                    consumer: Consumer
                    );
            }

            return(Task.CompletedTask);
        }
Example #3
0
        public WriterSystem(ServerId local, QueueConfiguration queueConfiguration,
                            NetReceiverConfiguration receiverConfigurationForWrite,
                            NetReceiverConfiguration receiverConfigurationForCollector,
                            HashMapConfiguration hashMapConfiguration,
                            ConnectionConfiguration connectionConfiguration,
                            RestoreModuleConfiguration transferRestoreConfiguration,
                            RestoreModuleConfiguration initiatorRestoreConfiguration,
                            ConnectionTimeoutConfiguration connectionTimeoutConfiguration,
                            RestoreModuleConfiguration timeoutRestoreConfiguration,
                            bool isNeedRestore = false,
                            QueueConfiguration queueConfigurationRestore = null)
        {
            Contract.Requires(local != null);
            Contract.Requires(queueConfiguration != null);
            Contract.Requires(receiverConfigurationForWrite != null);
            Contract.Requires(receiverConfigurationForCollector != null);
            Contract.Requires(hashMapConfiguration != null);
            Contract.Requires(connectionConfiguration != null);
            Contract.Requires(transferRestoreConfiguration != null);
            Contract.Requires(initiatorRestoreConfiguration != null);

            _queueConfigurationRestore = queueConfigurationRestore ?? new QueueConfiguration(1, 1000);

            _queueConfiguration                = queueConfiguration;
            _receiverConfigurationForWrite     = receiverConfigurationForWrite;
            _receiverConfigurationForCollector = receiverConfigurationForCollector;
            _hashMapConfiguration              = hashMapConfiguration;
            _connectionConfiguration           = connectionConfiguration;
            _initiatorRestoreConfiguration     = initiatorRestoreConfiguration;
            _connectionTimeoutConfiguration    = connectionTimeoutConfiguration;
            _timeoutRestoreConfiguration       = timeoutRestoreConfiguration;
            _isNeedRestore = isNeedRestore;
            _transferRestoreConfiguration = transferRestoreConfiguration;
            _local = local;
        }
Example #4
0
        protected ProxyApi(NetConfiguration netConfiguration, ProxyConfiguration proxyConfiguration,
                           CommonConfiguration commonConfiguration, TimeoutConfiguration timeoutConfiguration)
        {
            Contract.Requires(netConfiguration != null);
            Contract.Requires(proxyConfiguration != null);
            Contract.Requires(commonConfiguration != null);
            Contract.Requires(timeoutConfiguration != null);

            _isStarted = false;
            _isBuild   = false;
            _isDispose = false;

            var server = new ServerId(netConfiguration.Host, netConfiguration.Port);
            var queue  = new QueueConfiguration(commonConfiguration.CountThreads, commonConfiguration.QueueSize);

            var connection = new ConnectionConfiguration(netConfiguration.WcfServiceName,
                                                         netConfiguration.CountConnectionsToSingleServer);
            var proxyCacheConfiguration  = new ProxyCacheConfiguration(proxyConfiguration.ChangeDistributorTimeoutSec);
            var proxyCacheConfiguration2 = new ProxyCacheConfiguration(proxyConfiguration.SyncOperationsTimeoutSec);
            var netReceiveConfiguration  = new NetReceiverConfiguration(netConfiguration.Port, netConfiguration.Host,
                                                                        netConfiguration.WcfServiceName);
            var async   = new AsyncTasksConfiguration(proxyConfiguration.AsyncUpdateTimeout);
            var ping    = new AsyncTasksConfiguration(proxyConfiguration.AsyncPingTimeout);
            var timeout = new ConnectionTimeoutConfiguration(timeoutConfiguration.OpenTimeout,
                                                             timeoutConfiguration.SendTimeout);

            _proxySystem = new ProxySystem(server, queue, connection,
                                           proxyCacheConfiguration, proxyCacheConfiguration2, netReceiveConfiguration, async, ping, timeout);

            _apis = new Dictionary <string, ProxyHandlerBase>();
        }
Example #5
0
        private void RegisterMessageQueue()
        {
            ITinyThreadPool threadPool = TinyThreadPool.Create(x =>
            {
                x.MaxThreads             = 3;
                x.MultiThreadingCapacity = MultiThreadingCapacity.PerProcessor;
            });

            IMessageQueue inputQueue = QueueConfiguration.Create()
                                       .Configure(config =>
            {
                config.DataSerializer = KernelInstance.Get <IDataSerializer>();
                config.QueueUrl       = _settings.InputTweetQueueUrl;
            }).CreateLocalQueue();

            IMessageQueue outputQueue = QueueConfiguration.Create()
                                        .Configure(config =>
            {
                config.DataSerializer = KernelInstance.Get <IDataSerializer>();
                config.QueueUrl       = _settings.OutputTweetQueueUrl;
            }).CreateLocalQueue();

            Bind <StopWordsProcessor>().ToConstructor(x => new StopWordsProcessor(inputQueue, outputQueue, threadPool, x.Inject <IDataSerializer>()))
            .InSingletonScope();
        }
Example #6
0
        public async Task DripFeedMessages(uint prefetch, int numberofMessages)
        {
            var queueConfig = new QueueConfiguration
            {
                QueueName              = QueueName,
                PrefetchCount          = (ushort)prefetch,
                OnErrorAction          = QueueConfiguration.ErrorAction.RestartConnection,
                ConsumerTag            = "consumer",
                RetryIntervalInSeconds = 10
            };
            var queue = new QueueService(ExposedRabbitService.validConfig, subscriberlogger.Object);

            queue.Start(queueConfig, handler);

            for (uint messagecount = 1; messagecount < numberofMessages + 1; messagecount++)
            {
                basicService.ExposedChannel.BasicPublish(ExchangeName, "", true, null, Encoding.UTF8.GetBytes("message"));
                await Task.Delay(RoundTripWaitTime);

                var queueMessageCount = basicService.ExposedChannel.MessageCount(QueueName);

                if (prefetch > messagecount)
                {
                    queueMessageCount.Should().Be(0, "because there are {0} messages with {1} prefetch", messagecount, prefetch);
                    processCount.Should().Be(messagecount, "because the prefetch is {0}", prefetch);
                }
                // if prefetch is greater than message number, then the final count should be 0.
                else
                {
                    queueMessageCount.Should().Be(messagecount - prefetch, "because there are messages in queue exceeds prefetch");
                    processCount.Should().Be(prefetch, "because the prefetch exceeds number of messages");
                }
            }
            queue.Dispose();
        }
        public void When_rejecting_a_message_it_is_removed_from_the_queue()
        {
            // Arrange
            const string uri = @"amqp://*****:*****@sea-2600-53:5672/Dev_JoshR";

            var message = new TestMessage {
                Identifier = Guid.NewGuid().ToString(), Description = "something", SentAt = DateTime.Now
            };
            var receivedMessages = new List <TestMessage>();

            var queueConfiguration = new QueueConfiguration {
                IsQueueDurable = true, IsQueueExclusive = false, SupportsAutoDelete = false
            };
            var exchangeConfiguration = new ExchangeConfiguration {
                ExchangeType = ExchangeType.Direct, IsExchangeDurable = true
            };

            var receiver = new RabbitMqMessageSubscriber <TestMessage>(uri, new ClassTypeNameExchangeNameFactory(), new ClassTypeQueueNameFactory(), new EmptyRoutingKeyFactory(), new JsonMessageFormatter(), queueConfiguration);

            receiver.Subscribe(s =>
            {
                receivedMessages.Add(s.Message);
                s.Reject();
            });

            var sender = new RabbitMqMessenger(uri, new ClassTypeNameExchangeNameFactory(), new EmptyRoutingKeyFactory(), new JsonMessageFormatter(), exchangeConfiguration);

            // Act
            sender.Send(message);

            Thread.Sleep(5000);

            // Assert
            Assert.That(receivedMessages, Is.Not.Empty);
        }
Example #8
0
        public async Task NackOnException()
        {
            var queueConfig = new QueueConfiguration
            {
                QueueName              = QueueName,
                PrefetchCount          = 1,
                OnErrorAction          = QueueConfiguration.ErrorAction.NackOnException,
                ConsumerTag            = "consumer",
                RetryIntervalInSeconds = 1
            };
            var queue = new QueueService(ExposedRabbitService.validConfig, subscriberlogger.Object);

            queue.Start(queueConfig, handler);

            var initialConsumerCount = basicService.ExposedChannel.ConsumerCount(QueueName);

            basicService.ExposedChannel.BasicPublish(ExchangeName, "", true, null, Encoding.UTF8.GetBytes(TestMessages.Redeliver));
            await Task.Delay(RoundTripWaitTime);

            var inbetweenConsumerCount = basicService.ExposedChannel.ConsumerCount(QueueName);

            await Task.Delay(1000 + RoundTripWaitTime);

            var finalConsumerCount = basicService.ExposedChannel.ConsumerCount(QueueName);

            queue.Dispose();

            successCount.Should().Be(1, "because item is not excepted twice");
            processCount.Should().Be(2, "because first time exception, second time succesful process");
            initialConsumerCount.Should().Be(1);
            inbetweenConsumerCount.Should().Be(1, "because only the message is nacked");
            errorCount.Should().Be(1, "because error should be thrown first time");
            finalConsumerCount.Should().Be(1);
        }
Example #9
0
        public RequestConfiguration GetConfiguration(Type requestType, Type responseType, Action <IRequestConfigurationBuilder> configuration)
        {
            // leverage direct reply to: https://www.rabbitmq.com/direct-reply-to.html
            var replyQueueConfig = new QueueConfiguration
            {
                QueueName  = _directReplyTo,
                AutoDelete = true,
                Durable    = false,
                Exclusive  = true
            };

            var exchangeConfig = new ExchangeConfiguration(_clientConfig.Exchange)
            {
                ExchangeName = _conventions.ExchangeNamingConvention(requestType)
            };

            var defaultConfig = new RequestConfiguration
            {
                ReplyQueue           = replyQueueConfig,
                Exchange             = exchangeConfig,
                RoutingKey           = _conventions.QueueNamingConvention(requestType),
                ReplyQueueRoutingKey = replyQueueConfig.QueueName
            };

            var builder = new RequestConfigurationBuilder(defaultConfig);

            configuration?.Invoke(builder);
            return(builder.Configuration);
        }
Example #10
0
        public async Task ReleaseHeldMessages()
        {
            var queueConfig = new QueueConfiguration
            {
                QueueName              = QueueName,
                PrefetchCount          = 1,
                OnErrorAction          = QueueConfiguration.ErrorAction.RestartConnection,
                ConsumerTag            = "consumer",
                RetryIntervalInSeconds = 10
            };
            var queue = new QueueService(ExposedRabbitService.validConfig, subscriberlogger.Object);

            queue.Start(queueConfig, handler);
            basicService.ExposedChannel.BasicPublish(ExchangeName, "", true, null, Encoding.UTF8.GetBytes(TestMessages.Exception));
            await Task.Delay(RoundTripWaitTime);

            var freeMessages = basicService.ExposedChannel.MessageCount(QueueName);

            queue.Dispose();
            var finalMessageCount = basicService.ExposedChannel.MessageCount(QueueName);

            processCount.Should().Be(1, "because 1 messages were sent");
            successCount.Should().Be(0, "because 0 successful messages were sent");
            errorCount.Should().Be(1);
            freeMessages.Should().Be(1, "because excepted messages should be released");
            finalMessageCount.Should().Be(1, "because only one message caused exception");
        }
Example #11
0
        public async Task DropMessage()
        {
            var queueConfig = new QueueConfiguration
            {
                QueueName              = QueueName,
                PrefetchCount          = 1,
                OnErrorAction          = QueueConfiguration.ErrorAction.DropMessage,
                ConsumerTag            = "consumer",
                RetryIntervalInSeconds = 1
            };
            var queue = new QueueService(ExposedRabbitService.validConfig, subscriberlogger.Object);

            queue.Start(queueConfig, handler);

            var initialConsumerCount = basicService.ExposedChannel.ConsumerCount(QueueName);

            basicService.ExposedChannel.BasicPublish(ExchangeName, "", true, null, Encoding.UTF8.GetBytes(TestMessages.Redeliver));
            await Task.Delay(RoundTripWaitTime);

            var finalConsumerCount = basicService.ExposedChannel.ConsumerCount(QueueName);

            queue.Dispose();

            processCount.Should().Be(1, "because the message is dropped");
            initialConsumerCount.Should().Be(1);
            errorCount.Should().Be(1);
            finalConsumerCount.Should().Be(1, "because consumer shouldn't have stopped");
        }
Example #12
0
        public LightningQueue(int port, bool persistent, LightningQueueSettings settings)
        {
            Port       = port;
            Persistent = persistent;
            var queueConfiguration = new QueueConfiguration()
                                     .ReceiveMessagesAt(new IPEndPoint(IPAddress.Any, port))
                                     .ScheduleQueueWith(TaskPoolScheduler.Default);

            // TODO -- bring through whatever the Jasper logging abstraction ends up being here.
            //.LogWith(new FubuLoggingAdapter(_logger));

            if (persistent)
            {
                queueConfiguration.StoreWithLmdb(settings.QueuePath + "." + port, new EnvironmentConfiguration
                {
                    MaxDatabases = settings.MaxDatabases, MapSize = settings.MapSize
                });
            }
            else
            {
                queueConfiguration.UseNoStorage();
            }

            _queue = queueConfiguration.BuildQueue();
        }
Example #13
0
        public void can_start_two_instances_for_IIS_stop_and_start()
        {
            //This shows that the port doesn't have an exclusive lock, and that lmdb itself can have multiple instances
            var path  = _testDirectory.CreateNewDirectoryForTest();
            var store = new LmdbMessageStore(path);
            var queueConfiguration = new QueueConfiguration();

            queueConfiguration.LogWith(new RecordingLogger());
            queueConfiguration.AutomaticEndpoint();
            queueConfiguration.StoreMessagesWith(store);
            var queue  = queueConfiguration.BuildQueue();
            var queue2 = queueConfiguration.BuildQueue();

            using (queue)
                using (queue2)
                {
                    queue.CreateQueue("test");
                    queue.Start();
                    queue2.CreateQueue("test");
                    queue2.Start();
                    using (queue.Receive("test").Subscribe(x => { }))
                        using (queue2.Receive("test").Subscribe(x => { }))
                        {
                        }
                }
        }
 /// <summary>
 /// Create a new server configuration on a rabbitMQ server.
 /// </summary>
 /// <param name="emiter">Id/Name of application that is using the bus</param>
 /// <param name="connectionFactory">Configured connection factory</param>
 /// <param name="queueConfiguration">Queue configuration.</param>
 public RabbitMQServerConfiguration(string emiter,
                                    ConnectionFactory connectionFactory,
                                    QueueConfiguration queueConfiguration)
     : base(emiter, connectionFactory, null, null)
 {
     QueueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration));
 }
Example #15
0
        public DistributorModule(
            AsyncTasksConfiguration asyncPing,
            AsyncTasksConfiguration asyncCheck,
            DistributorHashConfiguration configuration,
            QueueConfiguration queueConfiguration,
            DistributorNetModule distributorNet,
            ServerId localfordb,
            ServerId localforproxy,
            HashMapConfiguration hashMapConfiguration)
        {
            Contract.Requires(configuration != null);
            Contract.Requires(queueConfiguration != null);
            Contract.Requires(distributorNet != null);
            Contract.Requires(localfordb != null);
            Contract.Requires(localforproxy != null);
            Contract.Requires(asyncPing != null);
            _asyncPing       = asyncPing;
            _asyncTaskModule = new AsyncTaskModule(queueConfiguration);

            _queueConfiguration         = queueConfiguration;
            _modelOfDbWriters           = new WriterSystemModel(configuration, hashMapConfiguration);
            _modelOfAnotherDistributors = new DistributorSystemModel();
            _distributorNet             = distributorNet;
            _localfordb    = localfordb;
            _localforproxy = localforproxy;
            _asyncCheck    = asyncCheck;
            _queue         = GlobalQueue.Queue;
        }
Example #16
0
        public void Build(int storageServer, string hashFile, int countReplics)
        {
            Q = new GlobalQueueInner();
            GlobalQueue.SetQueue(Q);

            var queueConfiguration   = new QueueConfiguration(1, 1000);
            var hashMapConfiguration = new HashMapConfiguration(hashFile,
                                                                HashMapCreationMode.ReadFromFile, 1, countReplics, HashFileType.Writer);
            var local = new ServerId("localhost", storageServer);

            _net = new WriterNetModule(new ConnectionConfiguration("testService", 10),
                                       new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));

            Db = new DbModuleCollection();
            Db.AddDbModule(new TestDbInMemory());

            _async  = new AsyncTaskModule(new QueueConfiguration(1, 10));
            Restore = new AsyncDbWorkModule(_net, _async, Db,
                                            new RestoreModuleConfiguration(10, TimeSpan.FromMinutes(100)),
                                            new RestoreModuleConfiguration(10, TimeSpan.FromMilliseconds(100)),
                                            new RestoreModuleConfiguration(-1, TimeSpan.FromHours(1), false, TimeSpan.FromHours(1)),
                                            new QueueConfiguration(1, 100), local);

            Distributor = new DistributorModule(_async, Restore, _net, local,
                                                hashMapConfiguration, new QueueConfiguration(2, 10), Db);
            _mainС = new MainLogicModule(Distributor, Db);
            Input  = new InputModule(_mainС, queueConfiguration);
            _netRc = new NetWriterReceiver(Input, Distributor,
                                           new NetReceiverConfiguration(storageServer, "localhost", "testService"),
                                           new NetReceiverConfiguration(1, "fake", "fake"));
        }
Example #17
0
        public virtual void PushMessage(Message message, QueueConfiguration queueConfiguration)
        {
            string msgJson = JsonConvert.SerializeObject(message);
            var    body    = Encoding.UTF8.GetBytes(msgJson);
            var    channel = _RabbitMQPersistent.Get();

            try
            {
                IBasicProperties properties = channel.CreateBasicProperties();
                var headers = new Dictionary <string, object>();
                properties.Persistent   = true;
                properties.DeliveryMode = 2;
                headers.Add("x-delay", queueConfiguration.Dealy);
                properties.Headers = headers;

                channel.BasicPublish(exchange: queueConfiguration.ExhangeName,
                                     routingKey: queueConfiguration.QueueName,
                                     basicProperties: properties,
                                     body: body);

                _logger.LogPublishing(message);
            }
            finally
            {
                _RabbitMQPersistent.Return(channel);
            }
        }
Example #18
0
        public void Build(int countReplics, int distrServer1, int distrServer12, string hashFile)
        {
            _q = new GlobalQueueInner();
            GlobalQueue.SetQueue(_q);

            var connection = new ConnectionConfiguration("testService", 10);

            var distrconfig = new DistributorHashConfiguration(countReplics);
            var queueconfig = new QueueConfiguration(1, 100);

            _dnet = new DistributorNetModule(connection,
                                             new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
            Distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
                                                new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
                                                queueconfig, _dnet,
                                                new ServerId("localhost", distrServer1),
                                                new ServerId("localhost", distrServer12),
                                                new HashMapConfiguration(hashFile,
                                                                         HashMapCreationMode.ReadFromFile,
                                                                         1, countReplics, HashFileType.Distributor));

            _dnet.SetDistributor(Distributor);

            _tranc = new TransactionModule(new QueueConfiguration(1, 1000), _dnet, new TransactionConfiguration(4),
                                           distrconfig);
            Main =
                new MainLogicModule(new DistributorTimeoutCache(TimeSpan.FromSeconds(200), TimeSpan.FromSeconds(200)),
                                    Distributor, _tranc);

            var netReceive1 = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
            var netReceive2 = new NetReceiverConfiguration(distrServer12, "localhost", "testService");

            Input     = new InputModuleWithParallel(new QueueConfiguration(2, 100000), Main, _tranc);
            _receiver = new NetDistributorReceiver(Main, Input, Distributor, netReceive1, netReceive2);
        }
Example #19
0
        public DistributorModule(AsyncTaskModule async, AsyncDbWorkModule asyncDbWork,
                                 WriterNetModule writerNet,
                                 ServerId local,
                                 HashMapConfiguration hashMapConfiguration,
                                 QueueConfiguration configuration,
                                 DbModuleCollection dbModuleCollection,
                                 AsyncTasksConfiguration pingConfiguration = null)
        {
            Contract.Requires(local != null);
            Contract.Requires(writerNet != null);
            Contract.Requires(configuration != null);
            Contract.Requires(asyncDbWork != null);
            Contract.Requires(async != null);
            Contract.Assert(dbModuleCollection != null);

            _async              = async;
            _asyncDbWork        = asyncDbWork;
            _model              = new WriterModel(local, hashMapConfiguration);
            _writerNet          = writerNet;
            _queueConfiguration = configuration;
            _dbModuleCollection = dbModuleCollection;
            _queue              = GlobalQueue.Queue;

            var ping = TimeSpan.FromMinutes(1);

            if (pingConfiguration != null)
            {
                ping = pingConfiguration.TimeoutPeriod;
            }

            _async.AddAsyncTask(
                new AsyncDataPeriod(ping, Ping, AsyncTasksNames.AsyncPing, -1), false);
        }
        private static void RegisterServiceBusConfig(
            ContainerBuilder containerBuilder,
            IConfigurationHelper configHelper)
        {
            var serviceBusOptions =
                configHelper.GetSectionValues <ServiceBusOptions>("ServiceBusSettings");

            containerBuilder.RegisterInstance(serviceBusOptions).As <ServiceBusOptions>().SingleInstance();

            var topicConfig = new ServiceBusTopicConfig(
                serviceBusOptions.ServiceBusConnectionString,
                serviceBusOptions.TopicName,
                serviceBusOptions.SubscriptionName,
                Environment.ProcessorCount,
                TimeSpan.FromMinutes(30));

            containerBuilder.Register(c =>
            {
                var topicSubscriptionSevice =
                    new TopicSubscriptionSevice <JobContextDto>(
                        topicConfig,
                        c.Resolve <IJsonSerializationService>(),
                        c.Resolve <ILogger>());
                return(topicSubscriptionSevice);
            }).As <ITopicSubscriptionService <JobContextDto> >();

            containerBuilder.Register(c =>
            {
                var topicPublishSevice =
                    new TopicPublishService <JobContextDto>(
                        topicConfig,
                        c.Resolve <IJsonSerializationService>());
                return(topicPublishSevice);
            }).As <ITopicPublishService <JobContextDto> >();

            containerBuilder.Register(c =>
            {
                var config = new QueueConfiguration(
                    serviceBusOptions.ServiceBusConnectionString,
                    serviceBusOptions.AuditQueueName,
                    1);

                return(new QueuePublishService <AuditingDto>(
                           config,
                           c.Resolve <IJsonSerializationService>()));
            }).As <IQueuePublishService <AuditingDto> >();

            containerBuilder.Register(c =>
            {
                var config = new QueueConfiguration(
                    serviceBusOptions.ServiceBusConnectionString,
                    serviceBusOptions.JobStatusQueueName,
                    1);

                return(new QueuePublishService <JobStatusDto>(
                           config,
                           c.Resolve <IJsonSerializationService>()));
            }).As <IQueuePublishService <JobStatusDto> >();
        }
Example #21
0
 public TimeoutReaderFull(Func <MetaData, bool> isMine, Action <InnerData> process,
                          QueueConfiguration queueConfiguration,
                          DbModuleCollection db, bool isBothTables, QueueWithParam <InnerData> queue)
     : base(process, queueConfiguration, isBothTables, queue)
 {
     _isMine = isMine;
     _db     = db;
 }
Example #22
0
 public InputModule(MainLogicModule mainLogic, QueueConfiguration queueConfiguration)
 {
     Contract.Requires(queueConfiguration != null);
     Contract.Requires(mainLogic != null);
     _queueConfiguration = queueConfiguration;
     _mainLogicModule    = mainLogic;
     _queue = GlobalQueue.Queue;
 }
Example #23
0
 public AsyncTaskModule(QueueConfiguration configuration)
 {
     _tasks      = new List <AsyncData>();
     _threadPool = new DynamicThreadPool(1, configuration.ProcessotCount, configuration.MaxSizeQueue, "AsyncTaskModule");
     _lock       = new ReaderWriterLockSlim();
     _event      = new AutoResetEvent(false);
     _token      = new CancellationTokenSource();
 }
Example #24
0
 public InputModuleWithParallel(QueueConfiguration configuration, MainLogicModule main, TransactionModule transactionModule)
     : base(configuration)
 {
     Contract.Requires(main != null);
     Contract.Requires(transactionModule != null);
     _main = main;
     _transactionModule = transactionModule;
 }
Example #25
0
        public IQueueConsumer GetConsumer(QueueConfiguration configuration)
        {
            string queueName = configuration.Name;

            return(_consumers.GetOrAdd(
                       queueName,
                       _ => new QueueConsumer(() => RemoveConsumer(queueName))));
        }
Example #26
0
 internal ReactiveMQPublisher(QueueConfiguration config)
 {
     this.config       = config;
     connectionFactory = new ConnectionFactory
     {
         Uri = config.ConnectionString
     };
 }
Example #27
0
        public void MainLogicModule_TransactionAnswerResult_ReceiveAnswersFromWriter()
        {
            const int distrServer1 = 22168;
            const int distrServer2 = 23168;

            #region hell

            var connectionConf = new ConnectionConfiguration("testService", 10);

            var distrconfig = new DistributorHashConfiguration(1);
            var queueconfig = new QueueConfiguration(1, 100);
            var dnet        = new DistributorNetModule(connectionConf,
                                                       new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
            var ddistributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
                                                     new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
                                                     queueconfig, dnet,
                                                     new ServerId("localhost", distrServer1),
                                                     new ServerId("localhost", distrServer2),
                                                     new HashMapConfiguration("TestDistributorReceiveAndDbSendAsync",
                                                                              HashMapCreationMode.ReadFromFile,
                                                                              1, 1, HashFileType.Distributor));
            dnet.SetDistributor(ddistributor);

            var tranc = new TransactionModule(new QueueConfiguration(1, 1000), dnet, new TransactionConfiguration(4),
                                              distrconfig);
            var cache = new DistributorTimeoutCache(TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(200000));
            var main  = new MainLogicModule(cache, ddistributor, tranc);
            cache.SetMainLogicModule(main);

            var netReceive4  = new NetReceiverConfiguration(distrServer1, "localhost", "testService");
            var netReceive42 = new NetReceiverConfiguration(distrServer2, "localhost", "testService");
            var input        = new InputModuleWithParallel(new QueueConfiguration(2, 100000), main, tranc);
            var receiver4    = new NetDistributorReceiver(main, input,
                                                          ddistributor, netReceive4, netReceive42);

            ddistributor.Start();
            receiver4.Start();

            #endregion

            var t = 0;
            GlobalQueue.Queue.TransactionQueue.Registrate(data => Interlocked.Increment(ref t));
            GlobalQueue.Queue.Start();

            var connection = new SingleConnectionToDistributor(
                new ServerId("localhost", distrServer1), new ConnectionConfiguration("testService", 10),
                new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));
            connection.Connect();

            connection.TransactionAnswerResult(new Transaction("123", "123"));
            Thread.Sleep(TimeSpan.FromMilliseconds(100));
            connection.TransactionAnswerResult(new Transaction("1243", "1423"));
            Thread.Sleep(TimeSpan.FromMilliseconds(100));
            Assert.AreEqual(2, t);

            connection.Dispose();
            receiver4.Dispose();
        }
Example #28
0
        public void TransactionModule_ProcessSyncWithExecutor_RollbackNoEnoughServers()
        {
            var server1 = new ServerId("localhost", 21141);
            var server2 = new ServerId("localhost", 21142);
            var server3 = new ServerId("localhost", 21143);

            var netconfig   = new ConnectionConfiguration("testService", 10);
            var queueconfig = new QueueConfiguration(1, 100);
            var distrconfig = new DistributorHashConfiguration(2);
            var distributor = new DistributorModule(new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)),
                                                    new AsyncTasksConfiguration(TimeSpan.FromMinutes(5)), distrconfig,
                                                    queueconfig, null, new ServerId("localhost", 1),
                                                    new ServerId("localhost", 1),
                                                    new HashMapConfiguration("test10", HashMapCreationMode.CreateNew, 1, 1, HashFileType.Distributor));

            var net = new DistributorNetModule(netconfig,
                                               new ConnectionTimeoutConfiguration(Consts.OpenTimeout, Consts.SendTimeout));

            net.SetDistributor(distributor);
            distributor.Start();
            net.Start();
            GlobalQueue.Queue.Start();

            var s1 = TestHelper.OpenWriterHost(server1, netconfig);
            var s2 = TestHelper.OpenWriterHost(server2, netconfig);

            net.ConnectToWriter(server1);
            net.ConnectToWriter(server2);

            Thread.Sleep(TimeSpan.FromMilliseconds(1000));

            var ev = new InnerData(new Transaction("", ""))
            {
                Transaction = { Destination = new List <ServerId> {
                                    server1, server2, server3
                                } }
            };

            var trm = new TransactionModule(new QueueConfiguration(1, 1000), net, new TransactionConfiguration(1),
                                            new DistributorHashConfiguration(3));

            trm.Start();

            using (var trans = trm.Rent())
            {
                trm.ProcessSyncWithExecutor(ev, trans.Element);
            }

            Thread.Sleep(TimeSpan.FromMilliseconds(100));

            Assert.IsTrue(s1.Value <= 0);
            Assert.IsTrue(s2.Value <= 0);
            Assert.IsTrue(ev.Transaction.IsError);

            net.Dispose();
            trm.Dispose();
        }
 /// <summary>
 /// Create a new server configuration on a rabbitMQ server.
 /// </summary>
 /// <param name="emiter">Id/Name of application that is using the bus</param>
 /// <param name="host">The host to connect to.</param>
 /// <param name="userName">The username to use.</param>
 /// <param name="password">The password to use.</param>
 /// <param name="queueConfiguration">Queue configuration.</param>
 public RabbitMQServerConfiguration(string emiter,
                                    string host,
                                    string userName,
                                    string password,
                                    QueueConfiguration queueConfiguration)
     : base(emiter, host, userName, password, null, null)
 {
     QueueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration));
 }
Example #30
0
 protected ParallelWorkModule(QueueConfiguration configuration)
 {
     Contract.Requires(configuration != null);
     _configuration = configuration;
     _token         = new CancellationTokenSource();
     _queue         = new BlockingCollection <T>();
     _threads       = new List <Thread>();
     _workers       = new List <SingleParallelWorkBase <T> >();
 }
        public ConcurrentSqsReceiver(IAmazonSQS amazonSQSClient, QueueConfiguration queueConfiguration, Func<Message, bool> onReceived,
            Action<Exception, Message> onError, int maxNumberOfMessages = 1, int concurrency = 1)
            : base(amazonSQSClient, queueConfiguration, onReceived, onError, maxNumberOfMessages)
        {
            if (concurrency < 1)
            {
                throw new ArgumentException("Concurrency must be greater than 0", "concurrency");
            }

            _activeTasks = new List<Task>();
            _maximumNumberOfTasks = MaxNumberOfMessages*concurrency;
        }
 public SequentialSQSReceiver(IAmazonSQS amazonSQSClient, QueueConfiguration queueConfiguration, Func<Message, bool> onRecieved, Action<Exception, Message> onError, int maxNumberOfMessages = DefaultNumberOfMessagesToReceiveAtATime)
 {
     if (maxNumberOfMessages > 10 | maxNumberOfMessages < 1)
     {
         throw new ArgumentException("MaxNumberOfMessages must be between 1 and 10", "maxNumberOfMessages");
     }
     
     _amazonSQSClient = amazonSQSClient;
     _queueConfiguration = queueConfiguration;
     MaxNumberOfMessages = maxNumberOfMessages;
     _onRecieved = onRecieved;
     _onError = onError;
 }
Example #33
0
        private Queue CreateQueue(int port, bool persist, int mapSize = 1024*1024*100, int maxDatabases = 5)
        {
            var queueConfiguration = new QueueConfiguration()
                .ReceiveMessagesAt(new IPEndPoint(IPAddress.Any, port))
                .ScheduleQueueWith(TaskPoolScheduler.Default)
                .LogWith(new FubuLoggingAdapter(_logger));

            if (persist)
            {
                queueConfiguration.StoreWithLmdb(QueuePath + "." + port, new EnvironmentConfiguration { MaxDatabases = maxDatabases, MapSize = mapSize });
            }
            else
            {
                queueConfiguration.UseNoStorage();
            }
            var queue = queueConfiguration.BuildQueue();
            _queueManagers.Fill(port, queue);
            return queue;
        }
		public QueueConfigurationBuilder(QueueConfiguration initialQueue = null)
		{
			Configuration = initialQueue ?? QueueConfiguration.Default;
		}
 public SQSPublisher(IAmazonSQS amazonSQSClient, QueueConfiguration queueConfiguration)
 {
     _amazonSQSClient = amazonSQSClient;
     _queueConfiguration = queueConfiguration;
 }