예제 #1
0
        public IKafkaMessageFlowReceiver Create(IMessageFlow messageFlow)
        {
            var settings          = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
            var stateInitSettings = new StateInitKafkaMessageFlowReceiverSettings(settings);

            return(new KafkaMessageFlowReceiver(stateInitSettings, _tracer));
        }
        public IServiceBusMessageReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            if (messageFlow.Id == ImportFactsFromErmFlow.Instance.Id)
                return new Settings
                {
                    ConnectionString = _serviceBusConnectionString,
                    TransportEntityPath = _ermOperationsFlowTopic.Value,
                };

            if (messageFlow.Id == CommonEventsFlow.Instance.Id)
                return new Settings
                {
                    ConnectionString = _serviceBusConnectionString,
                    TransportEntityPath = _commonEventsFlowTopic.Value,
                };

            if (messageFlow.Id == StatisticsEventsFlow.Instance.Id)
                return new Settings
                {
                    ConnectionString = _serviceBusConnectionString,
                    TransportEntityPath = _statisticsEventsFlowTopic.Value,
                };

            throw new ArgumentException($"Flow '{messageFlow.Description}' settings for MS ServiceBus are undefined");
        }
예제 #3
0
        private IEventLoggingStrategy <TEvent> ResolveServiceBusStrategy <TEvent>(IMessageFlow flow)
        {
            var serviceBusSettings = _settingsFactory.CreateSenderSettings(flow);
            var strategy           = _unityContainer.Resolve <SessionlessServiceBusEventLoggingStrategy <TEvent> >(new DependencyOverride <IServiceBusMessageSenderSettings>(serviceBusSettings));

            return(strategy);
        }
예제 #4
0
        public IServiceBusMessageReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            if (ErmFactsFlow.Instance.Equals(messageFlow))
            {
                return new Settings
                       {
                           ConnectionString    = _serviceBusConnectionString,
                           TransportEntityPath = _ermOperationsFlowTopic.Value,
                       }
            }
            ;

            if (AggregatesFlow.Instance.Equals(messageFlow))
            {
                return new Settings
                       {
                           ConnectionString    = _serviceBusConnectionString,
                           TransportEntityPath = _commonEventsFlowTopic.Value,
                       }
            }
            ;

            if (MessagesFlow.Instance.Equals(messageFlow))
            {
                return new Settings
                       {
                           ConnectionString    = _serviceBusConnectionString,
                           TransportEntityPath = _mesageEventsFlowTopic.Value,
                       }
            }
            ;

            throw new ArgumentException($"Flow '{messageFlow.Description}' settings for MS ServiceBus are undefined");
        }
        private void LoadDataFromKafka2Db(IMessageFlow messageFlow,
                                          IReadOnlyCollection <Type> dataObjectTypes,
                                          DataConnection dataConnection,
                                          int batchSize,
                                          int bulkReplaceCommandTimeoutSec)
        {
            var actors = CreateActors(dataObjectTypes,
                                      dataConnection,
                                      new BulkCopyOptions
            {
                BulkCopyTimeout = bulkReplaceCommandTimeoutSec
            });

            var initialStats = _kafkaMessageFlowInfoProvider.GetFlowStats(messageFlow)
                               .ToDictionary(x => x.TopicPartition, x => new MessageFlowStats(x.TopicPartition, x.End, Offset.Unset));

            using var receiver = _receiverFactory.Create(messageFlow);

            while (true)
            {
                var batch = receiver.ReceiveBatch(batchSize);

                var bulkCommands = _commandFactory.CreateCommands(batch);
                if (bulkCommands.Count > 0)
                {
                    using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
                    foreach (var actor in actors)
                    {
                        actor.ExecuteCommands(bulkCommands);
                    }
                    scope.Complete();
                }

                receiver.CompleteBatch(batch);

                var batchStats = batch
                                 .GroupBy(x => x.TopicPartition)
                                 .ToDictionary(x => x.Key, x => x.Max(y => y.Offset.Value));
                foreach (var batchStat in batchStats)
                {
                    if (!initialStats.TryGetValue(batchStat.Key, out var initialStat))
                    {
                        throw new KeyNotFoundException(batchStat.Key.ToString());
                    }

                    var currentStat = new MessageFlowStats(batchStat.Key, initialStat.End, batchStat.Value + 1);
                    _tracer.Info($"Topic {currentStat.TopicPartition}, End: {currentStat.End}, Offset: {currentStat.Offset}, Lag: {currentStat.Lag}");

                    initialStats[batchStat.Key] = currentStat;
                }

                var complete = initialStats.Values.All(x => x.Lag <= 0);
                if (complete)
                {
                    _tracer.Info($"Kafka state init for flow {messageFlow.Description} complete");
                    break;
                }
            }
        }
예제 #6
0
        public IKafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var settings))
            {
                throw new ArgumentOutOfRangeException($"Can't create kafka info settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config");
            }

            return(settings);
        }
        private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic,
                                          IReadOnlyCollection <Type> dataObjectTypes,
                                          DataConnection dataConnection,
                                          int batchSize,
                                          int bulkReplaceCommandTimeoutSec)
        {
            var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name;

            var actors = CreateActors(dataObjectTypes,
                                      dataConnection,
                                      new BulkCopyOptions
            {
                BulkCopyTimeout = bulkReplaceCommandTimeoutSec
            });

            using var receiver = _receiverFactory.Create(messageFlowForKafkaTopic);
            // retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86
            var lastTargetMessageOffset =
                Policy.Handle <KafkaException>(exception => exception.Error.Code == ErrorCode.LeaderNotAvailable)
                .WaitAndRetryForever(i => TimeSpan.FromSeconds(5),
                                     (exception, waitSpan) =>
                                     _tracer.Warn(exception,
                                                  $"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}"))
                .ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1)
                .Result;

            _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}");

            var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic))
                                           .ToList();

            for (var distance = lastTargetMessageOffset; distance > 0;)
            {
                var batch = receiver.ReceiveBatch(batchSize);

                var lastMessageOffset = batch.Last().Offset.Value;
                distance = lastTargetMessageOffset - lastMessageOffset;

                _tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {lastMessageOffset}. Target and current offsets distance: {distance}");

                var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch)).ToList();
                if (bulkCommands.Count > 0)
                {
                    using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
                    foreach (var actor in actors)
                    {
                        actor.ExecuteCommands(bulkCommands);
                    }
                    scope.Complete();
                }

                receiver.CompleteBatch(batch);
            }

            _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished");
        }
        public long GetFlowSize(IMessageFlow messageFlow)
        {
            var settings = _kafkaSettingsFactory.CreateInfoSettings(messageFlow);

            using (var consumer = new Consumer(settings.Config))
            {
                var offsets = consumer.QueryWatermarkOffsets(settings.TopicPartition, settings.InfoTimeout);
                return(offsets.High);
            }
        }
        public long GetFlowProcessedSize(IMessageFlow messageFlow)
        {
            var settings = _kafkaSettingsFactory.CreateInfoSettings(messageFlow);

            using (var consumer = new Consumer(settings.Config))
            {
                var committedOffset = consumer.Committed(new[] { settings.TopicPartition }, settings.InfoTimeout).First();
                return(committedOffset.Offset);
            }
        }
예제 #10
0
        public long GetFlowSize(IMessageFlow messageFlow)
        {
            var settings       = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
            var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);

            using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
            var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout);

            return(offsets.High);
        }
예제 #11
0
        public long GetFlowProcessedSize(IMessageFlow messageFlow)
        {
            var settings       = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
            var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);

            using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
            var committedOffset = consumer.Committed(new[] { topicPartition }, settings.PollTimeout).First();

            return(committedOffset.Offset);
        }
예제 #12
0
        public ConsumeResult <Ignore, Ignore> TryGetFlowLastMessage(IMessageFlow messageFlow)
        {
            var settings       = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
            var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);

            using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
            var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout);

            consumer.Assign(new[] { new TopicPartitionOffset(topicPartition, offsets.High - 1) });

            return(consumer.Consume(settings.PollTimeout));
        }
        public bool TryGetFlowLastMessage(IMessageFlow messageFlow, out Message message)
        {
            var settings = _kafkaSettingsFactory.CreateInfoSettings(messageFlow);

            using (var consumer = new Consumer(settings.Config))
            {
                var offsets = consumer.QueryWatermarkOffsets(settings.TopicPartition, settings.InfoTimeout);
                consumer.Assign(new[] { new TopicPartitionOffset(settings.TopicPartition, offsets.High - 1) });

                return(consumer.Consume(out message, settings.InfoTimeout));
            }
        }
예제 #14
0
        public IServiceBusMessageReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            if (ErmFactsFlow.Instance.Equals(messageFlow))
            {
                return new Settings
                       {
                           ConnectionString    = ServiceBusConnectionString,
                           TransportEntityPath = ErmFactsTopic.Value,
                       }
            }
            ;

            throw new ArgumentException($"Flow '{messageFlow.Description}' settings for MS ServiceBus are undefined");
        }
        public IKafkaMessageFlowInfoSettings CreateInfoSettings(IMessageFlow messageFlow)
        {
            if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var kafkaConfig))
            {
                throw new ArgumentOutOfRangeException($"Can't create kafka info settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config");
            }

            return(new KafkaMessageFlowInfoSettings
            {
                Config = kafkaConfig.KafkaClientSpecific,
                TopicPartition = new TopicPartition(kafkaConfig.Topic, SingleSupportedPartition),
                InfoTimeout = kafkaConfig.InfoTimeout
            });
        }
        public IKafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var kafkaConfig))
            {
                throw new ArgumentOutOfRangeException($"Can't create kafka receiver settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config");
            }

            return(new KafkaMessageFlowReceiverSettings
            {
                Config = kafkaConfig.KafkaClientSpecific,
                TopicPartitionOffsets = new [] { new TopicPartitionOffset(kafkaConfig.Topic, SingleSupportedPartition, _offset) },
                PollTimeout = kafkaConfig.PoolTimeout
            });
        }
예제 #17
0
        public IReadOnlyCollection <MessageFlowStats> GetFlowStats(IMessageFlow messageFlow)
        {
            var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);

            using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();

            var topicPartitions = GetTopicPartitions(settings.TopicPartitionOffsets.Select(x => x.Topic));
            var stats           = consumer.Committed(topicPartitions, settings.PollTimeout).Select(x =>
            {
                var offsets = consumer.QueryWatermarkOffsets(x.TopicPartition, settings.PollTimeout);
                return(new MessageFlowStats(x.TopicPartition, offsets.High, x.Offset));
            }).ToList();

            return(stats);
        }
예제 #18
0
        public KafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            var settings = _wrap.CreateReceiverSettings(messageFlow);

            // stateinit всегда начинает читать все partitions с Offset.Beginning
            settings.TopicPartitionOffsets = settings.TopicPartitionOffsets
                                             .Select(x => new TopicPartitionOffset(x.Topic, Partition.Any, Offset.Beginning));

            // хак для InfoRussia, интересные нам данные начинаются только с определённого offset
            if (messageFlow.Equals(InfoRussiaFactsFlow.Instance))
            {
                const long InfoRussiaOffset = 7_000_000;

                settings.TopicPartitionOffsets = settings.TopicPartitionOffsets.Select(x => new
                                                                                       TopicPartitionOffset(x.Topic, x.Partition, InfoRussiaOffset));
            }

            return(settings);
        }
예제 #19
0
        public KafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow)
        {
            var topics = new List <string>();

            if (messageFlow.Equals(KafkaFactsFlow.Instance))
            {
                topics.Add(ConfigFileSetting.String.Required("AmsFactsTopic").Value);
                topics.Add(ConfigFileSetting.String.Required("RulesetFactsTopic").Value);
                topics.Add(ConfigFileSetting.String.Required("InfoRussiaFactsTopic").Value);
                topics.Add(ConfigFileSetting.String.Required("FijiFactsTopic").Value);
            }
            else if (messageFlow.Equals(AmsFactsFlow.Instance))
            {
                topics.Add(ConfigFileSetting.String.Required("AmsFactsTopic").Value);
            }
            else if (messageFlow.Equals(RulesetFactsFlow.Instance))
            {
                topics.Add(ConfigFileSetting.String.Required("RulesetFactsTopic").Value);
            }
            else if (messageFlow.Equals(InfoRussiaFactsFlow.Instance))
            {
                topics.Add(ConfigFileSetting.String.Required("InfoRussiaFactsTopic").Value);
            }
            else if (messageFlow.Equals(FijiFactsFlow.Instance))
            {
                topics.Add(ConfigFileSetting.String.Required("FijiFactsTopic").Value);
            }

            if (topics.Count == 0)
            {
                throw new ArgumentException($"Unknown message flows provided");
            }

            return(new KafkaMessageFlowReceiverSettings
            {
                Config = _kafkaConfig,
                TopicPartitionOffsets = topics.Select(x => new TopicPartitionOffset(x, Partition.Any, Offset.Unset))
            });
        }
 public IEnumerable <IMessageFlow> GetConsumableFlows(IMessageFlow flow) =>
 flow switch
 {
예제 #21
0
        public IServiceBusMessageFlowReceiver Create(IMessageFlow messageFlow)
        {
            var settings = _settingsFactory.CreateReceiverSettings(messageFlow);

            return(new ServiceBusMessageFlowReceiver(_tracer, settings, _renewer, messageFlow));
        }
        private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic,
                                          IReadOnlyCollection <Type> dataObjectTypes,
                                          DataConnection dataConnection,
                                          int bulkReplaceCommandTimeoutSec)
        {
            var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name;

            var actors = CreateActors(dataObjectTypes,
                                      dataConnection,
                                      new BulkCopyOptions
            {
                BulkCopyTimeout = bulkReplaceCommandTimeoutSec
            });

            using (var receiver = _receiverFactory.Create(messageFlowForKafkaTopic))
            {
                // retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86
                var lastTargetMessageOffset =
                    Policy.Handle <Confluent.Kafka.KafkaException>(exception => exception.Error.Code == Confluent.Kafka.ErrorCode.LeaderNotAvailable)
                    .WaitAndRetryForever(i => TimeSpan.FromSeconds(5),
                                         (exception, waitSpan) =>
                                         _tracer.Warn(exception,
                                                      $"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}"))
                    .ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1)
                    .Result;

                _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}");

                var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic))
                                               .ToList();

                using (var transation = new TransactionScope(TransactionScopeOption.RequiresNew,
                                                             new TransactionOptions
                {
                    IsolationLevel = IsolationLevel.Serializable,
                    Timeout = TimeSpan.Zero
                }))
                {
                    long currentMessageOffset     = 0;
                    int  receivedMessagesQuantity = 0;
                    while (currentMessageOffset < lastTargetMessageOffset)
                    {
                        var batch = receiver.ReceiveBatch(_batchSizeSettings.BatchSize);
                        // крутим цикл пока не получим сообщения от kafka,
                        // т.к. у клиента kafka есть некоторое время прогрева, то после запуска некоторое время могут возвращаться пустые batch,
                        // несмотря на фактическое наличие сообщений в topic\partition
                        if (batch.Count == 0)
                        {
                            continue;
                        }

                        receivedMessagesQuantity += batch.Count;
                        currentMessageOffset      = batch.Last().Offset.Value;

                        _tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {currentMessageOffset}. Target and current offsets distance: {lastTargetMessageOffset - currentMessageOffset}");

                        var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch))
                                           .ToList();

                        if (bulkCommands.Count > 0)
                        {
                            foreach (var actor in actors)
                            {
                                actor.ExecuteCommands(bulkCommands);
                            }
                        }

                        receiver.CompleteBatch(batch);
                    }

                    _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished. Received messages quantity: {receivedMessagesQuantity}");
                    transation.Complete();
                }
            }
        }
 public FlowEvent(IMessageFlow flow, IEvent @event) =>
 (Flow, Event) = (flow, @event);
 public IServiceBusMessageFlowReceiver Create(IMessageFlow messageFlow)
 {
     var settings = _settingsFactory.CreateReceiverSettings(messageFlow);
     return new ServiceBusMessageFlowReceiver(_tracer, settings, _renewer, messageFlow);
 }
예제 #25
0
 public IServiceBusMessageFlowReceiver Create(IMessageFlow messageFlow)
 {
     return(_container.Resolve <ServiceBusMessageFlowReceiver>(new DependencyOverride(typeof(IMessageFlow), messageFlow)));
 }
 public KafkaReplicationCommand(IMessageFlow messageFlow, ReplicateInBulkCommand replicateInBulkCommand, int batchSize = 5000)
 {
     MessageFlow            = messageFlow;
     ReplicateInBulkCommand = replicateInBulkCommand;
     BatchSize = batchSize;
 }
예제 #27
0
 public FlowEvent(IMessageFlow flow, IEvent @event)
 {
     Flow  = flow;
     Event = @event;
 }
예제 #28
0
        public PerformedOperationFinalProcessing Serialize(AggregateOperation operation, IMessageFlow targetFlow)
        {
            var entityType = _registry.GetEntityName(operation.AggregateType);

            return(new PerformedOperationFinalProcessing
            {
                CreatedOn = DateTime.UtcNow,
                MessageFlowId = targetFlow.Id,
                EntityId = operation.AggregateId,
                EntityTypeId = entityType.Id,
                OperationId = GetIdentity(operation),
            });
        }
예제 #29
0
 public IServiceBusMessageSenderSettings CreateSenderSettings(IMessageFlow messageFlow)
 {
     throw new ArgumentException($"Flow '{messageFlow.Description}' settings for MS ServiceBus are undefined");
 }
        public IKafkaMessageFlowReceiver Create(IMessageFlow messageFlow)
        {
            var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);

            return(new KafkaTopicConsumerWrapper(settings, _tracer));
        }
예제 #31
0
 public KafkaReplicationCommand(IMessageFlow messageFlow, ReplicateInBulkCommand replicateInBulkCommand)
 {
     MessageFlow            = messageFlow;
     ReplicateInBulkCommand = replicateInBulkCommand;
 }
예제 #32
0
 public PerformedOperationFinalProcessing Serialize(RecalculateStatisticsOperation operation, IMessageFlow targetFlow)
 {
     return(new PerformedOperationFinalProcessing
     {
         CreatedOn = DateTime.UtcNow,
         MessageFlowId = targetFlow.Id,
         Context = Serialize(operation).ToString(),
         OperationId = GetIdentity(operation),
     });
 }
 public IServiceBusMessageFlowReceiver Create(IMessageFlow messageFlow)
 {
     return _container.Resolve<ServiceBusMessageFlowReceiver>(new DependencyOverride(typeof(IMessageFlow), messageFlow));
 }