private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic, IReadOnlyCollection <Type> dataObjectTypes, DataConnection dataConnection, int batchSize, int bulkReplaceCommandTimeoutSec) { var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name; var actors = CreateActors(dataObjectTypes, dataConnection, new BulkCopyOptions { BulkCopyTimeout = bulkReplaceCommandTimeoutSec }); using var receiver = _receiverFactory.Create(messageFlowForKafkaTopic); // retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86 var lastTargetMessageOffset = Policy.Handle <KafkaException>(exception => exception.Error.Code == ErrorCode.LeaderNotAvailable) .WaitAndRetryForever(i => TimeSpan.FromSeconds(5), (exception, waitSpan) => _tracer.Warn(exception, $"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}")) .ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1) .Result; _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}"); var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic)) .ToList(); for (var distance = lastTargetMessageOffset; distance > 0;) { var batch = receiver.ReceiveBatch(batchSize); var lastMessageOffset = batch.Last().Offset.Value; distance = lastTargetMessageOffset - lastMessageOffset; _tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {lastMessageOffset}. Target and current offsets distance: {distance}"); var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch)).ToList(); if (bulkCommands.Count > 0) { using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero }); foreach (var actor in actors) { actor.ExecuteCommands(bulkCommands); } scope.Complete(); } receiver.CompleteBatch(batch); } _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished"); }
private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic, IReadOnlyCollection <Type> dataObjectTypes, DataConnection dataConnection, int bulkReplaceCommandTimeoutSec) { var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name; var actors = CreateActors(dataObjectTypes, dataConnection, new BulkCopyOptions { BulkCopyTimeout = bulkReplaceCommandTimeoutSec }); using (var receiver = _receiverFactory.Create(messageFlowForKafkaTopic)) { // retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86 var lastTargetMessageOffset = Policy.Handle <Confluent.Kafka.KafkaException>(exception => exception.Error.Code == Confluent.Kafka.ErrorCode.LeaderNotAvailable) .WaitAndRetryForever(i => TimeSpan.FromSeconds(5), (exception, waitSpan) => _tracer.Warn(exception, $"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}")) .ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1) .Result; _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}"); var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic)) .ToList(); using (var transation = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero })) { long currentMessageOffset = 0; int receivedMessagesQuantity = 0; while (currentMessageOffset < lastTargetMessageOffset) { var batch = receiver.ReceiveBatch(_batchSizeSettings.BatchSize); // крутим цикл пока не получим сообщения от kafka, // т.к. у клиента kafka есть некоторое время прогрева, то после запуска некоторое время могут возвращаться пустые batch, // несмотря на фактическое наличие сообщений в topic\partition if (batch.Count == 0) { continue; } receivedMessagesQuantity += batch.Count; currentMessageOffset = batch.Last().Offset.Value; _tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {currentMessageOffset}. Target and current offsets distance: {lastTargetMessageOffset - currentMessageOffset}"); var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch)) .ToList(); if (bulkCommands.Count > 0) { foreach (var actor in actors) { actor.ExecuteCommands(bulkCommands); } } receiver.CompleteBatch(batch); } _tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished. Received messages quantity: {receivedMessagesQuantity}"); transation.Complete(); } } }
public IKafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow) { if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var kafkaConfig)) { throw new ArgumentOutOfRangeException($"Can't create kafka receiver settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config"); } return(new KafkaMessageFlowReceiverSettings { Config = kafkaConfig.KafkaClientSpecific, TopicPartitionOffsets = new [] { new TopicPartitionOffset(kafkaConfig.Topic, SingleSupportedPartition, _offset) }, PollTimeout = kafkaConfig.PoolTimeout }); }
public IKafkaMessageFlowReceiverSettings CreateReceiverSettings(IMessageFlow messageFlow) { if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var settings)) { throw new ArgumentOutOfRangeException($"Can't create kafka info settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config"); } return(settings); }
public IKafkaMessageFlowInfoSettings CreateInfoSettings(IMessageFlow messageFlow) { if (!_flows2ConsumerSettingsMap.TryGetValue(messageFlow, out var kafkaConfig)) { throw new ArgumentOutOfRangeException($"Can't create kafka info settings. Specified message flow \"{messageFlow.GetType().Name}\" doesn't has appropriate config"); } return(new KafkaMessageFlowInfoSettings { Config = kafkaConfig.KafkaClientSpecific, TopicPartition = new TopicPartition(kafkaConfig.Topic, SingleSupportedPartition), InfoTimeout = kafkaConfig.InfoTimeout }); }