private void LoadDataFromKafka2Db(IMessageFlow messageFlow,
IReadOnlyCollection <Type> dataObjectTypes,
DataConnection dataConnection,
int batchSize,
int bulkReplaceCommandTimeoutSec)
{
var actors = CreateActors(dataObjectTypes,
dataConnection,
new BulkCopyOptions
{
BulkCopyTimeout = bulkReplaceCommandTimeoutSec
});
var initialStats = _kafkaMessageFlowInfoProvider.GetFlowStats(messageFlow)
.ToDictionary(x => x.TopicPartition, x => new MessageFlowStats(x.TopicPartition, x.End, Offset.Unset));
using var receiver = _receiverFactory.Create(messageFlow);
while (true)
{
var batch = receiver.ReceiveBatch(batchSize);
var bulkCommands = _commandFactory.CreateCommands(batch);
if (bulkCommands.Count > 0)
{
using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
foreach (var actor in actors)
{
actor.ExecuteCommands(bulkCommands);
}
scope.Complete();
}
receiver.CompleteBatch(batch);
var batchStats = batch
.GroupBy(x => x.TopicPartition)
.ToDictionary(x => x.Key, x => x.Max(y => y.Offset.Value));
foreach (var batchStat in batchStats)
{
if (!initialStats.TryGetValue(batchStat.Key, out var initialStat))
{
throw new KeyNotFoundException(batchStat.Key.ToString());
}
var currentStat = new MessageFlowStats(batchStat.Key, initialStat.End, batchStat.Value + 1);
_tracer.Info($"Topic {currentStat.TopicPartition}, End: {currentStat.End}, Offset: {currentStat.Offset}, Lag: {currentStat.Lag}");
initialStats[batchStat.Key] = currentStat;
}
var complete = initialStats.Values.All(x => x.Lag <= 0);
if (complete)
{
_tracer.Info($"Kafka state init for flow {messageFlow.Description} complete");
break;
}
}
}
public KafkaReceiver(
MessageFlowMetadata sourceFlowMetadata,
IPerformedOperationsReceiverSettings messageReceiverSettings,
IKafkaMessageFlowReceiverFactory messageFlowReceiverFactory)
: base(sourceFlowMetadata, messageReceiverSettings)
{
_messageFlowReceiver = messageFlowReceiverFactory.Create(SourceFlowMetadata.MessageFlow);
}
private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic,
IReadOnlyCollection <Type> dataObjectTypes,
DataConnection dataConnection,
int batchSize,
int bulkReplaceCommandTimeoutSec)
{
var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name;
var actors = CreateActors(dataObjectTypes,
dataConnection,
new BulkCopyOptions
{
BulkCopyTimeout = bulkReplaceCommandTimeoutSec
});
using var receiver = _receiverFactory.Create(messageFlowForKafkaTopic);
// retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86
var lastTargetMessageOffset =
Policy.Handle <KafkaException>(exception => exception.Error.Code == ErrorCode.LeaderNotAvailable)
.WaitAndRetryForever(i => TimeSpan.FromSeconds(5),
(exception, waitSpan) =>
_tracer.Warn(exception,
$"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}"))
.ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1)
.Result;
_tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}");
var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic))
.ToList();
for (var distance = lastTargetMessageOffset; distance > 0;)
{
var batch = receiver.ReceiveBatch(batchSize);
var lastMessageOffset = batch.Last().Offset.Value;
distance = lastTargetMessageOffset - lastMessageOffset;
_tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {lastMessageOffset}. Target and current offsets distance: {distance}");
var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch)).ToList();
if (bulkCommands.Count > 0)
{
using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
foreach (var actor in actors)
{
actor.ExecuteCommands(bulkCommands);
}
scope.Complete();
}
receiver.CompleteBatch(batch);
}
_tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished");
}
