private void LoadDataFromKafka2Db(IMessageFlow messageFlowForKafkaTopic,
IReadOnlyCollection <Type> dataObjectTypes,
DataConnection dataConnection,
int batchSize,
int bulkReplaceCommandTimeoutSec)
{
var targetMessageFlowDescription = messageFlowForKafkaTopic.GetType().Name;
var actors = CreateActors(dataObjectTypes,
dataConnection,
new BulkCopyOptions
{
BulkCopyTimeout = bulkReplaceCommandTimeoutSec
});
using var receiver = _receiverFactory.Create(messageFlowForKafkaTopic);
// retry добавлен из-за https://github.com/confluentinc/confluent-kafka-dotnet/issues/86
var lastTargetMessageOffset =
Policy.Handle <KafkaException>(exception => exception.Error.Code == ErrorCode.LeaderNotAvailable)
.WaitAndRetryForever(i => TimeSpan.FromSeconds(5),
(exception, waitSpan) =>
_tracer.Warn(exception,
$"Can't get size of kafka topic. Message flow: {targetMessageFlowDescription}. Wait span: {waitSpan}"))
.ExecuteAndCapture(() => _kafkaMessageFlowInfoProvider.GetFlowSize(messageFlowForKafkaTopic) - 1)
.Result;
_tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription}. Last target message offset: {lastTargetMessageOffset}");
var resolvedCommandFactories = _commandFactories.Where(f => f.AppropriateFlows.Contains(messageFlowForKafkaTopic))
.ToList();
for (var distance = lastTargetMessageOffset; distance > 0;)
{
var batch = receiver.ReceiveBatch(batchSize);
var lastMessageOffset = batch.Last().Offset.Value;
distance = lastTargetMessageOffset - lastMessageOffset;
_tracer.Info($"Flow: {targetMessageFlowDescription}. Received messages: {batch.Count}. Last message offset for received batch: {lastMessageOffset}. Target and current offsets distance: {distance}");
var bulkCommands = resolvedCommandFactories.SelectMany(factory => factory.CreateCommands(batch)).ToList();
if (bulkCommands.Count > 0)
{
using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
foreach (var actor in actors)
{
actor.ExecuteCommands(bulkCommands);
}
scope.Complete();
}
receiver.CompleteBatch(batch);
}
_tracer.Info($"Receiving messages from kafka for flow: {targetMessageFlowDescription} finished");
}
public Task <long> WaitForVersion(Guid ermToken)
{
var amsCount = _kafkaMessageFlowInfoProvider.GetFlowSize(AliasForAmsFactsFlow.Instance);
return(WaitForVersion(ermToken, amsCount, WaitInterval, WaitTimeout));
}