private void LoadDataFromKafka2Db(IMessageFlow messageFlow,
IReadOnlyCollection <Type> dataObjectTypes,
DataConnection dataConnection,
int batchSize,
int bulkReplaceCommandTimeoutSec)
{
var actors = CreateActors(dataObjectTypes,
dataConnection,
new BulkCopyOptions
{
BulkCopyTimeout = bulkReplaceCommandTimeoutSec
});
var initialStats = _kafkaMessageFlowInfoProvider.GetFlowStats(messageFlow)
.ToDictionary(x => x.TopicPartition, x => new MessageFlowStats(x.TopicPartition, x.End, Offset.Unset));
using var receiver = _receiverFactory.Create(messageFlow);
while (true)
{
var batch = receiver.ReceiveBatch(batchSize);
var bulkCommands = _commandFactory.CreateCommands(batch);
if (bulkCommands.Count > 0)
{
using var scope = new TransactionScope(TransactionScopeOption.RequiresNew, new TransactionOptions { IsolationLevel = IsolationLevel.Serializable, Timeout = TimeSpan.Zero });
foreach (var actor in actors)
{
actor.ExecuteCommands(bulkCommands);
}
scope.Complete();
}
receiver.CompleteBatch(batch);
var batchStats = batch
.GroupBy(x => x.TopicPartition)
.ToDictionary(x => x.Key, x => x.Max(y => y.Offset.Value));
foreach (var batchStat in batchStats)
{
if (!initialStats.TryGetValue(batchStat.Key, out var initialStat))
{
throw new KeyNotFoundException(batchStat.Key.ToString());
}
var currentStat = new MessageFlowStats(batchStat.Key, initialStat.End, batchStat.Value + 1);
_tracer.Info($"Topic {currentStat.TopicPartition}, End: {currentStat.End}, Offset: {currentStat.Offset}, Lag: {currentStat.Lag}");
initialStats[batchStat.Key] = currentStat;
}
var complete = initialStats.Values.All(x => x.Lag <= 0);
if (complete)
{
_tracer.Info($"Kafka state init for flow {messageFlow.Description} complete");
break;
}
}
}
public Task <long> WaitForVersion(Guid ermToken)
{
var amsCount = _kafkaMessageFlowInfoProvider.GetFlowStats(AmsFactsFlow.Instance).Single().End;
return(WaitForVersion(ermToken, amsCount, WaitInterval, WaitTimeout));
}