private async Task <CdcState> SetInitialStateAsync(CancellationToken token, IKafkaProducer producer, string executionId, TableSchema tableSchema, TimeSpan maxInterval)
{
byte[] initialToLsn = await _cdcReaderClient.GetMaxLsnAsync();
var existingOffset = await _cdcReaderClient.GetLastCdcOffsetAsync(executionId, tableSchema.TableName);
if (existingOffset.Result == Result.NoStoredState)
{
Console.WriteLine($"Table {tableSchema.TableName} - No previous stored LSN. Starting from first change");
var hasFirstChange = false;
ChangeBatch syncBatch = null;
ChangeRecord firstChange = null;
while (!hasFirstChange && !token.IsCancellationRequested)
{
var initialFromLsn = await _cdcReaderClient.GetMinValidLsnAsync(tableSchema.TableName);
initialToLsn = await _cdcReaderClient.GetMaxLsnAsync();
syncBatch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, initialFromLsn, initialToLsn, 1);
if (syncBatch.Changes.Any())
{
firstChange = syncBatch.Changes.First();
await producer.SendAsync(token, firstChange);
hasFirstChange = true;
}
else
{
await Task.Delay(maxInterval);
}
}
var cdcState = new CdcState()
{
FromLsn = firstChange.Lsn,
FromSeqVal = firstChange.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = syncBatch.MoreOfLastTransaction
};
var offset = GetOffset(cdcState);
await _cdcReaderClient.StoreCdcOffsetAsync(executionId, tableSchema.TableName, offset);
return(cdcState);
}
else
{
Console.WriteLine($"Table {tableSchema.TableName} - Starting from stored LSN");
return(new CdcState()
{
FromLsn = existingOffset.State.Lsn,
FromSeqVal = existingOffset.State.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = existingOffset.State.UnfinishedLsn
});
}
}
public async Task DoWorkAsync()
{
foreach (var eventStoreItem in _eventStoreItems.OrderBy(e => e.OccurredOn))
{
Console.WriteLine($"**SynchroService::DoWorkAsync - topic : {Topics.Synchro},output : {eventStoreItem} ");
await _serviceBus.SendAsync(Topics.Synchro, eventStoreItem);
}
}
private async Task <long> PublishAsync(IKafkaProducer producer, CancellationToken token, FullLoadBatch batch, long ctr)
{
foreach (var row in batch.Records)
{
var change = new ChangeRecord();
change.ChangeKey = row.ChangeKey;
change.ChangeType = ChangeType.INSERT;
change.LsnStr = ctr.ToString();
change.SeqValStr = ctr.ToString();
change.Data = row.Data;
await producer.SendAsync(token, change);
ctr++;
}
return(ctr);
}
public async Task SendEmail([FromBody] string message)
{
await _kafkaProducer.SendAsync(message);
}