private async Task <CdcState> SetInitialStateAsync(CancellationToken token, IKafkaProducer producer, string executionId, TableSchema tableSchema, TimeSpan maxInterval) { byte[] initialToLsn = await _cdcReaderClient.GetMaxLsnAsync(); var existingOffset = await _cdcReaderClient.GetLastCdcOffsetAsync(executionId, tableSchema.TableName); if (existingOffset.Result == Result.NoStoredState) { Console.WriteLine($"Table {tableSchema.TableName} - No previous stored LSN. Starting from first change"); var hasFirstChange = false; ChangeBatch syncBatch = null; ChangeRecord firstChange = null; while (!hasFirstChange && !token.IsCancellationRequested) { var initialFromLsn = await _cdcReaderClient.GetMinValidLsnAsync(tableSchema.TableName); initialToLsn = await _cdcReaderClient.GetMaxLsnAsync(); syncBatch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, initialFromLsn, initialToLsn, 1); if (syncBatch.Changes.Any()) { firstChange = syncBatch.Changes.First(); await producer.SendAsync(token, firstChange); hasFirstChange = true; } else { await Task.Delay(maxInterval); } } var cdcState = new CdcState() { FromLsn = firstChange.Lsn, FromSeqVal = firstChange.SeqVal, ToLsn = initialToLsn, UnfinishedLsn = syncBatch.MoreOfLastTransaction }; var offset = GetOffset(cdcState); await _cdcReaderClient.StoreCdcOffsetAsync(executionId, tableSchema.TableName, offset); return(cdcState); } else { Console.WriteLine($"Table {tableSchema.TableName} - Starting from stored LSN"); return(new CdcState() { FromLsn = existingOffset.State.Lsn, FromSeqVal = existingOffset.State.SeqVal, ToLsn = initialToLsn, UnfinishedLsn = existingOffset.State.UnfinishedLsn }); } }
private Offset GetOffset(CdcState cdcState) { var offset = new Offset(); offset.Lsn = cdcState.FromLsn; offset.SeqVal = cdcState.FromSeqVal; offset.UnfinishedLsn = cdcState.UnfinishedLsn; return(offset); }