private async Task <CdcState> SetInitialStateAsync(CancellationToken token, string executionId, TableSchema tableSchema, TimeSpan maxInterval)
{
byte[] initialToLsn = await _cdcReaderClient.GetMaxLsnAsync();
var existingOffset = await _cdcReaderClient.GetLastCdcOffsetAsync(executionId, tableSchema.TableName);
if (existingOffset.Result == Result.NoStoredState)
{
Console.WriteLine($"Table {tableSchema.TableName} - No previous stored offset. Starting from first change");
var initialFromLsn = await _cdcReaderClient.GetMinValidLsnAsync(tableSchema.TableName);
var hasFirstChange = false;
ChangeBatch syncBatch = null;
ChangeRecord firstChange = null;
while (!hasFirstChange && !token.IsCancellationRequested)
{
syncBatch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, initialFromLsn, initialToLsn, 1);
if (syncBatch.Changes.Any())
{
firstChange = syncBatch.Changes.First();
hasFirstChange = true;
}
else
{
await Task.Delay(maxInterval);
}
}
await BlockingWriteToRedshiftAsync(token, tableSchema.TableName, syncBatch);
var cdcState = new CdcState()
{
FromLsn = firstChange.Lsn,
FromSeqVal = firstChange.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = syncBatch.MoreOfLastTransaction
};
var offset = GetOffset(cdcState);
await _cdcReaderClient.StoreCdcOffsetAsync(executionId, tableSchema.TableName, offset);
return(cdcState);
}
else
{
Console.WriteLine($"Table {tableSchema.TableName} - Starting from stored offset");
return(new CdcState()
{
FromLsn = existingOffset.State.Lsn,
FromSeqVal = existingOffset.State.SeqVal,
ToLsn = initialToLsn,
UnfinishedLsn = existingOffset.State.UnfinishedLsn
});
}
}
private Offset GetOffset(CdcState cdcState)
{
var offset = new Offset();
offset.Lsn = cdcState.FromLsn;
offset.SeqVal = cdcState.FromSeqVal;
offset.UnfinishedLsn = cdcState.UnfinishedLsn;
return(offset);
}