private async Task StartPublishingChanges(CancellationToken token,
string executionId,
string tableName,
TimeSpan maxInterval,
int batchSize)
{
var tableSchema = await _cdcReaderClient.GetTableSchemaAsync(tableName);
var cdcState = await SetInitialStateAsync(token, executionId, tableSchema, maxInterval);
var sw = new Stopwatch();
while (!token.IsCancellationRequested)
{
cdcState.ToLsn = await _cdcReaderClient.GetMaxLsnAsync();
sw.Start();
Console.WriteLine($"Table {tableName} - Starting to export LSN range {GetBigInteger(cdcState.FromLsn)} to {GetBigInteger(cdcState.ToLsn)}");
int blockCounter = 0;
bool more = true;
while (!token.IsCancellationRequested && more)
{
if (GetBigInteger(cdcState.FromLsn) <= GetBigInteger(cdcState.ToLsn))
{
blockCounter++;
ChangeBatch batch = null;
if (cdcState.UnfinishedLsn)
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.FromSeqVal, cdcState.ToLsn, batchSize);
}
else
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.ToLsn, batchSize);
}
if (batch.Changes.Any())
{
Console.WriteLine($"Table {tableName} - Retrieved block #{blockCounter} with {batch.Changes.Count} changes");
await BlockingWriteToRedshiftAsync(token, tableSchema.TableName, batch);
cdcState.FromLsn = batch.Changes.Last().Lsn;
cdcState.FromSeqVal = batch.Changes.Last().SeqVal;
more = batch.MoreChanges;
cdcState.UnfinishedLsn = batch.MoreOfLastTransaction;
if (cdcState.UnfinishedLsn)
{
cdcState.FromSeqVal = Increment(cdcState.FromSeqVal);
}
else
{
cdcState.FromLsn = Increment(cdcState.FromLsn);
}
var offset = GetOffset(cdcState);
await BlockingStoreCdcOffsetAsync(token, executionId, tableName, offset);
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
var remainingMs = maxInterval.TotalMilliseconds - sw.Elapsed.TotalMilliseconds;
if (remainingMs > 0)
{
await Task.Delay((int)remainingMs);
}
sw.Reset();
}
}
private async Task StartPublishingChanges(CancellationToken token,
string executionId,
string tableName,
TimeSpan maxInterval,
int batchSize,
bool sendWithKey,
SerializationMode serializationMode)
{
var tableTopic = _tableTopicPrefix + tableName.ToLower();
var tableSchema = await _cdcReaderClient.GetTableSchemaAsync(tableName);
using (var producer = ProducerFactory.GetProducer(tableTopic, tableSchema, serializationMode, sendWithKey, _kafkaBootstrapServers, _schemaRegistryUrl))
{
var cdcState = await SetInitialStateAsync(token, producer, executionId, tableSchema, maxInterval);
var sw = new Stopwatch();
while (!token.IsCancellationRequested)
{
cdcState.ToLsn = await _cdcReaderClient.GetMaxLsnAsync();
sw.Start();
Console.WriteLine($"Table {tableName} - Starting to export LSN range {GetBigInteger(cdcState.FromLsn)} to {GetBigInteger(cdcState.ToLsn)}");
bool more = true;
int blockCounter = 0;
while (!token.IsCancellationRequested && more)
{
if (GetBigInteger(cdcState.FromLsn) <= GetBigInteger(cdcState.ToLsn))
{
blockCounter++;
ChangeBatch batch = null;
if (cdcState.UnfinishedLsn)
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.FromSeqVal, cdcState.ToLsn, batchSize);
}
else
{
batch = await _cdcReaderClient.GetChangeBatchAsync(tableSchema, cdcState.FromLsn, cdcState.ToLsn, batchSize);
}
if (batch.Changes.Any())
{
Console.WriteLine($"Table {tableName} - Retrieved block #{blockCounter} with {batch.Changes.Count} changes");
foreach (var change in batch.Changes)
{
await producer.SendAsync(token, change);
cdcState.FromLsn = change.Lsn;
cdcState.FromSeqVal = change.SeqVal;
}
more = batch.MoreChanges;
cdcState.UnfinishedLsn = batch.MoreOfLastTransaction;
if (cdcState.UnfinishedLsn)
{
cdcState.FromSeqVal = Increment(cdcState.FromSeqVal);
}
else
{
cdcState.FromLsn = Increment(cdcState.FromLsn);
}
var offset = GetOffset(cdcState);
await BlockingStoreCdcOffsetAsync(token, executionId, tableName, offset);
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
else
{
more = false;
cdcState.UnfinishedLsn = false;
Console.WriteLine($"Table {tableName} - No changes");
}
}
var remainingMs = maxInterval.TotalMilliseconds - sw.Elapsed.TotalMilliseconds;
if (remainingMs > 0)
{
await Task.Delay((int)remainingMs);
}
sw.Reset();
}
}
}