private async Task <Tuple <bool, long> > WriteToRedshiftAsync(FullLoadBatch batch, long ctr)
{
if (batch.Records.Any())
{
var rowChanges = new List <RowChange>();
foreach (var record in batch.Records)
{
rowChanges.Add(new RowChange()
{
ChangeKey = record.ChangeKey,
ChangeType = CdcTools.Redshift.Changes.ChangeType.INSERT,
Data = record.Data,
Lsn = ctr.ToString(),
SeqVal = ctr.ToString()
});
ctr++;
}
try
{
await _redshiftClient.UploadAsCsvAsync(batch.TableSchema.TableName, rowChanges);
return(Tuple.Create(true, ctr));
}
catch (Exception ex)
{
Console.WriteLine($"{batch.TableSchema.TableName} upload failed. {ex}");
return(Tuple.Create(false, ctr));
}
}
return(Tuple.Create(true, ctr));
}
public async Task <FullLoadBatch> GetBatchAsync(TableSchema tableSchema, PrimaryKeyValue lastRetrievedKey, int batchSize)
{
var batch = new FullLoadBatch();
batch.TableSchema = tableSchema;
using (var conn = await GetOpenConnectionAsync())
{
var command = conn.CreateCommand();
command.CommandText = TableSchemaQueryBuilder.GetExtractQueryUsingAllKeys(tableSchema, batchSize);
foreach (var pk in tableSchema.PrimaryKeys.OrderBy(x => x.OrdinalPosition))
{
var columnSchema = tableSchema.GetColumn(pk.ColumnName);
var value = lastRetrievedKey.GetValue(pk.OrdinalPosition);
command.Parameters.Add(CreateSqlParameter(columnSchema, "@p" + pk.OrdinalPosition, value));
}
using (var reader = await command.ExecuteReaderAsync())
{
int ctr = 1;
while (await reader.ReadAsync())
{
var change = new FullLoadRecord();
change.ChangeKey = GetRecordId(reader, tableSchema);
change.BatchSeqNo = ctr;
foreach (var column in tableSchema.Columns)
{
change.Data.Add(column.Name, reader[column.Name]);
}
batch.Records.Add(change);
ctr++;
}
}
}
if (batch.Records.Any())
{
batch.FirstRowKey = GetKey(batch.Records.First(), tableSchema);
batch.LastRowKey = GetKey(batch.Records.Last(), tableSchema);
}
return(batch);
}
public async Task <FullLoadBatch> GetFirstBatchAsync(TableSchema tableSchema, int batchSize)
{
var batch = new FullLoadBatch();
batch.TableSchema = tableSchema;
using (var conn = await GetOpenConnectionAsync())
{
var command = conn.CreateCommand();
command.CommandText = $"SELECT TOP {batchSize} * FROM {tableSchema.Schema}.{tableSchema.TableName} ORDER BY {tableSchema.GetOrderedPrimaryKeyColumns()};";
using (var reader = await command.ExecuteReaderAsync())
{
int ctr = 1;
while (await reader.ReadAsync())
{
var change = new FullLoadRecord();
change.ChangeKey = GetRecordId(reader, tableSchema);
change.BatchSeqNo = ctr;
foreach (var column in tableSchema.Columns)
{
change.Data.Add(column.Name, reader[column.Name]);
}
batch.Records.Add(change);
ctr++;
}
}
}
if (batch.Records.Any())
{
batch.FirstRowKey = GetKey(batch.Records.First(), tableSchema);
batch.LastRowKey = GetKey(batch.Records.Last(), tableSchema);
}
return(batch);
}
private async Task <long> PublishAsync(IKafkaProducer producer, CancellationToken token, FullLoadBatch batch, long ctr)
{
foreach (var row in batch.Records)
{
var change = new ChangeRecord();
change.ChangeKey = row.ChangeKey;
change.ChangeType = ChangeType.INSERT;
change.LsnStr = ctr.ToString();
change.SeqValStr = ctr.ToString();
change.Data = row.Data;
await producer.SendAsync(token, change);
ctr++;
}
return(ctr);
}
