private async Task WriteToRedshiftAsync(CancellationToken token, TimeSpan windowSize, int windowSizeItems, string tableName, BlockingCollection <MessageProxy <RowChange> > accumulatedChanges)
{
tableName = tableName.ToLower();
while (!token.IsCancellationRequested && !accumulatedChanges.IsAddingCompleted)
{
// create change window
var messages = EmptyBuffer(accumulatedChanges);
if (messages.Any())
{
var changesToPut = messages.Select(x => x.Payload).ToList();
// upload change window to S3 then Redshift
await _redshiftClient.UploadAsCsvAsync(tableName, changesToPut);
// commit the last message in the batch
await messages.Last().CommitAsync();
}
// wait for interval but check buffered item count regularly and if max size is reached then upload
int secondsWaited = 0;
while (secondsWaited < (int)windowSize.TotalSeconds)
{
if (accumulatedChanges.Count >= windowSizeItems)
{
break;
}
secondsWaited++;
await Task.Delay(1000);
}
}
}
private async Task <bool> WriteToRedshiftAsync(string tableName, ChangeBatch batch)
{
if (batch.Changes.Any())
{
var rowChanges = new List <RowChange>();
foreach (var record in batch.Changes)
{
rowChanges.Add(new RowChange()
{
ChangeKey = record.ChangeKey,
ChangeType = (CdcTools.Redshift.Changes.ChangeType)record.ChangeType,
Data = record.Data,
Lsn = record.LsnStr,
SeqVal = record.SeqValStr
});
}
try
{
await _redshiftClient.UploadAsCsvAsync(tableName, rowChanges);
return(true);
}
catch (Exception ex)
{
Console.WriteLine($"{tableName} upload failed. {ex}");
return(false);
}
}
return(true);
}
private async Task <Tuple <bool, long> > WriteToRedshiftAsync(FullLoadBatch batch, long ctr)
{
if (batch.Records.Any())
{
var rowChanges = new List <RowChange>();
foreach (var record in batch.Records)
{
rowChanges.Add(new RowChange()
{
ChangeKey = record.ChangeKey,
ChangeType = CdcTools.Redshift.Changes.ChangeType.INSERT,
Data = record.Data,
Lsn = ctr.ToString(),
SeqVal = ctr.ToString()
});
ctr++;
}
try
{
await _redshiftClient.UploadAsCsvAsync(batch.TableSchema.TableName, rowChanges);
return(Tuple.Create(true, ctr));
}
catch (Exception ex)
{
Console.WriteLine($"{batch.TableSchema.TableName} upload failed. {ex}");
return(Tuple.Create(false, ctr));
}
}
return(Tuple.Create(true, ctr));
}
private async Task UploadBatchesAsync(List <TransactionBatch> batches)
{
var tableRowChanges = new Dictionary <string, List <RowChange> >();
foreach (var batch in batches)
{
foreach (var tableGroup in batch.Changes.GroupBy(x => x.TableName))
{
var orderedChanges = tableGroup.OrderBy(x => x.LsnInt).ThenBy(x => x.SeqValInt).ToList();
if (!tableRowChanges.ContainsKey(tableGroup.Key))
{
tableRowChanges.Add(tableGroup.Key, new List <RowChange>());
}
tableRowChanges[tableGroup.Key].AddRange(ConvertToRowChanges(orderedChanges));
}
}
await _redshiftClient.UploadAsCsvAsync(tableRowChanges);
}
