private async Task WriteToRedshiftAsync(CancellationToken token, TimeSpan windowSize, int windowSizeItems, string tableName, BlockingCollection <MessageProxy <RowChange> > accumulatedChanges) { tableName = tableName.ToLower(); while (!token.IsCancellationRequested && !accumulatedChanges.IsAddingCompleted) { // create change window var messages = EmptyBuffer(accumulatedChanges); if (messages.Any()) { var changesToPut = messages.Select(x => x.Payload).ToList(); // upload change window to S3 then Redshift await _redshiftClient.UploadAsCsvAsync(tableName, changesToPut); // commit the last message in the batch await messages.Last().CommitAsync(); } // wait for interval but check buffered item count regularly and if max size is reached then upload int secondsWaited = 0; while (secondsWaited < (int)windowSize.TotalSeconds) { if (accumulatedChanges.Count >= windowSizeItems) { break; } secondsWaited++; await Task.Delay(1000); } } }
private async Task <bool> WriteToRedshiftAsync(string tableName, ChangeBatch batch) { if (batch.Changes.Any()) { var rowChanges = new List <RowChange>(); foreach (var record in batch.Changes) { rowChanges.Add(new RowChange() { ChangeKey = record.ChangeKey, ChangeType = (CdcTools.Redshift.Changes.ChangeType)record.ChangeType, Data = record.Data, Lsn = record.LsnStr, SeqVal = record.SeqValStr }); } try { await _redshiftClient.UploadAsCsvAsync(tableName, rowChanges); return(true); } catch (Exception ex) { Console.WriteLine($"{tableName} upload failed. {ex}"); return(false); } } return(true); }
private async Task <Tuple <bool, long> > WriteToRedshiftAsync(FullLoadBatch batch, long ctr) { if (batch.Records.Any()) { var rowChanges = new List <RowChange>(); foreach (var record in batch.Records) { rowChanges.Add(new RowChange() { ChangeKey = record.ChangeKey, ChangeType = CdcTools.Redshift.Changes.ChangeType.INSERT, Data = record.Data, Lsn = ctr.ToString(), SeqVal = ctr.ToString() }); ctr++; } try { await _redshiftClient.UploadAsCsvAsync(batch.TableSchema.TableName, rowChanges); return(Tuple.Create(true, ctr)); } catch (Exception ex) { Console.WriteLine($"{batch.TableSchema.TableName} upload failed. {ex}"); return(Tuple.Create(false, ctr)); } } return(Tuple.Create(true, ctr)); }
private async Task UploadBatchesAsync(List <TransactionBatch> batches) { var tableRowChanges = new Dictionary <string, List <RowChange> >(); foreach (var batch in batches) { foreach (var tableGroup in batch.Changes.GroupBy(x => x.TableName)) { var orderedChanges = tableGroup.OrderBy(x => x.LsnInt).ThenBy(x => x.SeqValInt).ToList(); if (!tableRowChanges.ContainsKey(tableGroup.Key)) { tableRowChanges.Add(tableGroup.Key, new List <RowChange>()); } tableRowChanges[tableGroup.Key].AddRange(ConvertToRowChanges(orderedChanges)); } } await _redshiftClient.UploadAsCsvAsync(tableRowChanges); }