private async Task StreamTableAsync(CancellationToken token, string executionId, TableSchema tableSchema, SerializationMode serializationMode, bool sendWithKey, int batchSize, int printPercentProgressMod) { string topicName = _kafkaTopicPrefix + tableSchema.TableName.ToLower(); var rowCount = await _cdcReaderClient.GetRowCountAsync(tableSchema); Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} has {rowCount} rows to export"); int progress = 0; using (var producer = ProducerFactory.GetProducer(topicName, tableSchema, serializationMode, sendWithKey, _kafkaBootstrapServers, _schemaRegistryUrl)) { long ctr = 0; PrimaryKeyValue lastRetrievedKey = null; var existingOffset = await _cdcReaderClient.GetLastFullLoadOffsetAsync(executionId, tableSchema.TableName); if (existingOffset.Result == CdcReader.State.Result.NoStoredState) { Console.WriteLine($"Table {tableSchema.TableName} - No previous stored offset. Starting from first row"); var firstBatch = await _cdcReaderClient.GetFirstBatchAsync(tableSchema, batchSize); ctr = await PublishAsync(producer, token, firstBatch, ctr); lastRetrievedKey = firstBatch.LastRowKey; await _cdcReaderClient.StoreFullLoadOffsetAsync(executionId, tableSchema.TableName, firstBatch.LastRowKey); } else { Console.WriteLine($"Table {tableSchema.TableName} - No data to export"); lastRetrievedKey = existingOffset.State; } bool finished = false; while (!token.IsCancellationRequested && !finished) { var changes = new List <RowChange>(); var batch = await _cdcReaderClient.GetBatchAsync(tableSchema, lastRetrievedKey, batchSize); ctr = await PublishAsync(producer, token, batch, ctr); int latestProgress = (int)(((double)ctr / (double)rowCount) * 100); if (progress != latestProgress && latestProgress % printPercentProgressMod == 0) { Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - Progress at {latestProgress}% ({ctr} records)"); } progress = latestProgress; lastRetrievedKey = batch.LastRowKey; await _cdcReaderClient.StoreFullLoadOffsetAsync(executionId, tableSchema.TableName, lastRetrievedKey); if (!batch.Records.Any() || batch.Records.Count < batchSize) { finished = true; } } if (token.IsCancellationRequested) { Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - cancelled at progress at {progress}% ({ctr} records)"); } else { Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - complete ({ctr} records)"); } } }
private async Task ExportTableAsync(CancellationToken token, string executionId, TableSchema tableSchema, int batchSize, int printPercentProgressMod) { var rowCount = await _cdcReaderClient.GetRowCountAsync(tableSchema); Console.WriteLine($"Table {tableSchema.TableName} - {rowCount} rows to export"); int progress = 0; PrimaryKeyValue lastRetrievedKey = await SetStartingPosition(executionId, tableSchema, batchSize); long ctr = batchSize; bool finished = false; while (!token.IsCancellationRequested && !finished) { var changes = new List <RowChange>(); var batch = await _cdcReaderClient.GetBatchAsync(tableSchema, lastRetrievedKey, batchSize); var result = await WriteToRedshiftAsync(batch, ctr); if (result.Item1) { ctr = result.Item2; int latestProgress = (int)(((double)ctr / (double)rowCount) * 100); if (progress != latestProgress && latestProgress % printPercentProgressMod == 0) { Console.WriteLine($"Table {tableSchema.TableName} - Progress at {latestProgress}% ({ctr} records)"); } progress = latestProgress; lastRetrievedKey = batch.LastRowKey; if (batch.Records.Any()) { await _cdcReaderClient.StoreFullLoadOffsetAsync(executionId, tableSchema.TableName, lastRetrievedKey); } if (!batch.Records.Any() || batch.Records.Count < batchSize) { finished = true; } } else { Console.WriteLine($"Table {tableSchema.TableName} - Failed to upload to Redshift. Will try again in 10 seconds."); await WaitForSeconds(token, 10); } } if (token.IsCancellationRequested) { Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - cancelled at progress at {progress}% ({ctr} records)"); } else { Console.WriteLine($"Table {tableSchema.Schema}.{tableSchema.TableName} - complete ({ctr} records)"); } }