public ChangeExporter(CdcReaderClient cdcReaderClient,
RedshiftClient redshiftClient)
{
_cdcReaderClient = cdcReaderClient;
_redshiftClient = redshiftClient;
_readerTasks = new List <Task>();
}
public FullLoadStreamer(IConfiguration configuration, CdcReaderClient cdcReaderClient)
{
_cdcReaderClient = cdcReaderClient;
_kafkaTopicPrefix = configuration["TableTopicPrefix"];
_kafkaBootstrapServers = configuration["KafkaBootstrapServers"];
_schemaRegistryUrl = configuration["KafkaSchemaRegistryUrl"];
_loadTasks = new List <Task>();
}
public FullLoadExporter(CdcReaderClient cdcReaderClient, RedshiftClient redshiftClient)
{
_cdcReaderClient = cdcReaderClient;
_redshiftClient = redshiftClient;
_loadTasks = new List <Task>();
}
static void Main(string[] args)
{
Console.Title = "CDC to Redshift";
// support graceful shutdown in Docker
var ended = new ManualResetEventSlim();
var starting = new ManualResetEventSlim();
AssemblyLoadContext.Default.Unloading += ctx =>
{
System.Console.WriteLine("Unloading fired");
starting.Set();
System.Console.WriteLine("Waiting for completion");
ended.Wait();
};
// set up configuration
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddCommandLine(args)
.AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;
IConfigurationRoot configuration = builder.Build();
var executionId = GetExecutionId(configuration);
var runMode = GetRunMode(configuration);
var tables = GetTables(configuration);
var batchSize = GetNonTransactionalTableBatchSize(configuration);
var redshiftClient = GetRedshiftClient(configuration);
var cts = new CancellationTokenSource();
if (runMode == RunMode.FullLoad)
{
var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
var printMod = GetPrintMod(configuration);
var fullLoadExporter = new FullLoadExporter(cdcReaderClient, redshiftClient);
fullLoadExporter.ExportTablesAsync(cts.Token, executionId, tables, batchSize, printMod).Wait();
Console.WriteLine("Export to Redshift started.");
Thread.Sleep(2000);
bool shutdown = false;
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
while (!shutdown)
{
if (Console.KeyAvailable)
{
shutdown = true;
}
else if (fullLoadExporter.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#else
while (!shutdown)
{
if (starting.IsSet)
{
shutdown = true;
}
else if (fullLoadExporter.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
fullLoadExporter.WaitForCompletion();
ended.Set();
}
else if (runMode == RunMode.NonTransactionalCdc)
{
var interval = GetInterval(configuration);
var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
var cdcExporter = new ChangeExporter(cdcReaderClient, redshiftClient);
cdcExporter.StartExportingChangesAsync(cts.Token, executionId, tables, interval, batchSize).Wait();
Console.WriteLine("Export to Redshift in progress.");
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
Console.ReadKey();
#else
starting.Wait();
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
cdcExporter.WaitForCompletion();
ended.Set();
}
else
{
var interval = GetInterval(configuration);
var cdcReaderClient = new CdcTransactionClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
var cdcExporter = new TransactionExporter(cdcReaderClient, redshiftClient);
var perTableBufferLimit = GetPerTableBufferLimit(configuration);
var tranBufferLimit = GetTransactionBufferLimit(configuration);
var tranBatchSizeLimit = GetTransactionBatchSizeLimit(configuration);
cdcExporter.StartExportingChangesAsync(cts.Token, executionId, tables, interval, perTableBufferLimit, tranBufferLimit, tranBatchSizeLimit).Wait();
Console.WriteLine("Export to Redshift in progress.");
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
Console.ReadKey();
#else
starting.Wait();
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
cdcExporter.WaitForCompletion();
ended.Set();
}
}
static void Main(string[] args)
{
Console.Title = "CDC To Kafka Streamer";
// support graceful shutdown in Docker
var ended = new ManualResetEventSlim();
var starting = new ManualResetEventSlim();
AssemblyLoadContext.Default.Unloading += ctx =>
{
System.Console.WriteLine("Unloading fired");
starting.Set();
System.Console.WriteLine("Waiting for completion");
ended.Wait();
};
// set up configuration
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddCommandLine(args)
.AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;
IConfigurationRoot configuration = builder.Build();
// get parameters and start
var executionId = GetExecutionId(configuration);
var runMode = GetRunMode(configuration);
var tables = GetTables(configuration);
var serializationMode = GetSerializationMode(configuration);
var sendWithKey = GetSendWithKey(configuration);
var batchSize = GetBatchSize(configuration);
var kafkaBootstrapServers = GetBootstrapServers(configuration);
var schemaRegistryUrl = GetSchemaRegistryUrl(configuration);
var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
var cts = new CancellationTokenSource();
if (runMode == RunMode.FullLoad)
{
var printMod = GetPrintMod(configuration);
var fullLoadStreamer = new FullLoadStreamer(configuration, cdcReaderClient);
fullLoadStreamer.StreamTablesAsync(cts.Token, executionId, tables, serializationMode, sendWithKey, batchSize, printMod).Wait();
Console.WriteLine("Streaming to Kafka in progress.");
Thread.Sleep(2000);
bool shutdown = false;
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
while (!shutdown)
{
if (Console.KeyAvailable)
{
shutdown = true;
}
else if (fullLoadStreamer.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#else
while (!shutdown)
{
if (starting.IsSet)
{
shutdown = true;
}
else if (fullLoadStreamer.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
fullLoadStreamer.WaitForCompletion();
ended.Set();
}
else
{
var interval = GetInterval(configuration);
var cdcRequest = new CdcRequest()
{
BatchSize = batchSize,
ExecutionId = executionId,
Interval = interval,
SendWithKey = sendWithKey,
SerializationMode = serializationMode,
Tables = tables
};
var cdcStreamer = new ChangeStreamer(configuration, cdcReaderClient);
cdcStreamer.StartReading(cts.Token, cdcRequest);
Console.WriteLine("Streaming to Kafka started.");
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
Console.ReadKey();
#else
starting.Wait();
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
cdcStreamer.WaitForCompletion();
ended.Set();
}
}