public void StartReading(CancellationToken token, CdcRequest cdcRequest)
{
foreach (var tableName in cdcRequest.Tables)
{
var readerTask = Task.Run(async() =>
{
try
{
await StartPublishingChanges(token,
cdcRequest.ExecutionId,
tableName,
cdcRequest.Interval,
cdcRequest.BatchSize,
cdcRequest.SendWithKey,
cdcRequest.SerializationMode);
}
catch (Exception ex)
{
Console.WriteLine($"CDC reader failure. Table {tableName}. Error: {ex}");
}
});
_readerTasks.Add(readerTask);
}
}
static void Main(string[] args)
{
Console.Title = "CDC To Kafka Streamer";
// support graceful shutdown in Docker
var ended = new ManualResetEventSlim();
var starting = new ManualResetEventSlim();
AssemblyLoadContext.Default.Unloading += ctx =>
{
System.Console.WriteLine("Unloading fired");
starting.Set();
System.Console.WriteLine("Waiting for completion");
ended.Wait();
};
// set up configuration
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddCommandLine(args)
.AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;
IConfigurationRoot configuration = builder.Build();
// get parameters and start
var executionId = GetExecutionId(configuration);
var runMode = GetRunMode(configuration);
var tables = GetTables(configuration);
var serializationMode = GetSerializationMode(configuration);
var sendWithKey = GetSendWithKey(configuration);
var batchSize = GetBatchSize(configuration);
var kafkaBootstrapServers = GetBootstrapServers(configuration);
var schemaRegistryUrl = GetSchemaRegistryUrl(configuration);
var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
var cts = new CancellationTokenSource();
if (runMode == RunMode.FullLoad)
{
var printMod = GetPrintMod(configuration);
var fullLoadStreamer = new FullLoadStreamer(configuration, cdcReaderClient);
fullLoadStreamer.StreamTablesAsync(cts.Token, executionId, tables, serializationMode, sendWithKey, batchSize, printMod).Wait();
Console.WriteLine("Streaming to Kafka in progress.");
Thread.Sleep(2000);
bool shutdown = false;
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
while (!shutdown)
{
if (Console.KeyAvailable)
{
shutdown = true;
}
else if (fullLoadStreamer.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#else
while (!shutdown)
{
if (starting.IsSet)
{
shutdown = true;
}
else if (fullLoadStreamer.HasFinished())
{
shutdown = true;
}
Thread.Sleep(500);
}
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
fullLoadStreamer.WaitForCompletion();
ended.Set();
}
else
{
var interval = GetInterval(configuration);
var cdcRequest = new CdcRequest()
{
BatchSize = batchSize,
ExecutionId = executionId,
Interval = interval,
SendWithKey = sendWithKey,
SerializationMode = serializationMode,
Tables = tables
};
var cdcStreamer = new ChangeStreamer(configuration, cdcReaderClient);
cdcStreamer.StartReading(cts.Token, cdcRequest);
Console.WriteLine("Streaming to Kafka started.");
// wait for shutdown signal
#if DEBUG
Console.WriteLine("Press any key to shutdown");
Console.ReadKey();
#else
starting.Wait();
#endif
Console.WriteLine("Received signal gracefully shutting down");
cts.Cancel();
cdcStreamer.WaitForCompletion();
ended.Set();
}
}