public void StartReading(CancellationToken token, CdcRequest cdcRequest) { foreach (var tableName in cdcRequest.Tables) { var readerTask = Task.Run(async() => { try { await StartPublishingChanges(token, cdcRequest.ExecutionId, tableName, cdcRequest.Interval, cdcRequest.BatchSize, cdcRequest.SendWithKey, cdcRequest.SerializationMode); } catch (Exception ex) { Console.WriteLine($"CDC reader failure. Table {tableName}. Error: {ex}"); } }); _readerTasks.Add(readerTask); } }
static void Main(string[] args) { Console.Title = "CDC To Kafka Streamer"; // support graceful shutdown in Docker var ended = new ManualResetEventSlim(); var starting = new ManualResetEventSlim(); AssemblyLoadContext.Default.Unloading += ctx => { System.Console.WriteLine("Unloading fired"); starting.Set(); System.Console.WriteLine("Waiting for completion"); ended.Wait(); }; // set up configuration var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) .AddCommandLine(args) .AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix; IConfigurationRoot configuration = builder.Build(); // get parameters and start var executionId = GetExecutionId(configuration); var runMode = GetRunMode(configuration); var tables = GetTables(configuration); var serializationMode = GetSerializationMode(configuration); var sendWithKey = GetSendWithKey(configuration); var batchSize = GetBatchSize(configuration); var kafkaBootstrapServers = GetBootstrapServers(configuration); var schemaRegistryUrl = GetSchemaRegistryUrl(configuration); var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]); var cts = new CancellationTokenSource(); if (runMode == RunMode.FullLoad) { var printMod = GetPrintMod(configuration); var fullLoadStreamer = new FullLoadStreamer(configuration, cdcReaderClient); fullLoadStreamer.StreamTablesAsync(cts.Token, executionId, tables, serializationMode, sendWithKey, batchSize, printMod).Wait(); Console.WriteLine("Streaming to Kafka in progress."); Thread.Sleep(2000); bool shutdown = false; // wait for shutdown signal #if DEBUG Console.WriteLine("Press any key to shutdown"); while (!shutdown) { if (Console.KeyAvailable) { shutdown = true; } else if (fullLoadStreamer.HasFinished()) { shutdown = true; } Thread.Sleep(500); } #else while (!shutdown) { if (starting.IsSet) { shutdown = true; } else if (fullLoadStreamer.HasFinished()) { shutdown = true; } Thread.Sleep(500); } #endif Console.WriteLine("Received signal gracefully shutting down"); cts.Cancel(); fullLoadStreamer.WaitForCompletion(); ended.Set(); } else { var interval = GetInterval(configuration); var cdcRequest = new CdcRequest() { BatchSize = batchSize, ExecutionId = executionId, Interval = interval, SendWithKey = sendWithKey, SerializationMode = serializationMode, Tables = tables }; var cdcStreamer = new ChangeStreamer(configuration, cdcReaderClient); cdcStreamer.StartReading(cts.Token, cdcRequest); Console.WriteLine("Streaming to Kafka started."); // wait for shutdown signal #if DEBUG Console.WriteLine("Press any key to shutdown"); Console.ReadKey(); #else starting.Wait(); #endif Console.WriteLine("Received signal gracefully shutting down"); cts.Cancel(); cdcStreamer.WaitForCompletion(); ended.Set(); } }