示例#1
0
 public void StartReading(CancellationToken token, CdcRequest cdcRequest)
 {
     foreach (var tableName in cdcRequest.Tables)
     {
         var readerTask = Task.Run(async() =>
         {
             try
             {
                 await StartPublishingChanges(token,
                                              cdcRequest.ExecutionId,
                                              tableName,
                                              cdcRequest.Interval,
                                              cdcRequest.BatchSize,
                                              cdcRequest.SendWithKey,
                                              cdcRequest.SerializationMode);
             }
             catch (Exception ex)
             {
                 Console.WriteLine($"CDC reader failure. Table {tableName}. Error: {ex}");
             }
         });
         _readerTasks.Add(readerTask);
     }
 }
示例#2
0
        static void Main(string[] args)
        {
            Console.Title = "CDC To Kafka Streamer";

            // support graceful shutdown in Docker
            var ended    = new ManualResetEventSlim();
            var starting = new ManualResetEventSlim();

            AssemblyLoadContext.Default.Unloading += ctx =>
            {
                System.Console.WriteLine("Unloading fired");
                starting.Set();
                System.Console.WriteLine("Waiting for completion");
                ended.Wait();
            };

            // set up configuration
            var builder = new ConfigurationBuilder()
                          .SetBasePath(Directory.GetCurrentDirectory())
                          .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                          .AddCommandLine(args)
                          .AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;

            IConfigurationRoot configuration = builder.Build();

            // get parameters and start
            var executionId           = GetExecutionId(configuration);
            var runMode               = GetRunMode(configuration);
            var tables                = GetTables(configuration);
            var serializationMode     = GetSerializationMode(configuration);
            var sendWithKey           = GetSendWithKey(configuration);
            var batchSize             = GetBatchSize(configuration);
            var kafkaBootstrapServers = GetBootstrapServers(configuration);
            var schemaRegistryUrl     = GetSchemaRegistryUrl(configuration);
            var cdcReaderClient       = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
            var cts = new CancellationTokenSource();

            if (runMode == RunMode.FullLoad)
            {
                var printMod         = GetPrintMod(configuration);
                var fullLoadStreamer = new FullLoadStreamer(configuration, cdcReaderClient);
                fullLoadStreamer.StreamTablesAsync(cts.Token, executionId, tables, serializationMode, sendWithKey, batchSize, printMod).Wait();
                Console.WriteLine("Streaming to Kafka in progress.");

                Thread.Sleep(2000);
                bool shutdown = false;
                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");

                while (!shutdown)
                {
                    if (Console.KeyAvailable)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadStreamer.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#else
                while (!shutdown)
                {
                    if (starting.IsSet)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadStreamer.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                fullLoadStreamer.WaitForCompletion();
                ended.Set();
            }
            else
            {
                var interval = GetInterval(configuration);

                var cdcRequest = new CdcRequest()
                {
                    BatchSize         = batchSize,
                    ExecutionId       = executionId,
                    Interval          = interval,
                    SendWithKey       = sendWithKey,
                    SerializationMode = serializationMode,
                    Tables            = tables
                };
                var cdcStreamer = new ChangeStreamer(configuration, cdcReaderClient);
                cdcStreamer.StartReading(cts.Token, cdcRequest);
                Console.WriteLine("Streaming to Kafka started.");

                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");
                Console.ReadKey();
#else
                starting.Wait();
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                cdcStreamer.WaitForCompletion();
                ended.Set();
            }
        }