public ChangeExporter(CdcReaderClient cdcReaderClient,
                       RedshiftClient redshiftClient)
 {
     _cdcReaderClient = cdcReaderClient;
     _redshiftClient  = redshiftClient;
     _readerTasks     = new List <Task>();
 }
Exemple #2
0
        public FullLoadStreamer(IConfiguration configuration, CdcReaderClient cdcReaderClient)
        {
            _cdcReaderClient       = cdcReaderClient;
            _kafkaTopicPrefix      = configuration["TableTopicPrefix"];
            _kafkaBootstrapServers = configuration["KafkaBootstrapServers"];
            _schemaRegistryUrl     = configuration["KafkaSchemaRegistryUrl"];

            _loadTasks = new List <Task>();
        }
 public FullLoadExporter(CdcReaderClient cdcReaderClient, RedshiftClient redshiftClient)
 {
     _cdcReaderClient = cdcReaderClient;
     _redshiftClient  = redshiftClient;
     _loadTasks       = new List <Task>();
 }
Exemple #4
0
        static void Main(string[] args)
        {
            Console.Title = "CDC to Redshift";

            // support graceful shutdown in Docker
            var ended    = new ManualResetEventSlim();
            var starting = new ManualResetEventSlim();

            AssemblyLoadContext.Default.Unloading += ctx =>
            {
                System.Console.WriteLine("Unloading fired");
                starting.Set();
                System.Console.WriteLine("Waiting for completion");
                ended.Wait();
            };

            // set up configuration
            var builder = new ConfigurationBuilder()
                          .SetBasePath(Directory.GetCurrentDirectory())
                          .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                          .AddCommandLine(args)
                          .AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;

            IConfigurationRoot configuration = builder.Build();

            var executionId    = GetExecutionId(configuration);
            var runMode        = GetRunMode(configuration);
            var tables         = GetTables(configuration);
            var batchSize      = GetNonTransactionalTableBatchSize(configuration);
            var redshiftClient = GetRedshiftClient(configuration);

            var cts = new CancellationTokenSource();

            if (runMode == RunMode.FullLoad)
            {
                var cdcReaderClient  = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
                var printMod         = GetPrintMod(configuration);
                var fullLoadExporter = new FullLoadExporter(cdcReaderClient, redshiftClient);
                fullLoadExporter.ExportTablesAsync(cts.Token, executionId, tables, batchSize, printMod).Wait();
                Console.WriteLine("Export to Redshift started.");

                Thread.Sleep(2000);
                bool shutdown = false;
                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");

                while (!shutdown)
                {
                    if (Console.KeyAvailable)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadExporter.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#else
                while (!shutdown)
                {
                    if (starting.IsSet)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadExporter.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                fullLoadExporter.WaitForCompletion();
                ended.Set();
            }
            else if (runMode == RunMode.NonTransactionalCdc)
            {
                var interval        = GetInterval(configuration);
                var cdcReaderClient = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
                var cdcExporter     = new ChangeExporter(cdcReaderClient, redshiftClient);
                cdcExporter.StartExportingChangesAsync(cts.Token, executionId, tables, interval, batchSize).Wait();
                Console.WriteLine("Export to Redshift in progress.");

                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");
                Console.ReadKey();
#else
                starting.Wait();
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                cdcExporter.WaitForCompletion();
                ended.Set();
            }
            else
            {
                var interval            = GetInterval(configuration);
                var cdcReaderClient     = new CdcTransactionClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
                var cdcExporter         = new TransactionExporter(cdcReaderClient, redshiftClient);
                var perTableBufferLimit = GetPerTableBufferLimit(configuration);
                var tranBufferLimit     = GetTransactionBufferLimit(configuration);
                var tranBatchSizeLimit  = GetTransactionBatchSizeLimit(configuration);

                cdcExporter.StartExportingChangesAsync(cts.Token, executionId, tables, interval, perTableBufferLimit, tranBufferLimit, tranBatchSizeLimit).Wait();
                Console.WriteLine("Export to Redshift in progress.");

                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");
                Console.ReadKey();
#else
                starting.Wait();
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                cdcExporter.WaitForCompletion();
                ended.Set();
            }
        }
Exemple #5
0
        static void Main(string[] args)
        {
            Console.Title = "CDC To Kafka Streamer";

            // support graceful shutdown in Docker
            var ended    = new ManualResetEventSlim();
            var starting = new ManualResetEventSlim();

            AssemblyLoadContext.Default.Unloading += ctx =>
            {
                System.Console.WriteLine("Unloading fired");
                starting.Set();
                System.Console.WriteLine("Waiting for completion");
                ended.Wait();
            };

            // set up configuration
            var builder = new ConfigurationBuilder()
                          .SetBasePath(Directory.GetCurrentDirectory())
                          .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
                          .AddCommandLine(args)
                          .AddEnvironmentVariables("CDCTOOLS_"); // all environment variables with this prefix;

            IConfigurationRoot configuration = builder.Build();

            // get parameters and start
            var executionId           = GetExecutionId(configuration);
            var runMode               = GetRunMode(configuration);
            var tables                = GetTables(configuration);
            var serializationMode     = GetSerializationMode(configuration);
            var sendWithKey           = GetSendWithKey(configuration);
            var batchSize             = GetBatchSize(configuration);
            var kafkaBootstrapServers = GetBootstrapServers(configuration);
            var schemaRegistryUrl     = GetSchemaRegistryUrl(configuration);
            var cdcReaderClient       = new CdcReaderClient(configuration["DatabaseConnection"], configuration["StateManagmentConnection"]);
            var cts = new CancellationTokenSource();

            if (runMode == RunMode.FullLoad)
            {
                var printMod         = GetPrintMod(configuration);
                var fullLoadStreamer = new FullLoadStreamer(configuration, cdcReaderClient);
                fullLoadStreamer.StreamTablesAsync(cts.Token, executionId, tables, serializationMode, sendWithKey, batchSize, printMod).Wait();
                Console.WriteLine("Streaming to Kafka in progress.");

                Thread.Sleep(2000);
                bool shutdown = false;
                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");

                while (!shutdown)
                {
                    if (Console.KeyAvailable)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadStreamer.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#else
                while (!shutdown)
                {
                    if (starting.IsSet)
                    {
                        shutdown = true;
                    }
                    else if (fullLoadStreamer.HasFinished())
                    {
                        shutdown = true;
                    }

                    Thread.Sleep(500);
                }
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                fullLoadStreamer.WaitForCompletion();
                ended.Set();
            }
            else
            {
                var interval = GetInterval(configuration);

                var cdcRequest = new CdcRequest()
                {
                    BatchSize         = batchSize,
                    ExecutionId       = executionId,
                    Interval          = interval,
                    SendWithKey       = sendWithKey,
                    SerializationMode = serializationMode,
                    Tables            = tables
                };
                var cdcStreamer = new ChangeStreamer(configuration, cdcReaderClient);
                cdcStreamer.StartReading(cts.Token, cdcRequest);
                Console.WriteLine("Streaming to Kafka started.");

                // wait for shutdown signal
#if DEBUG
                Console.WriteLine("Press any key to shutdown");
                Console.ReadKey();
#else
                starting.Wait();
#endif

                Console.WriteLine("Received signal gracefully shutting down");
                cts.Cancel();
                cdcStreamer.WaitForCompletion();
                ended.Set();
            }
        }