Ejemplo n.º 1
0
        static void Main(string[] args)
        {
            Parser.Default.ParseArguments <Options>(args).WithParsed(a =>
            {
                var loggerConfiguration = new LoggerConfiguration()
                                          .MinimumLevel.Verbose()
                                          .WriteTo.Console(theme: Theme);
                if (a.Log)
                {
                    loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log");
                }

                Log.Logger = loggerConfiguration.CreateLogger();
                Log.Logger.Information($"Options: {string.Join(" ", args)}");
                var mode = a.Mode.ToLower();
                if (mode != "parallel" && mode != "sequential")
                {
                    Log.Logger.Error("Only support two mode: parallel, sequential.");
                    return;
                }
                if (AppContext.GetData("WELCOME") == null)
                {
                    Console.ForegroundColor = ConsoleColor.Green;
                    Console.WriteLine("=============================================================================================");
                    Console.WriteLine("======                ClickHouse Migrator V1.0 MIT   [email protected]               =======");
                    Console.WriteLine("=============================================================================================");
                    Console.ForegroundColor = ConsoleColor.White;
                }
                if (string.IsNullOrWhiteSpace(a.TargetDatabase))
                {
                    a.TargetDatabase = a.SourceDatabase;
                }
                if (string.IsNullOrWhiteSpace(a.TargetTable))
                {
                    a.TargetTable = a.SourceTable;
                }
                if (string.IsNullOrWhiteSpace(a.SourceHost))
                {
                    a.SourceHost = "127.0.0.1";
                }
                if (string.IsNullOrWhiteSpace(a.User))
                {
                    a.User = "******";
                }
                if (a.Batch < 1000)
                {
                    a.Batch = 1000;
                    Log.Logger.Warning("Batch should not less than 1000.");
                }
                var start    = DateTime.Now;
                var migrator = MigratorFactory.Create(a);
                migrator.Run();
                var end = DateTime.Now;
                Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
            }).WithNotParsed(errors =>
            {
            });
        }
Ejemplo n.º 2
0
        static void Main(string[] args)
        {
            args =
                "--source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourcedb test --sourcetb ch -h localhost --targetdb test --targettb ch --thread 1 -b 2000 --drop true --log true"
                .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);

            Parser.Default.ParseArguments <Options>(args).WithParsed(a =>
            {
                var loggerConfiguration = new LoggerConfiguration()
                                          .MinimumLevel.Verbose()
                                          .WriteTo.Console(theme: Theme);
                if (a.Log)
                {
                    loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log");
                }

                Log.Logger = loggerConfiguration.CreateLogger();
                Log.Logger.Information($"Options: {string.Join(" ", args)}");
                var mode = a.Mode.ToLower();
                if (mode != "parallel" && mode != "sequential")
                {
                    Log.Logger.Error("Only support two mode: parallel, sequential.");
                    return;
                }
                if (AppContext.GetData("WELCOME") == null)
                {
                    Console.ForegroundColor = ConsoleColor.Green;
                    Console.WriteLine("=============================================================================================");
                    Console.WriteLine("======                ClickHouse Migrator V1.0 MIT   [email protected]               =======");
                    Console.WriteLine("=============================================================================================");
                    Console.ForegroundColor = ConsoleColor.White;
                }
                if (string.IsNullOrWhiteSpace(a.TargetDatabase))
                {
                    a.TargetDatabase = a.SourceDatabase;
                }
                if (string.IsNullOrWhiteSpace(a.TargetTable))
                {
                    a.TargetTable = a.SourceTable;
                }
                if (string.IsNullOrWhiteSpace(a.SourceHost))
                {
                    a.SourceHost = "127.0.0.1";
                }
                if (string.IsNullOrWhiteSpace(a.User))
                {
                    a.User = "******";
                }
                if (a.Batch < 1000)
                {
                    a.Batch = 1000;
                    Log.Logger.Warning("Batch should not less than 1000.");
                }
                //preventing SQL Exception about "The server supports a maximum of 2000 parameters"
                if (a.Source == "mssql" && a.Batch > 2000)
                {
                    a.Batch = 2000;
                    Log.Logger.Warning("Unfortunally on MsSQL server Batch size  should not greater than 2000.");
                }
                var start    = DateTime.Now;
                var migrator = MigratorFactory.Create(a);
                migrator.Run();
                var end = DateTime.Now;
                Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
            }).WithNotParsed(errors =>
            {
            });
        }
Ejemplo n.º 3
0
        static async Task Main(string[] args)
        {
            Logger.Information($"Options: {string.Join(" ", Environment.GetCommandLineArgs())}");

            if (!AppContext.TryGetSwitch("WELCOME", out var printWelcome) && !printWelcome)
            {
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine(Line);
                Console.WriteLine(
                    "ClickHouse Migrator V1.0.8 MIT   [email protected]");
                Console.WriteLine(@"
--src            : data source: MySql, SqlServer, Excel etc
--src-host       : host of data source, for example: 192.168.90.100, **default value: 127.0.0.1**
--src-port       : port of data source, for example: 3306
--src-user       : user of data source
--src-password   : password of data source
--src-database   : database of data source
--src-table      : table of data source
--host           : Clickhouse host: 192.168.90.101, **default value: 127.0.0.1**
--port           : Clickhouse port, for example: 9000, **default value: 9000**
--user           : Clickhouse user
--password       : Clickhouse password
--database       : Clickhouse database, if this arg is null, will use --src-database as target database name
--table          : Clickhouse table, if this arg is null, will use --src-table as target table name
--thread         : how many thread use to insert data to ClickHouse, **default value: process count of your machine**
--batch          : how many rows insert to ClickHouse one time, **default value: 10000**
--drop-table     : whether drop the exits table in clickhouse before migrating, **default value: false**
--file           : File path of Excel etc
--sheets         : Which sheets will be migrated, columns are same in every sheet, used like: Sheet1,Sheet2,Sheet3
--start-row      : 
--lowercase      : ignore the word case in clickhouse, **default value: true**
");
                Console.WriteLine(Line);
                Console.ForegroundColor = ConsoleColor.White;
                AppContext.SetSwitch("WELCOME", true);
            }

            if (args.Length == 0)
            {
                return;
            }

            if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development" || args.Contains("test"))
            {
                args =
                    "--src mysql --src-host 192.168.192.2 --src-port 3306 --src-user root --src-password 1qazZAQ! --src-database test --src-table user1000w --drop-table true"
                    .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);

                // args =
                //  "--src excel --file Book.xlsx --database test --table t1 --start-row 2 --sheet-start 2 --drop-table true"
                //      .Split(new[] {" "}, StringSplitOptions.RemoveEmptyEntries);
            }

            try
            {
                var dataSource = GetDataSource(args);
                var migrator   = MigratorFactory.Create(dataSource);
                await migrator.RunAsync(args);
            }
            catch (Exception e)
            {
                Logger.Error(e.Message);
            }
        }
Ejemplo n.º 4
0
        static void Main(string[] args)
        {
            if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development")
            {
                args =
                    "--lowercase true --source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourced cnblogs --sourcet Cnblogs_Entity_Model -h localhost --targetd cnblogs --targett Cnblogs_Entity_Model --thread 1 -b 2000 --drop true --log false"
                    .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
            }

            var options = BuildOptions(args);

            ConfigureLog(options);

            var mode = options.Mode.ToLower();

            if (mode != "parallel" && mode != "sequential")
            {
                Log.Logger.Error("Only support two modes: parallel, sequential.");
                return;
            }

            if (!AppContext.TryGetSwitch("WELCOME", out bool printWelcome) && !printWelcome)
            {
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine(
                    "==========================================================================================================================================");
                Console.WriteLine(
                    "======                                    ClickHouse Migrator V1.0.7 MIT   [email protected]                                      =======");
                Console.WriteLine(@"
--source    or -s    : which RDBMS you want to migrate, right now i implement mysql migrator, for example: mysql
--shost     or -sh   : host of RDBMS, for example: 192.168.90.100, **default value: 127.0.0.1**
--sport     or -sport: port of RDBMS, for example: 3306
--suser     or -su   : user of RDBMS
--spass     or -sp   : password of RDBMS
--host      or -h    : host of Clickhouse for example: 192.168.90.101, **default value: 127.0.0.1**
--port      or -port : port of clickhouse, for example: 9000, **default value: 9000**
--user      or -u    : user of Clickhouse
--pass      or -p    : password of Clickhouse
--thread    or -t    : how many thread use to read data from mysql, **default value: process count of your machine**
--batch     or -b    : how many rows read from mysql one time and submit to clickhouse, **default value: 5000**
--sourced   or -sd   : database of RDBMS
--sourcet   or -st   : table of RDBMS which you want to migrate
--targetd   or -td   : migrate data to which target database in clickhouse, create it if not exists
--targett   or -tt   : migrate data to which target table in clickhouse, create it if not exists
--drop      or -d    : whether drop the exits table in clickhouse before migrating, **default value: false**
--lowercase or -lc   : ignore the word case in clickhouse, **default value: true**
--orderby   or -o    : when order by is null, use primary as order by in clickhouse, if use orderby, then will miss primary
--trace     or -t    : record performance information, **default value: false**
--mode      or -m    : migrate mode, parallel or sequential, when use sequential thread argument are useless, **default value: parallel**
--log       or -l    : whether write file log, **default value: false**
");
                Console.WriteLine(
                    "==========================================================================================================================================");
                Console.ForegroundColor = ConsoleColor.White;
                AppContext.SetSwitch("WELCOME", true);
            }

            var start    = DateTime.Now;
            var migrator = MigratorFactory.Create(options);

            migrator.Run();
            var end = DateTime.Now;

            Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
        }