static void Main(string[] args) { Parser.Default.ParseArguments <Options>(args).WithParsed(a => { var loggerConfiguration = new LoggerConfiguration() .MinimumLevel.Verbose() .WriteTo.Console(theme: Theme); if (a.Log) { loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log"); } Log.Logger = loggerConfiguration.CreateLogger(); Log.Logger.Information($"Options: {string.Join(" ", args)}"); var mode = a.Mode.ToLower(); if (mode != "parallel" && mode != "sequential") { Log.Logger.Error("Only support two mode: parallel, sequential."); return; } if (AppContext.GetData("WELCOME") == null) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("============================================================================================="); Console.WriteLine("====== ClickHouse Migrator V1.0 MIT [email protected] ======="); Console.WriteLine("============================================================================================="); Console.ForegroundColor = ConsoleColor.White; } if (string.IsNullOrWhiteSpace(a.TargetDatabase)) { a.TargetDatabase = a.SourceDatabase; } if (string.IsNullOrWhiteSpace(a.TargetTable)) { a.TargetTable = a.SourceTable; } if (string.IsNullOrWhiteSpace(a.SourceHost)) { a.SourceHost = "127.0.0.1"; } if (string.IsNullOrWhiteSpace(a.User)) { a.User = "******"; } if (a.Batch < 1000) { a.Batch = 1000; Log.Logger.Warning("Batch should not less than 1000."); } var start = DateTime.Now; var migrator = MigratorFactory.Create(a); migrator.Run(); var end = DateTime.Now; Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s."); }).WithNotParsed(errors => { }); }
static void Main(string[] args) { args = "--source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourcedb test --sourcetb ch -h localhost --targetdb test --targettb ch --thread 1 -b 2000 --drop true --log true" .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); Parser.Default.ParseArguments <Options>(args).WithParsed(a => { var loggerConfiguration = new LoggerConfiguration() .MinimumLevel.Verbose() .WriteTo.Console(theme: Theme); if (a.Log) { loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log"); } Log.Logger = loggerConfiguration.CreateLogger(); Log.Logger.Information($"Options: {string.Join(" ", args)}"); var mode = a.Mode.ToLower(); if (mode != "parallel" && mode != "sequential") { Log.Logger.Error("Only support two mode: parallel, sequential."); return; } if (AppContext.GetData("WELCOME") == null) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("============================================================================================="); Console.WriteLine("====== ClickHouse Migrator V1.0 MIT [email protected] ======="); Console.WriteLine("============================================================================================="); Console.ForegroundColor = ConsoleColor.White; } if (string.IsNullOrWhiteSpace(a.TargetDatabase)) { a.TargetDatabase = a.SourceDatabase; } if (string.IsNullOrWhiteSpace(a.TargetTable)) { a.TargetTable = a.SourceTable; } if (string.IsNullOrWhiteSpace(a.SourceHost)) { a.SourceHost = "127.0.0.1"; } if (string.IsNullOrWhiteSpace(a.User)) { a.User = "******"; } if (a.Batch < 1000) { a.Batch = 1000; Log.Logger.Warning("Batch should not less than 1000."); } //preventing SQL Exception about "The server supports a maximum of 2000 parameters" if (a.Source == "mssql" && a.Batch > 2000) { a.Batch = 2000; Log.Logger.Warning("Unfortunally on MsSQL server Batch size should not greater than 2000."); } var start = DateTime.Now; var migrator = MigratorFactory.Create(a); migrator.Run(); var end = DateTime.Now; Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s."); }).WithNotParsed(errors => { }); }
static async Task Main(string[] args) { Logger.Information($"Options: {string.Join(" ", Environment.GetCommandLineArgs())}"); if (!AppContext.TryGetSwitch("WELCOME", out var printWelcome) && !printWelcome) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine(Line); Console.WriteLine( "ClickHouse Migrator V1.0.8 MIT [email protected]"); Console.WriteLine(@" --src : data source: MySql, SqlServer, Excel etc --src-host : host of data source, for example: 192.168.90.100, **default value: 127.0.0.1** --src-port : port of data source, for example: 3306 --src-user : user of data source --src-password : password of data source --src-database : database of data source --src-table : table of data source --host : Clickhouse host: 192.168.90.101, **default value: 127.0.0.1** --port : Clickhouse port, for example: 9000, **default value: 9000** --user : Clickhouse user --password : Clickhouse password --database : Clickhouse database, if this arg is null, will use --src-database as target database name --table : Clickhouse table, if this arg is null, will use --src-table as target table name --thread : how many thread use to insert data to ClickHouse, **default value: process count of your machine** --batch : how many rows insert to ClickHouse one time, **default value: 10000** --drop-table : whether drop the exits table in clickhouse before migrating, **default value: false** --file : File path of Excel etc --sheets : Which sheets will be migrated, columns are same in every sheet, used like: Sheet1,Sheet2,Sheet3 --start-row : --lowercase : ignore the word case in clickhouse, **default value: true** "); Console.WriteLine(Line); Console.ForegroundColor = ConsoleColor.White; AppContext.SetSwitch("WELCOME", true); } if (args.Length == 0) { return; } if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development" || args.Contains("test")) { args = "--src mysql --src-host 192.168.192.2 --src-port 3306 --src-user root --src-password 1qazZAQ! --src-database test --src-table user1000w --drop-table true" .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); // args = // "--src excel --file Book.xlsx --database test --table t1 --start-row 2 --sheet-start 2 --drop-table true" // .Split(new[] {" "}, StringSplitOptions.RemoveEmptyEntries); } try { var dataSource = GetDataSource(args); var migrator = MigratorFactory.Create(dataSource); await migrator.RunAsync(args); } catch (Exception e) { Logger.Error(e.Message); } }
static void Main(string[] args) { if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development") { args = "--lowercase true --source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourced cnblogs --sourcet Cnblogs_Entity_Model -h localhost --targetd cnblogs --targett Cnblogs_Entity_Model --thread 1 -b 2000 --drop true --log false" .Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); } var options = BuildOptions(args); ConfigureLog(options); var mode = options.Mode.ToLower(); if (mode != "parallel" && mode != "sequential") { Log.Logger.Error("Only support two modes: parallel, sequential."); return; } if (!AppContext.TryGetSwitch("WELCOME", out bool printWelcome) && !printWelcome) { Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine( "=========================================================================================================================================="); Console.WriteLine( "====== ClickHouse Migrator V1.0.7 MIT [email protected] ======="); Console.WriteLine(@" --source or -s : which RDBMS you want to migrate, right now i implement mysql migrator, for example: mysql --shost or -sh : host of RDBMS, for example: 192.168.90.100, **default value: 127.0.0.1** --sport or -sport: port of RDBMS, for example: 3306 --suser or -su : user of RDBMS --spass or -sp : password of RDBMS --host or -h : host of Clickhouse for example: 192.168.90.101, **default value: 127.0.0.1** --port or -port : port of clickhouse, for example: 9000, **default value: 9000** --user or -u : user of Clickhouse --pass or -p : password of Clickhouse --thread or -t : how many thread use to read data from mysql, **default value: process count of your machine** --batch or -b : how many rows read from mysql one time and submit to clickhouse, **default value: 5000** --sourced or -sd : database of RDBMS --sourcet or -st : table of RDBMS which you want to migrate --targetd or -td : migrate data to which target database in clickhouse, create it if not exists --targett or -tt : migrate data to which target table in clickhouse, create it if not exists --drop or -d : whether drop the exits table in clickhouse before migrating, **default value: false** --lowercase or -lc : ignore the word case in clickhouse, **default value: true** --orderby or -o : when order by is null, use primary as order by in clickhouse, if use orderby, then will miss primary --trace or -t : record performance information, **default value: false** --mode or -m : migrate mode, parallel or sequential, when use sequential thread argument are useless, **default value: parallel** --log or -l : whether write file log, **default value: false** "); Console.WriteLine( "=========================================================================================================================================="); Console.ForegroundColor = ConsoleColor.White; AppContext.SetSwitch("WELCOME", true); } var start = DateTime.Now; var migrator = MigratorFactory.Create(options); migrator.Run(); var end = DateTime.Now; Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s."); }