static void Main(string[] args)
{
Parser.Default.ParseArguments <Options>(args).WithParsed(a =>
{
var loggerConfiguration = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.Console(theme: Theme);
if (a.Log)
{
loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log");
}
Log.Logger = loggerConfiguration.CreateLogger();
Log.Logger.Information($"Options: {string.Join(" ", args)}");
var mode = a.Mode.ToLower();
if (mode != "parallel" && mode != "sequential")
{
Log.Logger.Error("Only support two mode: parallel, sequential.");
return;
}
if (AppContext.GetData("WELCOME") == null)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=============================================================================================");
Console.WriteLine("====== ClickHouse Migrator V1.0 MIT [email protected] =======");
Console.WriteLine("=============================================================================================");
Console.ForegroundColor = ConsoleColor.White;
}
if (string.IsNullOrWhiteSpace(a.TargetDatabase))
{
a.TargetDatabase = a.SourceDatabase;
}
if (string.IsNullOrWhiteSpace(a.TargetTable))
{
a.TargetTable = a.SourceTable;
}
if (string.IsNullOrWhiteSpace(a.SourceHost))
{
a.SourceHost = "127.0.0.1";
}
if (string.IsNullOrWhiteSpace(a.User))
{
a.User = "******";
}
if (a.Batch < 1000)
{
a.Batch = 1000;
Log.Logger.Warning("Batch should not less than 1000.");
}
var start = DateTime.Now;
var migrator = MigratorFactory.Create(a);
migrator.Run();
var end = DateTime.Now;
Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
}).WithNotParsed(errors =>
{
});
}
static void Main(string[] args)
{
args =
"--source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourcedb test --sourcetb ch -h localhost --targetdb test --targettb ch --thread 1 -b 2000 --drop true --log true"
.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
Parser.Default.ParseArguments <Options>(args).WithParsed(a =>
{
var loggerConfiguration = new LoggerConfiguration()
.MinimumLevel.Verbose()
.WriteTo.Console(theme: Theme);
if (a.Log)
{
loggerConfiguration = loggerConfiguration.WriteTo.RollingFile("ClickHouseMigrator.log");
}
Log.Logger = loggerConfiguration.CreateLogger();
Log.Logger.Information($"Options: {string.Join(" ", args)}");
var mode = a.Mode.ToLower();
if (mode != "parallel" && mode != "sequential")
{
Log.Logger.Error("Only support two mode: parallel, sequential.");
return;
}
if (AppContext.GetData("WELCOME") == null)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("=============================================================================================");
Console.WriteLine("====== ClickHouse Migrator V1.0 MIT [email protected] =======");
Console.WriteLine("=============================================================================================");
Console.ForegroundColor = ConsoleColor.White;
}
if (string.IsNullOrWhiteSpace(a.TargetDatabase))
{
a.TargetDatabase = a.SourceDatabase;
}
if (string.IsNullOrWhiteSpace(a.TargetTable))
{
a.TargetTable = a.SourceTable;
}
if (string.IsNullOrWhiteSpace(a.SourceHost))
{
a.SourceHost = "127.0.0.1";
}
if (string.IsNullOrWhiteSpace(a.User))
{
a.User = "******";
}
if (a.Batch < 1000)
{
a.Batch = 1000;
Log.Logger.Warning("Batch should not less than 1000.");
}
//preventing SQL Exception about "The server supports a maximum of 2000 parameters"
if (a.Source == "mssql" && a.Batch > 2000)
{
a.Batch = 2000;
Log.Logger.Warning("Unfortunally on MsSQL server Batch size should not greater than 2000.");
}
var start = DateTime.Now;
var migrator = MigratorFactory.Create(a);
migrator.Run();
var end = DateTime.Now;
Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
}).WithNotParsed(errors =>
{
});
}
static async Task Main(string[] args)
{
Logger.Information($"Options: {string.Join(" ", Environment.GetCommandLineArgs())}");
if (!AppContext.TryGetSwitch("WELCOME", out var printWelcome) && !printWelcome)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(Line);
Console.WriteLine(
"ClickHouse Migrator V1.0.8 MIT [email protected]");
Console.WriteLine(@"
--src : data source: MySql, SqlServer, Excel etc
--src-host : host of data source, for example: 192.168.90.100, **default value: 127.0.0.1**
--src-port : port of data source, for example: 3306
--src-user : user of data source
--src-password : password of data source
--src-database : database of data source
--src-table : table of data source
--host : Clickhouse host: 192.168.90.101, **default value: 127.0.0.1**
--port : Clickhouse port, for example: 9000, **default value: 9000**
--user : Clickhouse user
--password : Clickhouse password
--database : Clickhouse database, if this arg is null, will use --src-database as target database name
--table : Clickhouse table, if this arg is null, will use --src-table as target table name
--thread : how many thread use to insert data to ClickHouse, **default value: process count of your machine**
--batch : how many rows insert to ClickHouse one time, **default value: 10000**
--drop-table : whether drop the exits table in clickhouse before migrating, **default value: false**
--file : File path of Excel etc
--sheets : Which sheets will be migrated, columns are same in every sheet, used like: Sheet1,Sheet2,Sheet3
--start-row :
--lowercase : ignore the word case in clickhouse, **default value: true**
");
Console.WriteLine(Line);
Console.ForegroundColor = ConsoleColor.White;
AppContext.SetSwitch("WELCOME", true);
}
if (args.Length == 0)
{
return;
}
if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development" || args.Contains("test"))
{
args =
"--src mysql --src-host 192.168.192.2 --src-port 3306 --src-user root --src-password 1qazZAQ! --src-database test --src-table user1000w --drop-table true"
.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
// args =
// "--src excel --file Book.xlsx --database test --table t1 --start-row 2 --sheet-start 2 --drop-table true"
// .Split(new[] {" "}, StringSplitOptions.RemoveEmptyEntries);
}
try
{
var dataSource = GetDataSource(args);
var migrator = MigratorFactory.Create(dataSource);
await migrator.RunAsync(args);
}
catch (Exception e)
{
Logger.Error(e.Message);
}
}
static void Main(string[] args)
{
if (Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == "Development")
{
args =
"--lowercase true --source sqlserver --shost localhost --suser sa --spass 1qazZAQ! --sport 1433 --sourced cnblogs --sourcet Cnblogs_Entity_Model -h localhost --targetd cnblogs --targett Cnblogs_Entity_Model --thread 1 -b 2000 --drop true --log false"
.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
}
var options = BuildOptions(args);
ConfigureLog(options);
var mode = options.Mode.ToLower();
if (mode != "parallel" && mode != "sequential")
{
Log.Logger.Error("Only support two modes: parallel, sequential.");
return;
}
if (!AppContext.TryGetSwitch("WELCOME", out bool printWelcome) && !printWelcome)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(
"==========================================================================================================================================");
Console.WriteLine(
"====== ClickHouse Migrator V1.0.7 MIT [email protected] =======");
Console.WriteLine(@"
--source or -s : which RDBMS you want to migrate, right now i implement mysql migrator, for example: mysql
--shost or -sh : host of RDBMS, for example: 192.168.90.100, **default value: 127.0.0.1**
--sport or -sport: port of RDBMS, for example: 3306
--suser or -su : user of RDBMS
--spass or -sp : password of RDBMS
--host or -h : host of Clickhouse for example: 192.168.90.101, **default value: 127.0.0.1**
--port or -port : port of clickhouse, for example: 9000, **default value: 9000**
--user or -u : user of Clickhouse
--pass or -p : password of Clickhouse
--thread or -t : how many thread use to read data from mysql, **default value: process count of your machine**
--batch or -b : how many rows read from mysql one time and submit to clickhouse, **default value: 5000**
--sourced or -sd : database of RDBMS
--sourcet or -st : table of RDBMS which you want to migrate
--targetd or -td : migrate data to which target database in clickhouse, create it if not exists
--targett or -tt : migrate data to which target table in clickhouse, create it if not exists
--drop or -d : whether drop the exits table in clickhouse before migrating, **default value: false**
--lowercase or -lc : ignore the word case in clickhouse, **default value: true**
--orderby or -o : when order by is null, use primary as order by in clickhouse, if use orderby, then will miss primary
--trace or -t : record performance information, **default value: false**
--mode or -m : migrate mode, parallel or sequential, when use sequential thread argument are useless, **default value: parallel**
--log or -l : whether write file log, **default value: false**
");
Console.WriteLine(
"==========================================================================================================================================");
Console.ForegroundColor = ConsoleColor.White;
AppContext.SetSwitch("WELCOME", true);
}
var start = DateTime.Now;
var migrator = MigratorFactory.Create(options);
migrator.Run();
var end = DateTime.Now;
Log.Logger.Information($"Complete migrate: {(end - start).TotalSeconds} s.");
}