private BulkCopyInfo GetBulkCopyInfo(Table table, SchemaInfo schemaInfo, DbTransaction transaction = null) { string tableName = this.GetMappedTableName(table.Name); BulkCopyInfo bulkCopyInfo = new BulkCopyInfo() { KeepIdentity = this.Target.DbInterpreter.Option.TableScriptsGenerateOption.GenerateIdentity, DestinationTableOwner = this.Target.DbOwner, DestinationTableName = tableName, Columns = schemaInfo.TableColumns.Where(item => item.TableName == tableName), Transaction = transaction, CancellationToken = this.CancellationTokenSource.Token }; return(bulkCopyInfo); }
private DataTable ConvertDataTable(DataTable dataTable, BulkCopyInfo bulkCopyInfo) { var columns = dataTable.Columns.Cast <DataColumn>(); if (!columns.Any(item => item.DataType == typeof(TimeSpan) || item.DataType == typeof(byte[]) || item.DataType == typeof(decimal))) { return(dataTable); } Dictionary <int, Type> changedColumnTypes = new Dictionary <int, Type>(); Dictionary <(int RowIndex, int ColumnIndex), object> changedValues = new Dictionary <(int RowIndex, int ColumnIndex), object>(); DataTable dtChanged = dataTable.Clone(); int rowIndex = 0; Func <DataColumn, TableColumn> getTableColumn = (column) => { return(bulkCopyInfo.Columns.FirstOrDefault(item => item.Name == column.ColumnName)); }; foreach (DataRow row in dataTable.Rows) { for (int i = 0; i < dataTable.Columns.Count; i++) { object value = row[i]; if (value != null) { Type type = value.GetType(); if (type != typeof(DBNull)) { if (type == typeof(TimeSpan)) { TimeSpan ts = TimeSpan.Parse(value.ToString()); if (ts.Days > 0) { TableColumn tableColumn = getTableColumn(dataTable.Columns[i]); string dataType = tableColumn.DataType.ToLower(); Type columnType = null; if (dataType.Contains("datetime")) { DateTime dateTime = this.MinDateTime.AddSeconds(ts.TotalSeconds); columnType = typeof(DateTime); changedValues.Add((rowIndex, i), dateTime); } else if (DataTypeHelper.IsCharType(dataType)) { columnType = typeof(string); changedValues.Add((rowIndex, i), ts.ToString()); } if (columnType != null && !changedColumnTypes.ContainsKey(i)) { changedColumnTypes.Add(i, columnType); } } } else if (type == typeof(byte[])) { TableColumn tableColumn = getTableColumn(dataTable.Columns[i]); if (tableColumn.DataType.ToLower() == "uniqueidentifier") { changedValues.Add((rowIndex, i), ValueHelper.ConvertGuidBytesToString(value as byte[], this.DatabaseType, tableColumn.DataType, tableColumn.MaxLength, true)); if (!changedColumnTypes.ContainsKey(i)) { changedColumnTypes.Add(i, typeof(Guid)); } } } else if (type == typeof(decimal)) { TableColumn tableColumn = getTableColumn(dataTable.Columns[i]); string dataType = tableColumn.DataType.ToLower(); Type columnType = null; if (dataType == "bigint") { columnType = typeof(Int64); } else if (dataType == "int") { columnType = typeof(Int32); if ((decimal)value > Int32.MaxValue) { columnType = typeof(Int64); } } else if (dataType == "smallint") { columnType = typeof(Int16); if ((decimal)value > Int16.MaxValue) { columnType = typeof(Int32); } } if (columnType != null && !changedColumnTypes.ContainsKey(i)) { changedColumnTypes.Add(i, columnType); } } } } } rowIndex++; } if (changedColumnTypes.Count == 0) { return(dataTable); } for (int i = 0; i < dtChanged.Columns.Count; i++) { if (changedColumnTypes.ContainsKey(i)) { dtChanged.Columns[i].DataType = changedColumnTypes[i]; } } rowIndex = 0; foreach (DataRow row in dataTable.Rows) { DataRow r = dtChanged.NewRow(); for (int i = 0; i < dataTable.Columns.Count; i++) { var value = row[i]; if (changedValues.ContainsKey((rowIndex, i))) { r[i] = changedValues[(rowIndex, i)]; }
public override async Task BulkCopyAsync(DbConnection connection, DataTable dataTable, BulkCopyInfo bulkCopyInfo) { SqlBulkCopy bulkCopy = await this.GetBulkCopy(connection, bulkCopyInfo); { await bulkCopy.WriteToServerAsync(this.ConvertDataTable(dataTable, bulkCopyInfo), bulkCopyInfo.CancellationToken); } }
public abstract Task BulkCopyAsync(DbConnection connection, DataTable dataTable, BulkCopyInfo bulkCopyInfo);
private DataTable ConvertDataTable(DataTable dataTable, BulkCopyInfo bulkCopyInfo) { var columns = dataTable.Columns.Cast <DataColumn>(); if (!columns.Any(item => item.DataType == typeof(MySql.Data.Types.MySqlDateTime))) { return(dataTable); } Dictionary <int, Type> changedColumnTypes = new Dictionary <int, Type>(); Dictionary <(int RowIndex, int ColumnIndex), object> changedValues = new Dictionary <(int RowIndex, int ColumnIndex), object>(); DataTable dtChanged = dataTable.Clone(); int rowIndex = 0; foreach (DataRow row in dataTable.Rows) { for (int i = 0; i < dataTable.Columns.Count; i++) { object value = row[i]; if (value != null) { Type type = value.GetType(); if (type != typeof(DBNull)) { if (type == typeof(MySql.Data.Types.MySqlDateTime)) { MySql.Data.Types.MySqlDateTime mySqlDateTime = (MySql.Data.Types.MySqlDateTime)value; TableColumn tableColumn = bulkCopyInfo.Columns.FirstOrDefault(item => item.Name == dataTable.Columns[i].ColumnName); string dataType = tableColumn.DataType.ToLower(); Type columnType = null; if (dataType.Contains("date") || dataType.Contains("timestamp")) { DateTime dateTime = mySqlDateTime.GetDateTime(); columnType = typeof(DateTime); changedValues.Add((rowIndex, i), dateTime); } if (columnType != null && !changedColumnTypes.ContainsKey(i)) { changedColumnTypes.Add(i, columnType); } } } } } rowIndex++; } if (changedColumnTypes.Count == 0) { return(dataTable); } for (int i = 0; i < dtChanged.Columns.Count; i++) { if (changedColumnTypes.ContainsKey(i)) { dtChanged.Columns[i].DataType = changedColumnTypes[i]; } } rowIndex = 0; foreach (DataRow row in dataTable.Rows) { DataRow r = dtChanged.NewRow(); for (int i = 0; i < dataTable.Columns.Count; i++) { var value = row[i]; if (changedValues.ContainsKey((rowIndex, i))) { r[i] = changedValues[(rowIndex, i)]; }
public override async Task BulkCopyAsync(DbConnection connection, DataTable dataTable, BulkCopyInfo bulkCopyInfo) { if (!(connection is OracleConnection conn)) { return; } if (conn.State != ConnectionState.Open) { await conn.OpenAsync(); } using (var bulkCopy = new OracleBulkCopy(conn, bulkCopyInfo.Transaction as OracleTransaction)) { bulkCopy.BatchSize = dataTable.Rows.Count; bulkCopy.DestinationTableName = this.GetQuotedString(bulkCopyInfo.DestinationTableName); bulkCopy.BulkCopyTimeout = bulkCopyInfo.Timeout.HasValue ? bulkCopyInfo.Timeout.Value : SettingManager.Setting.CommandTimeout;; bulkCopy.ColumnNameNeedQuoted = this.DbObjectNameMode == DbObjectNameMode.WithQuotation; bulkCopy.DetectDateTimeTypeByValues = bulkCopyInfo.DetectDateTimeTypeByValues; await bulkCopy.WriteToServerAsync(this.ConvertDataTable(dataTable, bulkCopyInfo)); } }
private DataTable ConvertDataTable(DataTable dataTable, BulkCopyInfo bulkCopyInfo) { bool hasSpecialColumn = false; foreach (DataColumn column in dataTable.Columns) { if (DataTypeHelper.SpecialDataTypes.Contains(column.DataType.Name)) { hasSpecialColumn = true; break; } } if (hasSpecialColumn) { Dictionary <string, Type> dictColumnTypes = new Dictionary <string, Type>(); DataTable dtSpecial = dataTable.Clone(); foreach (DataColumn column in dtSpecial.Columns) { if (DataTypeHelper.SpecialDataTypes.Contains(column.DataType.Name)) { TableColumn tableColumn = bulkCopyInfo.Columns.FirstOrDefault(item => item.Name == column.ColumnName); string dataType = tableColumn.DataType.ToLower(); Type columnType = null; if (DataTypeHelper.IsCharType(dataType) || DataTypeHelper.IsTextType(dataType)) { columnType = typeof(string); } else if (DataTypeHelper.IsBinaryType(dataType) || dataType.ToLower().Contains("blob")) { columnType = typeof(Byte[]); } if (columnType != null) { column.DataType = columnType; dictColumnTypes[column.ColumnName] = columnType; } } } foreach (DataRow row in dataTable.Rows) { DataRow r = dtSpecial.NewRow(); for (int i = 0; i < dataTable.Columns.Count; i++) { var value = row[i]; if (dictColumnTypes.ContainsKey(dataTable.Columns[i].ColumnName)) { Type type = dictColumnTypes[dataTable.Columns[i].ColumnName]; if (type == typeof(string)) { r[i] = value == null ? null : (type == typeof(string) ? value?.ToString() : Convert.ChangeType(value, type)); } else { r[i] = value; } } else { r[i] = value; } } dtSpecial.Rows.Add(r); } return(dtSpecial); } return(dataTable); }
public override async Task BulkCopyAsync(DbConnection connection, DataTable dataTable, BulkCopyInfo bulkCopyInfo) { if (dataTable == null || dataTable.Rows.Count <= 0) { return; } MySqlBulkCopy bulkCopy = new MySqlBulkCopy(connection as MySqlConnection, bulkCopyInfo.Transaction as MySqlTransaction); bulkCopy.DestinationTableName = bulkCopyInfo.DestinationTableName; await this.OpenConnectionAsync(connection); await bulkCopy.WriteToServerAsync(this.ConvertDataTable(dataTable, bulkCopyInfo), bulkCopyInfo.CancellationToken); }
private async Task InternalConvert(SchemaInfo schemaInfo = null) { DbInterpreter sourceInterpreter = this.Source.DbInterpreter; sourceInterpreter.Option.BulkCopy = this.Option.BulkCopy; sourceInterpreter.Subscribe(this.observer); sourceInterpreter.Option.GetTableAllObjects = false; sourceInterpreter.Option.ThrowExceptionWhenErrorOccurs = false; this.Target.DbInterpreter.Option.ThrowExceptionWhenErrorOccurs = false; if (string.IsNullOrEmpty(this.Target.DbOwner)) { if (this.Target.DbInterpreter.DatabaseType == DatabaseType.Oracle) { this.Target.DbOwner = (this.Target.DbInterpreter as OracleInterpreter).GetDbOwner(); } } DatabaseObjectType databaseObjectType = (DatabaseObjectType)Enum.GetValues(typeof(DatabaseObjectType)).Cast <int>().Sum(); if (schemaInfo != null && !this.Source.DbInterpreter.Option.GetTableAllObjects && (schemaInfo.TableTriggers == null || schemaInfo.TableTriggers.Count == 0)) { databaseObjectType = databaseObjectType ^ DatabaseObjectType.TableTrigger; } SchemaInfoFilter filter = new SchemaInfoFilter() { Strict = true, DatabaseObjectType = databaseObjectType }; SchemaInfoHelper.SetSchemaInfoFilterValues(filter, schemaInfo); SchemaInfo sourceSchemaInfo = await sourceInterpreter.GetSchemaInfoAsync(filter); if (sourceInterpreter.HasError) { return; } sourceSchemaInfo.TableColumns = DbObjectHelper.ResortTableColumns(sourceSchemaInfo.Tables, sourceSchemaInfo.TableColumns); if (SettingManager.Setting.NotCreateIfExists) { this.Target.DbInterpreter.Option.GetTableAllObjects = false; SchemaInfo targetSchema = await this.Target.DbInterpreter.GetSchemaInfoAsync(filter); SchemaInfoHelper.ExcludeExistingObjects(sourceSchemaInfo, targetSchema); } #region Set data type by user define type List <UserDefinedType> utypes = new List <UserDefinedType>(); if (sourceInterpreter.DatabaseType != this.Target.DbInterpreter.DatabaseType) { utypes = await sourceInterpreter.GetUserDefinedTypesAsync(); if (utypes != null && utypes.Count > 0) { foreach (TableColumn column in sourceSchemaInfo.TableColumns) { UserDefinedType utype = utypes.FirstOrDefault(item => item.Name == column.DataType); if (utype != null) { column.DataType = utype.Type; column.MaxLength = utype.MaxLength; } } } } #endregion SchemaInfo targetSchemaInfo = SchemaInfoHelper.Clone(sourceSchemaInfo); if (this.Source.TableNameMappings != null && this.Source.TableNameMappings.Count > 0) { SchemaInfoHelper.MapTableNames(targetSchemaInfo, this.Source.TableNameMappings); } if (this.Option.RenameTableChildren) { SchemaInfoHelper.RenameTableChildren(targetSchemaInfo); } if (this.Option.IgnoreNotSelfForeignKey) { targetSchemaInfo.TableForeignKeys = targetSchemaInfo.TableForeignKeys.Where(item => item.TableName == item.ReferencedTableName).ToList(); } #region Translate TranslateEngine translateEngine = new TranslateEngine(sourceSchemaInfo, targetSchemaInfo, sourceInterpreter, this.Target.DbInterpreter, this.Option, this.Target.DbOwner); translateEngine.SkipError = this.Option.SkipScriptError || this.Option.OnlyForTranslate; DatabaseObjectType translateDbObjectType = TranslateEngine.SupportDatabaseObjectType; if (!this.Option.GenerateScriptMode.HasFlag(GenerateScriptMode.Schema) && this.Option.BulkCopy && this.Target.DbInterpreter.SupportBulkCopy) { translateDbObjectType = DatabaseObjectType.TableColumn; } translateEngine.UserDefinedTypes = utypes; translateEngine.OnTranslated += this.Translated; translateEngine.Subscribe(this.observer); translateEngine.Translate(translateDbObjectType); if (this.Option.OnlyForTranslate) { if (targetSchemaInfo.Tables.Count == 0 && targetSchemaInfo.UserDefinedTypes.Count == 0) { return; } } #endregion if (targetSchemaInfo.Tables.Any()) { if (this.Option.EnsurePrimaryKeyNameUnique) { SchemaInfoHelper.EnsurePrimaryKeyNameUnique(targetSchemaInfo); } if (this.Option.EnsureIndexNameUnique) { SchemaInfoHelper.EnsureIndexNameUnique(targetSchemaInfo); } } DbInterpreter targetInterpreter = this.Target.DbInterpreter; bool generateIdentity = targetInterpreter.Option.TableScriptsGenerateOption.GenerateIdentity; if (generateIdentity) { targetInterpreter.Option.InsertIdentityValue = true; } string script = ""; targetInterpreter.Subscribe(this.observer); ScriptBuilder scriptBuilder = null; DbScriptGenerator targetDbScriptGenerator = DbScriptGeneratorHelper.GetDbScriptGenerator(targetInterpreter); if (this.Option.GenerateScriptMode.HasFlag(GenerateScriptMode.Schema)) { scriptBuilder = targetDbScriptGenerator.GenerateSchemaScripts(targetSchemaInfo); if (targetSchemaInfo.Tables.Any()) { this.Translated(targetInterpreter.DatabaseType, targetSchemaInfo.Tables.First(), new TranslateResult() { Data = scriptBuilder.ToString() }); } } if (this.Option.OnlyForTranslate) { return; } DataTransferErrorProfile dataErrorProfile = null; using (DbConnection dbConnection = this.Option.ExecuteScriptOnTargetServer ? targetInterpreter.CreateConnection() : null) { this.isBusy = true; if (this.Option.ExecuteScriptOnTargetServer && this.Option.UseTransaction) { dbConnection.Open(); this.transaction = dbConnection.BeginTransaction(); } #region Schema sync if (scriptBuilder != null && this.Option.ExecuteScriptOnTargetServer) { List <Script> scripts = scriptBuilder.Scripts; if (scripts.Count == 0) { this.Feedback(targetInterpreter, $"The script to create schema is empty.", FeedbackInfoType.Info); this.isBusy = false; return; } targetInterpreter.Feedback(FeedbackInfoType.Info, "Begin to sync schema..."); try { if (!this.Option.SplitScriptsToExecute) { targetInterpreter.Feedback(FeedbackInfoType.Info, script); await targetInterpreter.ExecuteNonQueryAsync(dbConnection, this.GetCommandInfo(script, null, this.transaction)); } else { Func <Script, bool> isValidScript = (s) => { return(!(s is NewLineSript || s is SpliterScript || string.IsNullOrEmpty(s.Content) || s.Content == targetInterpreter.ScriptsDelimiter)); }; int count = scripts.Where(item => isValidScript(item)).Count(); int i = 0; foreach (Script s in scripts) { if (targetInterpreter.HasError) { break; } if (!isValidScript(s)) { continue; } bool isCreateScript = s.ObjectType == nameof(Function) || s.ObjectType == nameof(Procedure) || s.ObjectType == nameof(TableTrigger); bool skipError = this.Option.SkipScriptError && isCreateScript; string sql = s.Content?.Trim(); if (!string.IsNullOrEmpty(sql) && sql != targetInterpreter.ScriptsDelimiter) { i++; if (!isCreateScript && targetInterpreter.ScriptsDelimiter.Length == 1 && sql.EndsWith(targetInterpreter.ScriptsDelimiter)) { sql = sql.TrimEnd(targetInterpreter.ScriptsDelimiter.ToArray()); } if (!targetInterpreter.HasError) { targetInterpreter.Feedback(FeedbackInfoType.Info, $"({i}/{count}), executing:{Environment.NewLine} {sql}"); CommandInfo commandInfo = this.GetCommandInfo(sql, null, transaction); commandInfo.SkipError = skipError; await targetInterpreter.ExecuteNonQueryAsync(dbConnection, commandInfo); } } } } } catch (Exception ex) { targetInterpreter.CancelRequested = true; this.Rollback(); ConnectionInfo sourceConnectionInfo = sourceInterpreter.ConnectionInfo; ConnectionInfo targetConnectionInfo = targetInterpreter.ConnectionInfo; SchemaTransferException schemaTransferException = new SchemaTransferException(ex) { SourceServer = sourceConnectionInfo.Server, SourceDatabase = sourceConnectionInfo.Database, TargetServer = targetConnectionInfo.Server, TargetDatabase = targetConnectionInfo.Database }; this.HandleError(schemaTransferException); } targetInterpreter.Feedback(FeedbackInfoType.Info, "End sync schema."); } #endregion #region Data sync if (!targetInterpreter.HasError && this.Option.GenerateScriptMode.HasFlag(GenerateScriptMode.Data) && sourceSchemaInfo.Tables.Count > 0) { List <TableColumn> identityTableColumns = new List <TableColumn>(); if (generateIdentity) { identityTableColumns = targetSchemaInfo.TableColumns.Where(item => item.IsIdentity).ToList(); } if (this.Option.PickupTable) { dataErrorProfile = DataTransferErrorProfileManager.GetProfile(sourceInterpreter.ConnectionInfo, targetInterpreter.ConnectionInfo); if (dataErrorProfile != null) { sourceSchemaInfo.PickupTable = new Table() { Owner = schemaInfo.Tables.FirstOrDefault()?.Owner, Name = dataErrorProfile.SourceTableName }; } } await this.SetIdentityEnabled(identityTableColumns, targetInterpreter, targetDbScriptGenerator, dbConnection, transaction, false); if (this.Option.ExecuteScriptOnTargetServer || targetInterpreter.Option.ScriptOutputMode.HasFlag(GenerateScriptOutputMode.WriteToFile)) { Dictionary <Table, long> dictTableDataTransferredCount = new Dictionary <Table, long>(); sourceInterpreter.OnDataRead += async(TableDataReadInfo tableDataReadInfo) => { if (!this.hasError) { Table table = tableDataReadInfo.Table; List <TableColumn> columns = tableDataReadInfo.Columns; try { (Table Table, List <TableColumn> Columns)targetTableAndColumns = this.GetTargetTableColumns(targetSchemaInfo, this.Target.DbOwner, table, columns); if (targetTableAndColumns.Table == null || targetTableAndColumns.Columns == null) { return; } if (this.Option.ExecuteScriptOnTargetServer) { DataTable dataTable = tableDataReadInfo.DataTable; List <Dictionary <string, object> > data = tableDataReadInfo.Data; if (this.Option.BulkCopy && targetInterpreter.SupportBulkCopy) { BulkCopyInfo bulkCopyInfo = this.GetBulkCopyInfo(table, targetSchemaInfo, this.transaction); if (targetInterpreter.DatabaseType == DatabaseType.Oracle) { if (columns.Any(item => item.DataType.ToLower().Contains("datetime2") || item.DataType.ToLower().Contains("timestamp"))) { bulkCopyInfo.DetectDateTimeTypeByValues = true; } } if (this.Option.ConvertComputeColumnExpression) { IEnumerable <DataColumn> dataColumns = dataTable.Columns.OfType <DataColumn>(); foreach (TableColumn column in bulkCopyInfo.Columns) { if (column.IsComputed && dataColumns.Any(item => item.ColumnName == column.Name)) { dataTable.Columns.Remove(column.Name); } } } await targetInterpreter.BulkCopyAsync(dbConnection, dataTable, bulkCopyInfo); } else { StringBuilder sb = new StringBuilder(); Dictionary <string, object> paramters = targetDbScriptGenerator.AppendDataScripts(sb, targetTableAndColumns.Table, targetTableAndColumns.Columns, new Dictionary <long, List <Dictionary <string, object> > >() { { 1, data } }); script = sb.ToString().Trim().Trim(';'); await targetInterpreter.ExecuteNonQueryAsync(dbConnection, this.GetCommandInfo(script, paramters, this.transaction)); } if (!dictTableDataTransferredCount.ContainsKey(table)) { dictTableDataTransferredCount.Add(table, dataTable.Rows.Count); } else { dictTableDataTransferredCount[table] += dataTable.Rows.Count; } long transferredCount = dictTableDataTransferredCount[table]; double percent = (transferredCount * 1.0 / tableDataReadInfo.TotalCount) * 100; string strPercent = (percent == (int)percent) ? (percent + "%") : (percent / 100).ToString("P2"); targetInterpreter.FeedbackInfo($"Table \"{table.Name}\":{dataTable.Rows.Count} records transferred.({transferredCount}/{tableDataReadInfo.TotalCount},{strPercent})"); } } catch (Exception ex) { sourceInterpreter.CancelRequested = true; this.Rollback(); ConnectionInfo sourceConnectionInfo = sourceInterpreter.ConnectionInfo; ConnectionInfo targetConnectionInfo = targetInterpreter.ConnectionInfo; string mappedTableName = this.GetMappedTableName(table.Name); DataTransferException dataTransferException = new DataTransferException(ex) { SourceServer = sourceConnectionInfo.Server, SourceDatabase = sourceConnectionInfo.Database, SourceObject = table.Name, TargetServer = targetConnectionInfo.Server, TargetDatabase = targetConnectionInfo.Database, TargetObject = mappedTableName }; this.HandleError(dataTransferException); if (!this.Option.UseTransaction) { DataTransferErrorProfileManager.Save(new DataTransferErrorProfile { SourceServer = sourceConnectionInfo.Server, SourceDatabase = sourceConnectionInfo.Database, SourceTableName = table.Name, TargetServer = targetConnectionInfo.Server, TargetDatabase = targetConnectionInfo.Database, TargetTableName = mappedTableName }); } } } }; } DbScriptGenerator sourceDbScriptGenerator = DbScriptGeneratorHelper.GetDbScriptGenerator(sourceInterpreter); await sourceDbScriptGenerator.GenerateDataScriptsAsync(sourceSchemaInfo); await this.SetIdentityEnabled(identityTableColumns, targetInterpreter, targetDbScriptGenerator, dbConnection, transaction, true); } #endregion if (this.transaction != null && this.transaction.Connection != null && !this.cancelRequested) { this.transaction.Commit(); } this.isBusy = false; } if (dataErrorProfile != null && !this.hasError && !this.cancelRequested) { DataTransferErrorProfileManager.Remove(dataErrorProfile); } }