protected virtual BulkCopyRowsCopied RowByRowCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { // This limitation could be lifted later for some providers that supports identity insert if we will get such request // It will require support from DataConnection.Insert if (options.KeepIdentity == true) { throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by {nameof(BulkCopyType)}.{nameof(BulkCopyType.RowByRow)} mode"); } var rowsCopied = new BulkCopyRowsCopied(); foreach (var item in source) { dataConnection.Insert(item, options.TableName, options.DatabaseName, options.SchemaName); rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } } return(rowsCopied); }
protected virtual BulkCopyRowsCopied RowByRowCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { var rowsCopied = new BulkCopyRowsCopied(); foreach (var item in source) { dataConnection.Insert(item, options.TableName, options.DatabaseName, options.SchemaName); rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (table.DataContext is DataConnection dataConnection) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); var transaction = dataConnection.Transaction; if (connection != null && transaction != null) { transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema); } if (connection != null && (dataConnection.Transaction == null || transaction != null)) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = new BulkCopyReader(dataConnection, columns, source); var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.SqlRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField))); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (!(table?.DataContext is DataConnection dataConnection)) { throw new ArgumentNullException(nameof(dataConnection)); } var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); var transaction = dataConnection.Transaction; if (connection != null && transaction != null) { transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema); } if (connection != null && (dataConnection.Transaction == null || transaction != null)) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rc = new BulkCopyRowsCopied(); var hanaOptions = SapHanaProviderAdapter.HanaBulkCopyOptions.Default; if (options.KeepIdentity == true) { hanaOptions |= SapHanaProviderAdapter.HanaBulkCopyOptions.KeepIdentity; } using (var bc = _provider.Adapter.CreateBulkCopy(connection, hanaOptions, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.HanaRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema); var tableName = GetTableName(sqlBuilder, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, columns[i].ColumnName)); } var rd = new BulkCopyReader(dataConnection, columns, source); TraceAction( dataConnection, () => "INSERT BULK " + tableName + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken) where T : notnull { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection == null) { return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)); } var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length]; for (var i = 0; i < columns.Length; i++) { var npgsqlType = _provider.GetNativeType(columns[i].DbType, true); if (npgsqlType == null) { var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null; if (columnType == null || columnType.Type.DataType == DataType.Undefined) { columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType); } var sb = new System.Text.StringBuilder(); sqlBuilder.BuildTypeName(sb, columnType); npgsqlType = _provider.GetNativeType(sb.ToString(), true); } if (npgsqlType == null) { throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping."); } npgsqlTypes[i] = npgsqlType.Value; } var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); if (!writer.SupportsAsync) { // seems to be missing one of the required async methods; fallback to sync importer var enumerator = source.GetAsyncEnumerator(cancellationToken); await using (enumerator.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { return(ProviderSpecificCopySyncImpl(dataConnection, options, EnumerableHelper.AsyncToSyncEnumerable(enumerator), connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, writer)); } } var rowsCopied = new BulkCopyRowsCopied(); var currentCount = 0; try { await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); for (var i = 0; i < columns.Length; i++) { await writer.WriteAsync(columns[i].GetValue(item !), npgsqlTypes[i], cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } if (currentCount >= batchSize) { await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { await TraceActionAsync( dataConnection, () => "INSERT ASYNC BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, async() => { var ret = await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); return((int)rowsCopied.RowsCopied); }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } finally { await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (!(table?.DataContext is DataConnection dataConnection)) { throw new ArgumentNullException(nameof(dataConnection)); } if (dataConnection.Connection is DbConnection dbConnection) { if (Proxy.GetUnderlyingObject(dbConnection) is SqlConnection connection) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _dataProvider.CreateSqlBuilder(); var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source); var sqlopt = SqlBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SqlBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SqlBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SqlBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SqlBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SqlBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SqlBulkCopyOptions.UseInternalTransaction; } using (var bc = new SqlBulkCopy(connection, sqlopt, (SqlTransaction)dataConnection.Transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.SqlRowsCopied += (sender, e) => { rc.RowsCopied = e.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { e.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var tableName = GetTableName(sqlBuilder, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(new SqlBulkCopyColumnMapping( i, sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField).ToString())); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", bc.ColumnMappings.Cast <SqlBulkCopyColumnMapping>().Select(x => x.DestinationColumn)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyInternal <T>( ProviderConnections providerConnections, ITable <T> table, BulkCopyOptions options, Func <List <Mapping.ColumnDescriptor>, BulkCopyReader <T> > createDataReader, bool runAsync, CancellationToken cancellationToken) where T : notnull { var dataConnection = providerConnections.DataConnection; var connection = providerConnections.ProviderConnection; var transaction = providerConnections.ProviderTransaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = createDataReader(columns); var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.SqlRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField))); } await TraceActionAsync( dataConnection, () => (runAsync ? "INSERT ASYNC BULK " : "INSERT BULK ") + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine, async() => { if (runAsync) { await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } else { bc.WriteToServer(rd); } return(rd.Count); }, true).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyInternal <T>( ProviderConnections providerConnections, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken) { var dataConnection = providerConnections.DataConnection; var connection = providerConnections.ProviderConnection; var transaction = providerConnections.ProviderTransaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rc = new BulkCopyRowsCopied(); var bc = _provider.Adapter.BulkCopy !.Create(connection, transaction); if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.MySqlRowsCopied += (sender, args) => { rc.RowsCopied += args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = GetTableName(sb, options, table); for (var i = 0; i < columns.Count; i++) { bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } // emulate missing BatchSize property // this is needed, because MySql fails on big batches, so users should be able to limit batch size var batches = EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue); await foreach (var batch in batches.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { var rd = new BulkCopyReader <T>(dataConnection, columns, batch, cancellationToken); await TraceActionAsync( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, async() => { if (bc.CanWriteToServerAsync2) { await bc.WriteToServerAsync2(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } else if (bc.CanWriteToServerAsync) { await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } else { bc.WriteToServer(rd); } return(rd.Count); }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); rc.RowsCopied += rd.Count; } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } return(rc); }
protected BulkCopyRowsCopied IDSProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, DataConnection dataConnection, IDbConnection connection, InformixProviderAdapter.BulkCopyAdapter bulkCopy) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = new BulkCopyReader(dataConnection, columns, source); var sqlopt = InformixProviderAdapter.IfxBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.KeepIdentity == true) { sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.TableLock; } using (var bc = bulkCopy.Create(connection, sqlopt)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.IfxRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField))); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (table.DataContext is DataConnection dataConnection && _provider.Adapter.BulkCopy != null) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); // for run in transaction see // https://stackoverflow.com/questions/57675379 // provider will call sp_oledb_columns which creates temp table var transaction = dataConnection.Transaction; if (connection != null && transaction != null) { transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema); } if (connection != null && (dataConnection.Transaction == null || transaction != null)) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = new BulkCopyReader(dataConnection, columns, source); var sqlopt = SybaseProviderAdapter.AseBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.AseRowsCopied += (sender, args) => { rc.RowsCopied = args.RowCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); // do not convert table and column names to valid sql name, as sybase bulk copy implementation // doesn't understand escaped names (facepalm) // which means it will probably fail when escaping required anyways... bc.DestinationTableName = GetDestinationTableName(sb, options, table); for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(columns[i].ColumnName, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (!(table?.DataContext is DataConnection dataConnection)) { throw new ArgumentNullException(nameof(dataConnection)); } if (dataConnection.Transaction == null) { var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); if (_bulkCopyCreator == null) { var bulkCopyType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopy", false); var bulkCopyOptionType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyOptions", false); var columnMappingType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyColumnMapping", false); if (bulkCopyType != null) { _bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator != null) { var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source); var rc = new BulkCopyRowsCopied(); var bcOptions = 0; // Default if (options.KeepIdentity == true) { bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4, } if (options.TableLock == true) { bcOptions |= 2; } using (var bc = _bulkCopyCreator(Proxy.GetUnderlyingObject((DbConnection)dataConnection.Connection), bcOptions)) { dynamic dbc = bc; var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2RowsCopied"); } dbc.NotifyAfter = notifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied MultipleRowsCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, descriptor); var iszOS = _dataProvider.Version == DB2Version.zOS; var rowsCopied = new BulkCopyRowsCopied(); var sb = new StringBuilder(); var buildValue = BasicSqlBuilder.GetBuildValue(sqlBuilder, sb); var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToArray(); var pname = sqlBuilder.Convert("p", ConvertType.NameToQueryParameter).ToString(); sb .AppendFormat("INSERT INTO {0}", tableName).AppendLine() .Append("("); foreach (var column in columns) { sb .AppendLine() .Append("\t") .Append(sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField)) .Append(","); } sb.Length--; sb .AppendLine() .Append(")"); if (!iszOS) { sb .AppendLine() .Append("VALUES"); } var headerLen = sb.Length; var currentCount = 0; var batchSize = options.MaxBatchSize ?? 1000; if (batchSize <= 0) { batchSize = 1000; } var parms = new List <DataParameter>(); var pidx = 0; foreach (var item in source) { sb .AppendLine() .Append(iszOS ? "SELECT " : "("); foreach (var column in columns) { var value = column.GetValue(item); if (value == null) { sb.Append("NULL"); } else { switch (Type.GetTypeCode(value.GetType())) { case TypeCode.DBNull: sb.Append("NULL"); break; case TypeCode.String: var isString = false; switch (column.DataType) { case DataType.NVarChar: case DataType.Char: case DataType.VarChar: case DataType.NChar: case DataType.Undefined: isString = true; break; } if (isString) { goto case TypeCode.Int32; } goto default; case TypeCode.Boolean: case TypeCode.Char: case TypeCode.SByte: case TypeCode.Byte: case TypeCode.Int16: case TypeCode.UInt16: case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: case TypeCode.DateTime: //SetParameter(dataParam, "", column.DataType, value); buildValue(value); break; default: var name = pname + ++pidx; sb.Append(name); parms.Add(new DataParameter("p" + pidx, value, column.DataType)); break; } } sb.Append(","); } sb.Length--; sb.Append(iszOS ? " FROM SYSIBM.SYSDUMMY1 UNION ALL" : "),"); rowsCopied.RowsCopied++; currentCount++; if (currentCount >= batchSize || parms.Count > 100000 || sb.Length > 100000) { if (iszOS) { sb.Length -= " UNION ALL".Length; } else { sb.Length--; } dataConnection.Execute(sb.AppendLine().ToString(), parms.ToArray()); if (options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { return(rowsCopied); } } parms.Clear(); pidx = 0; currentCount = 0; sb.Length = headerLen; } } if (currentCount > 0) { if (iszOS) { sb.Length -= " UNION ALL".Length; } else { sb.Length--; } dataConnection.Execute(sb.ToString(), parms.ToArray()); sb.Length = headerLen; if (options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } return(rowsCopied); }
private BulkCopyRowsCopied ProviderSpecificCopyImpl_AccessClient <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, DataConnection dataConnection, IDbConnection connection, DB2iSeriesAccessClientProviderAdapter adapter, Action <DataConnection, Func <string>, Func <int> > traceAction) { var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rd = new BulkCopyReader <T>(dataConnection, columns, source); var rc = new BulkCopyRowsCopied(); var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema); var tableName = GetTableName(sqlBuilder, options, table); var columnNames = columns.Select(x => x.ColumnName); var fields = string.Join(", ", columnNames); var parameters = string.Join(", ", columnNames.Select(x => "@" + x)); var sql = $"INSERT INTO {tableName} ({fields}) VALUES({parameters})"; var batchSize = Math.Min(options.MaxBatchSize ?? MAX_ACCESS_CLIENT_BATCH_SIZE, MAX_ACCESS_CLIENT_BATCH_SIZE); var sw = Stopwatch.StartNew(); using (var cmd = connection.CreateCommand()) { cmd.CommandText = sql; adapter.DeriveParameters(cmd); var columnDataTypes = cmd.Parameters.Cast <IDbDataParameter>() .Select(adapter.GetDbType) .Select(adapter.GetDataType) .ToList(); //*LOB and XML types not supported, fallback to multiple rows if (columnDataTypes.Any(x => x.In(DataType.Blob, DataType.Text, DataType.NText, DataType.Xml))) { return(MultipleRowsCopy(table, options, source)); } var columnDbDataTypes = columns .Select((x, i) => x .GetDbDataType(true) .WithDataType(columnDataTypes[i])) .ToList(); var count = 0; var notificationBatchCount = 0; var bufferEmpty = false; while (rd.Read()) { var i = 0; bufferEmpty = false; foreach (IDbDataParameter parameter in cmd.Parameters) { dataConnection.DataProvider.SetParameter( dataConnection, parameter, columns[i].ColumnName, columnDbDataTypes[i], rd.GetValue(i++)); } try { adapter.AddBatch(cmd); } //In case columns can't be mapped fall back to multiple rows catch { return(MultipleRowsCopy(table, options, source)); } count++; notificationBatchCount++; if (count % batchSize == 0) { execute(); if (rc.Abort) { return(rc); } bufferEmpty = true; } } if (!bufferEmpty) { execute(); if (rc.Abort) { return(rc); } } void execute() { cmd.ExecuteNonQuery(); rc.RowsCopied = count; if (sw.Elapsed.TotalSeconds > options.BulkCopyTimeout) { traceAction( dataConnection, () => $"INSERT BULK {tableName} TIMEOUT AFTER {sw.Elapsed.TotalSeconds} SECONDS" + Environment.NewLine, () => (int)rc.RowsCopied); rc.Abort = true; return; } if (options.NotifyAfter > 0 && notificationBatchCount > options.NotifyAfter) { options.RowsCopiedCallback?.Invoke(rc); notificationBatchCount = 0; } if (rc.Abort) { traceAction( dataConnection, () => $"INSERT BULK {tableName} ABORTED BY USER" + Environment.NewLine, () => (int)rc.RowsCopied); return; } } } return(rc); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken) where T : notnull { var connection = _provider.TryGetProviderConnection(dataConnection, dataConnection.Connection); if (connection == null) { return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)); } var sqlBuilder = (PostgreSQLSqlBuilder)_provider.CreateSqlBuilder(table.DataContext.MappingSchema); var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var(npgsqlTypes, dbTypes, columnTypes) = BuildTypes(_provider.Adapter, sqlBuilder, columns); var writer = _provider.Adapter.BeginBinaryImportAsync != null ? await _provider.Adapter.BeginBinaryImportAsync(connection, copyCommand, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext) : _provider.Adapter.BeginBinaryImport(connection, copyCommand); if (!writer.SupportsAsync) { // seems to be missing one of the required async methods; fallback to sync importer var enumerator = source.GetAsyncEnumerator(cancellationToken); await using (enumerator.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { return(ProviderSpecificCopySyncImpl(dataConnection, options, EnumerableHelper.AsyncToSyncEnumerable(enumerator), connection, tableName, columns, columnTypes, npgsqlTypes, dbTypes, copyCommand, batchSize, writer)); } } var rowsCopied = new BulkCopyRowsCopied(); var currentCount = 0; try { await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); for (var i = 0; i < columns.Length; i++) { if (npgsqlTypes[i] != null) { await writer.WriteAsync(_provider.NormalizeTimeStamp(columns[i].GetProviderValue(item !), columnTypes[i]), npgsqlTypes[i] !.Value, cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } else { await writer.WriteAsync(_provider.NormalizeTimeStamp(columns[i].GetProviderValue(item !), columnTypes[i]), dbTypes[i] !, cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } if (currentCount >= batchSize) { await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); writer = _provider.Adapter.BeginBinaryImportAsync != null ? await _provider.Adapter.BeginBinaryImportAsync(connection, copyCommand, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext) : _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { await TraceActionAsync( dataConnection, () => $"INSERT ASYNC BULK {tableName}({string.Join(", ", columns.Select(x => x.ColumnName))}){Environment.NewLine}", async() => { var ret = await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); return((int)rowsCopied.RowsCopied); }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } finally { await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } var connection = dataConnection.Connection; if (connection == null) { return(MultipleRowsCopy(dataConnection, options, source)); } if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOf(_connectionType))) { return(MultipleRowsCopy(dataConnection, options, source)); } var transaction = dataConnection.Transaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rc = new BulkCopyRowsCopied(); if (_bulkCopyCreator == null) { var clientNamespace = _dataProvider.ConnectionNamespace; var bulkCopyType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopy", false); var bulkCopyOptionType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopyOptions", false); var columnMappingType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false); var transactionType = _connectionType.Assembly.GetType(clientNamespace + ".HanaTransaction", false); if (bulkCopyType != null) { _bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType, transactionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator == null) { return(MultipleRowsCopy(dataConnection, options, source)); } const int hanaOptions = 0; //default; using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction)) { dynamic dbc = bc; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied"); } dbc.NotifyAfter = options.NotifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.MaxBatchSize.HasValue) { dbc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, descriptor); dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } var rd = new BulkCopyReader(_dataProvider, columns, source); TraceAction( dataConnection, "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (table.DataContext is DataConnection dataConnection && dataConnection.Transaction == null && _provider.Adapter.BulkCopy != null) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection != null) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = new BulkCopyReader <T>(dataConnection, columns, source); var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); var tableName = GetTableName(sb, options, table); if (options.UseInternalTransaction == true) { sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt)) { var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.OracleRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (table.DataContext is DataConnection dataConnection) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection == null) { return(MultipleRowsCopy(table, options, source)); } var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; var rowsCopied = new BulkCopyRowsCopied(); // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var currentCount = 0; var key = new { Type = typeof(T), options.KeepIdentity, ed }; var rowWriter = (Action <MappingSchema, NpgsqlProviderAdapter.NpgsqlBinaryImporter, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd( key, _ => _provider.Adapter.CreateBinaryImportRowWriter <T>(_provider, sqlBuilder, columns, dataConnection.MappingSchema)); var useComplete = _provider.Adapter.BinaryImporterHasComplete; var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); try { foreach (var item in source) { rowWriter(dataConnection.MappingSchema, writer, columns, item); currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { if (!useComplete) { writer.Cancel(); } break; } } if (currentCount >= batchSize) { if (useComplete) { writer.Complete(); } writer.Dispose(); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { if (useComplete) { writer.Complete(); } return((int)rowsCopied.RowsCopied); }); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } catch when(!useComplete) { writer.Cancel(); throw; } finally { writer.Dispose(); } return(rowsCopied); } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } if (dataConnection.Transaction == null) { var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, descriptor); if (_bulkCopyCreator == null) { throw new NotImplementedException("Not able to do bulk copy in DB2iSeries Provider."); } if (_bulkCopyCreator != null) { var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source); var rc = new BulkCopyRowsCopied(); var bcOptions = 0; // Default if (options.KeepIdentity == true) { bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4, } if (options.TableLock == true) { bcOptions |= 2; } using (var bc = _bulkCopyCreator(dataConnection.Connection, bcOptions)) { dynamic dbc = bc; var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2iSeriesRowsCopied"); } dbc.NotifyAfter = notifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } dbc.WriteToServer(rd); } rc.RowsCopied = rd.Count; return(rc); } } return(MultipleRowsCopy(dataConnection, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { // database name is not a part of table FQN in oracle var serverName = options.ServerName ?? table.ServerName; if (table.DataContext is DataConnection dataConnection && _provider.Adapter.BulkCopy != null && serverName == null) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection != null) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); // ODP.NET doesn't bulk copy doesn't work if columns that require escaping: // - if escaping applied, pre-flight validation fails as it performs uppercase comparison and quotes make it fail with // InvalidOperationException: Column mapping is invalid // - if escaping not applied - if fails as expected on server, because it treats passed name as uppercased name // and gives "ORA-00904: "STRINGVALUE": invalid identifier" error // That's quite common error in bulk copy implementation error by providers... var supported = true; foreach (var column in columns) { if (column.ColumnName != sb.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)) { // fallback to sql-based copy // TODO: we should add support for by-ordinal column mapping to workaround it supported = false; break; } } if (supported) { var rd = new BulkCopyReader <T>(dataConnection, columns, source); var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); var tableName = sb.ConvertInline(options.TableName ?? table.TableName, ConvertType.NameToQueryTable); var schemaName = options.SchemaName ?? table.SchemaName; if (schemaName != null) { schemaName = sb.ConvertInline(schemaName, ConvertType.NameToSchema); } if (options.UseInternalTransaction == true) { sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt)) { var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.OracleRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } bc.DestinationTableName = tableName; bc.DestinationSchemaName = schemaName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + (schemaName == null ? tableName : schemaName + "." + tableName) + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } } return(MultipleRowsCopy(table, options, source)); }
internal static BulkCopyRowsCopied ProviderSpecificCopyImpl <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, DataConnection dataConnection, DbConnection connection, DB2ProviderAdapter.BulkCopyAdapter bulkCopy, Action <DataConnection, Func <string>, Func <int> > traceAction) where T : notnull { var descriptor = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rd = new BulkCopyReader <T>(dataConnection, columns, source); var rc = new BulkCopyRowsCopied(); var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(table.DataContext.MappingSchema); var tableName = GetTableName(sqlBuilder, options, table); var bcOptions = DB2BulkCopyOptions.Default; if (options.KeepIdentity == true) { bcOptions |= DB2BulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { bcOptions |= DB2BulkCopyOptions.TableLock; } using (var bc = bulkCopy.Create(connection, bcOptions)) { var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = notifyAfter; bc.DB2RowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sqlBuilder.ConvertInline(columns[i].ColumnName, SqlProvider.ConvertType.NameToQueryField))); } traceAction( dataConnection, () => "INSERT BULK " + tableName + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException(nameof(dataConnection)); } var connection = dataConnection.Connection; if (connection == null) { return(MultipleRowsCopy(dataConnection, options, source)); } if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType))) { return(MultipleRowsCopy(dataConnection, options, source)); } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, ed); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var writerType = _connectionType.AssemblyEx().GetType("Npgsql.NpgsqlBinaryImporter", true); var fields = string.Join(", ", columns.Select(column => sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; var rowsCopied = new BulkCopyRowsCopied(); // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var currentCount = 0; var dc = (dynamic)connection; var key = new { Type = typeof(T), options.KeepIdentity, ed }; var rowWriter = (Action <MappingSchema, object, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd( key, _ => BuildRowWriter <T>(writerType, columns, dataConnection.MappingSchema)); var writer = dc.BeginBinaryImport(copyCommand); // https://github.com/npgsql/npgsql/issues/1646 // npgsql 4.0 will revert logic by removing explicit Cancel() and add explicit Complete() var hasCancel = writer.GetType().GetMethod("Cancel") != null; var hasComplete = writer.GetType().GetMethod("Complete") != null; try { foreach (var item in source) { rowWriter(dataConnection.MappingSchema, writer, columns, item); currentCount++; rowsCopied.RowsCopied++; if (currentCount >= batchSize) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { if (hasCancel) { writer.Cancel(); } break; } } if (hasComplete) { writer.Complete(); } writer.Dispose(); writer = dc.BeginBinaryImport(copyCommand); currentCount = 0; } } if (!rowsCopied.Abort && hasComplete) { writer.Complete(); } } catch when(hasCancel) { writer.Cancel(); throw; } finally { writer.Dispose(); } return(rowsCopied); }
private async Task<BulkCopyRowsCopied> ProviderSpecificCopyInternal<T>( ProviderConnections providerConnections, ITable<T> table, BulkCopyOptions options, Func<List<Mapping.ColumnDescriptor>, BulkCopyReader<T>> createDataReader, bool runAsync, CancellationToken cancellationToken) where T : notnull { var dataConnection = providerConnections.DataConnection; var connection = providerConnections.ProviderConnection; var transaction = providerConnections.ProviderTransaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rc = new BulkCopyRowsCopied(); var hanaOptions = SapHanaProviderAdapter.HanaBulkCopyOptions.Default; if (options.KeepIdentity == true) hanaOptions |= SapHanaProviderAdapter.HanaBulkCopyOptions.KeepIdentity; using (var bc = _provider.Adapter.CreateBulkCopy(connection, hanaOptions, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.HanaRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) args.Abort = true; }; } if (options.MaxBatchSize.HasValue) bc.BatchSize = options.MaxBatchSize.Value; if (options.BulkCopyTimeout.HasValue) bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout) bc.BulkCopyTimeout = connection.ConnectionTimeout; var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema); var tableName = GetTableName(sqlBuilder, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, columns[i].ColumnName)); var rd = createDataReader(columns); await TraceActionAsync( dataConnection, () => (runAsync && bc.CanWriteToServerAsync ? "INSERT ASYNC BULK " : "INSERT BULK ") + tableName + Environment.NewLine, async () => { if (runAsync && bc.CanWriteToServerAsync) await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); else bc.WriteToServer(rd); return rd.Count; }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) options.RowsCopiedCallback(rc); } return rc; } }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } var connection = dataConnection.Connection as SqlConnection; if (connection != null) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _dataProvider.CreateSqlBuilder(); var rd = new BulkCopyReader(_dataProvider, columns, source); var sqlopt = SqlBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SqlBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SqlBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SqlBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SqlBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SqlBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SqlBulkCopyOptions.UseInternalTransaction; } using (var bc = new SqlBulkCopy(connection, sqlopt, (SqlTransaction)dataConnection.Transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.SqlRowsCopied += (sender, e) => { rc.RowsCopied = e.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { e.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, descriptor); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(new SqlBulkCopyColumnMapping( i, sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField).ToString())); } bc.WriteToServer(rd); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } return(MultipleRowsCopy(dataConnection, options, source)); }
private BulkCopyRowsCopied ProviderSpecificCopySyncImpl <T>( DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source, IDbConnection connection, string tableName, ColumnDescriptor[] columns, NpgsqlProviderAdapter.NpgsqlDbType[] npgsqlTypes, string copyCommand, int batchSize, NpgsqlProviderAdapter.NpgsqlBinaryImporter writer) { var currentCount = 0; var rowsCopied = new BulkCopyRowsCopied(); try { foreach (var item in source) { writer.StartRow(); for (var i = 0; i < columns.Length; i++) { writer.Write(columns[i].GetValue(item !), npgsqlTypes[i]); } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { if (!writer.HasComplete && !writer.HasComplete5) { writer.Cancel(); } break; } } if (currentCount >= batchSize) { if (writer.HasComplete) { writer.Complete(); } else if (writer.HasComplete5) { writer.Complete5(); } writer.Dispose(); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { if (writer.HasComplete) { writer.Complete(); } else if (writer.HasComplete5) { writer.Complete5(); } return((int)rowsCopied.RowsCopied); }); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } catch when(!writer.HasComplete && !writer.HasComplete5) { writer.Cancel(); throw; } finally { writer.Dispose(); } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } var connection = dataConnection.Connection; if (connection == null) { return(MultipleRowsCopy(dataConnection, options, source)); } if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType))) { return(MultipleRowsCopy(dataConnection, options, source)); } var transaction = dataConnection.Transaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rc = new BulkCopyRowsCopied(); if (_bulkCopyCreator == null) { var clientNamespace = _dataProvider.ConnectionNamespace; var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopy", false); _bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyOptions", false); var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false); var transactionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaTransaction", false); if (bulkCopyType != null) { _bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, _bulkCopyOptionType, transactionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator == null) { return(MultipleRowsCopy(dataConnection, options, source)); } int hanaOptions = 0; //default; if (options.KeepIdentity == true) { // instead of adding new option in HANA 2 provider to a free bit to preserve compatibility, // SAP reused value, assigned to TableLock before if (Enum.GetNames(_bulkCopyOptionType).Any(_ => _ == KeepIdentityOptionName)) { hanaOptions = hanaOptions | (int)Enum.Parse(_bulkCopyOptionType, KeepIdentityOptionName); } else { throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by your SAP HANA provider version"); } } using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction)) { dynamic dbc = bc; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied"); } dbc.NotifyAfter = options.NotifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.MaxBatchSize.HasValue) { dbc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, descriptor); dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source); TraceAction( dataConnection, "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, descriptor); if (dataConnection.Transaction == null) { if (_bulkCopyCreator == null) { var clientNamespace = ((OracleDataProvider)dataConnection.DataProvider).AssemblyName + ".Client."; var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopy", false); var bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopyOptions", false); var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopyColumnMapping", false); if (bulkCopyType != null) { _bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator != null) { var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToList(); var rd = new BulkCopyReader(_dataProvider, columns, source); var rc = new BulkCopyRowsCopied(); var bcOptions = 0; // Default if (options.UseInternalTransaction == true) { bcOptions |= 1; // UseInternalTransaction = 1, } using (var bc = _bulkCopyCreator(dataConnection.Connection, bcOptions)) { dynamic dbc = bc; var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "OracleRowsCopied"); } dbc.NotifyAfter = notifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.MaxBatchSize.HasValue) { dbc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } TraceAction( dataConnection, "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } options.BulkCopyType = BulkCopyType.MultipleRows; return(MultipleRowsCopy(dataConnection, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (_provider.Adapter.BulkCopy != null && table.DataContext is DataConnection dataConnection) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); var transaction = dataConnection.Transaction; if (connection != null && transaction != null) { transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema); } if (connection != null && (dataConnection.Transaction == null || transaction != null)) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rc = new BulkCopyRowsCopied(); var bc = _provider.Adapter.BulkCopy.Create(connection, transaction); if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.MySqlRowsCopied += (sender, args) => { rc.RowsCopied += args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = GetTableName(sb, options, table); for (var i = 0; i < columns.Count; i++) { bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } // emulate missing BatchSize property // this is needed, because MySql fails on big batches, so users should be able to limit batch size foreach (var batch in EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue)) { var rd = new BulkCopyReader(dataConnection, columns, batch); TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); rc.RowsCopied += rd.Count; } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } return(rc); } } return(MultipleRowsCopy(table, options, source)); }