private BulkCopyRowsCopied ProviderSpecificCopySyncImpl <T>( DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source, IDbConnection connection, string tableName, ColumnDescriptor[] columns, NpgsqlProviderAdapter.NpgsqlDbType[] npgsqlTypes, string copyCommand, int batchSize, bool useComplete, NpgsqlProviderAdapter.NpgsqlBinaryImporter writer) { var currentCount = 0; var rowsCopied = new BulkCopyRowsCopied(); try { foreach (var item in source) { writer.StartRow(); for (var i = 0; i < columns.Length; i++) { writer.Write(columns[i].GetValue(item !), npgsqlTypes[i]); } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { if (!useComplete) { writer.Cancel(); } break; } } if (currentCount >= batchSize) { if (useComplete) { writer.Complete(); } writer.Dispose(); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { if (useComplete) { writer.Complete(); } return((int)rowsCopied.RowsCopied); }); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } catch when(!useComplete) { writer.Cancel(); throw; } finally { writer.Dispose(); } return(rowsCopied); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, CancellationToken cancellationToken) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection == null) { return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)); } var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length]; for (var i = 0; i < columns.Length; i++) { var npgsqlType = _provider.GetNativeType(columns[i].DbType, true); if (npgsqlType == null) { var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null; if (columnType == null || columnType.Type.DataType == DataType.Undefined) { columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType); } var sb = new System.Text.StringBuilder(); sqlBuilder.BuildTypeName(sb, columnType); npgsqlType = _provider.GetNativeType(sb.ToString(), true); } if (npgsqlType == null) { throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping."); } npgsqlTypes[i] = npgsqlType.Value; } var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); if (!writer.SupportsAsync) { // seems to be missing one of the required async methods; fallback to sync importer return(ProviderSpecificCopySyncImpl(dataConnection, options, source, connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, true, writer)); } var rowsCopied = new BulkCopyRowsCopied(); var currentCount = 0; try { foreach (var item in source) { await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); for (var i = 0; i < columns.Length; i++) { await writer.WriteAsync(columns[i].GetValue(item !), npgsqlTypes[i], cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } if (currentCount >= batchSize) { await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { await TraceActionAsync( dataConnection, () => "INSERT ASYNC BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, async() => { var ret = await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); return((int)rowsCopied.RowsCopied); }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } finally { await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (!(table?.DataContext is DataConnection dataConnection)) { throw new ArgumentNullException(nameof(dataConnection)); } if (dataConnection.Transaction == null) { var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); if (_bulkCopyCreator == null) { var bulkCopyType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopy", false); var bulkCopyOptionType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyOptions", false); var columnMappingType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyColumnMapping", false); if (bulkCopyType != null) { _bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator != null) { var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source); var rc = new BulkCopyRowsCopied(); var bcOptions = 0; // Default if (options.KeepIdentity == true) { bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4, } if (options.TableLock == true) { bcOptions |= 2; } using (var bc = _bulkCopyCreator(Proxy.GetUnderlyingObject((DbConnection)dataConnection.Connection), bcOptions)) { dynamic dbc = bc; var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2RowsCopied"); } dbc.NotifyAfter = notifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { // database name is not a part of table FQN in oracle var serverName = options.ServerName ?? table.ServerName; if (table.TryGetDataConnection(out var dataConnection) && _provider.Adapter.BulkCopy != null && serverName == null) { var connection = _provider.TryGetProviderConnection(dataConnection, dataConnection.Connection); if (connection != null) { var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema); // ODP.NET doesn't bulk copy doesn't work if columns that require escaping: // - if escaping applied, pre-flight validation fails as it performs uppercase comparison and quotes make it fail with // InvalidOperationException: Column mapping is invalid // - if escaping not applied - if fails as expected on server, because it treats passed name as uppercased name // and gives "ORA-00904: "STRINGVALUE": invalid identifier" error // That's quite common error in bulk copy implementation error by providers... var supported = true; foreach (var column in columns) { if (column.ColumnName != sb.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)) { // fallback to sql-based copy // TODO: we should add support for by-ordinal column mapping to workaround it supported = false; break; } } if (supported) { var rd = new BulkCopyReader <T>(dataConnection, columns, source); var sqlopt = OracleProviderAdapter.BulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); var tableName = sb.ConvertInline(options.TableName ?? table.TableName, ConvertType.NameToQueryTable); var schemaName = options.SchemaName ?? table.SchemaName; if (schemaName != null) { schemaName = sb.ConvertInline(schemaName, ConvertType.NameToSchema); } if (options.UseInternalTransaction == true) { sqlopt |= OracleProviderAdapter.BulkCopyOptions.UseInternalTransaction; } if (options.CheckConstraints == true) { sqlopt |= OracleProviderAdapter.BulkCopyOptions.KeepConstraints; } if (options.FireTriggers != true) { sqlopt |= OracleProviderAdapter.BulkCopyOptions.DisableTriggers; } var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; using (var bc = _provider.Adapter.BulkCopy.Create( connection, sqlopt, tableName, schemaName, notifyAfter != 0 && options.RowsCopiedCallback != null ? notifyAfter : null, options.RowsCopiedCallback, rc, options.MaxBatchSize, options.BulkCopyTimeout ?? (Configuration.Data.BulkCopyUseConnectionCommandTimeout ? connection.ConnectionTimeout : null))) { for (var i = 0; i < columns.Count; i++) { bc.AddColumn(i, columns[i]); } // TraceAction( dataConnection, () => "INSERT BULK " + (schemaName == null ? tableName : schemaName + "." + tableName) + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine, () => { bc.Execute(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (_provider.Adapter.BulkCopy != null && table.DataContext is DataConnection dataConnection) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); var transaction = dataConnection.Transaction; if (connection != null && transaction != null) { transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema); } if (connection != null && (dataConnection.Transaction == null || transaction != null)) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rc = new BulkCopyRowsCopied(); var bc = _provider.Adapter.BulkCopy.Create(connection, transaction); if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.MySqlRowsCopied += (sender, args) => { rc.RowsCopied += args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = GetTableName(sb, options, table); for (var i = 0; i < columns.Count; i++) { bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } // emulate missing BatchSize property // this is needed, because MySql fails on big batches, so users should be able to limit batch size foreach (var batch in EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue)) { var rd = new BulkCopyReader(dataConnection, columns, batch); TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); rc.RowsCopied += rd.Count; } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } return(rc); } } return(MultipleRowsCopy(table, options, source)); }
protected override BulkCopyRowsCopied MultipleRowsCopy <T>( DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, descriptor); var sb = new StringBuilder(); var buildValue = BasicSqlBuilder.GetBuildValue(sqlBuilder, sb); var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToArray(); var pname = sqlBuilder.Convert("p", ConvertType.NameToQueryParameter).ToString(); var rowsCopied = new BulkCopyRowsCopied(); sb .AppendFormat("INSERT INTO {0}", tableName).AppendLine() .Append("("); foreach (var column in columns) { sb .AppendLine() .Append("\t") .Append(sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField)) .Append(","); } sb.Length--; sb .AppendLine() .Append(")"); sb .AppendLine() .Append("VALUES"); var headerLen = sb.Length; var currentCount = 0; var batchSize = options.MaxBatchSize ?? 1000; if (batchSize <= 0) { batchSize = 1000; } var parms = new List <DataParameter>(); var pidx = 0; foreach (var item in source) { sb .AppendLine() .Append("("); foreach (var column in columns) { var value = column.GetValue(item); if (value == null) { sb.Append("NULL"); } else { switch (Type.GetTypeCode(value.GetType())) { case TypeCode.DBNull: sb.Append("NULL"); break; case TypeCode.String: var isString = false; switch (column.DataType) { case DataType.NVarChar: case DataType.Char: case DataType.VarChar: case DataType.NChar: case DataType.Undefined: isString = true; break; } if (isString) { goto case TypeCode.Int32; } goto default; case TypeCode.Boolean: case TypeCode.Char: case TypeCode.SByte: case TypeCode.Byte: case TypeCode.Int16: case TypeCode.UInt16: case TypeCode.Int32: case TypeCode.UInt32: case TypeCode.Int64: case TypeCode.UInt64: case TypeCode.Single: case TypeCode.Double: case TypeCode.Decimal: case TypeCode.DateTime: //SetParameter(dataParam, "", column.DataType, value); buildValue(value); break; default: var name = pname + ++pidx; sb.Append(name); parms.Add(new DataParameter("p" + pidx, value, column.DataType)); break; } } sb.Append(","); } sb.Length--; sb.Append("),"); rowsCopied.RowsCopied++; currentCount++; if (currentCount >= batchSize || parms.Count > 100000 || sb.Length > 100000) { sb.Length--; dataConnection.Execute(sb.AppendLine().ToString(), parms.ToArray()); if (options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { return(rowsCopied); } } parms.Clear(); pidx = 0; currentCount = 0; sb.Length = headerLen; } } if (currentCount > 0) { sb.Length--; dataConnection.Execute(sb.ToString(), parms.ToArray()); sb.Length = headerLen; if (options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } return(rowsCopied); }
protected BulkCopyRowsCopied IDSProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, DataConnection dataConnection, IDbConnection connection, InformixProviderAdapter.BulkCopyAdapter bulkCopy) where T : notnull { var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema); var rd = new BulkCopyReader <T>(dataConnection, columns, source); var sqlopt = InformixProviderAdapter.IfxBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.KeepIdentity == true) { sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.TableLock; } using (var bc = bulkCopy.Create(connection, sqlopt)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.IfxRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField))); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); }
private BulkCopyRowsCopied ProviderSpecificCopyInternal <T>( ProviderConnections providerConnections, ITable <T> table, BulkCopyOptions options, Func <List <Mapping.ColumnDescriptor>, BulkCopyReader <T> > createDataReader) where T : notnull { var dataConnection = providerConnections.DataConnection; var connection = providerConnections.ProviderConnection; var transaction = providerConnections.ProviderTransaction; var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema); var rd = createDataReader(columns); var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); if (options.CheckConstraints == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints; } if (options.KeepIdentity == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity; } if (options.TableLock == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock; } if (options.KeepNulls == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls; } if (options.FireTriggers == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers; } if (options.UseInternalTransaction == true) { sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction)) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.SqlRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout) { bc.BulkCopyTimeout = connection.ConnectionTimeout; } var tableName = GetTableName(sb, options, table); bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField))); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (!(table?.DataContext is DataConnection dataConnection)) { throw new ArgumentNullException(nameof(dataConnection)); } var connection = dataConnection.Connection; if (connection == null) { return(MultipleRowsCopy(table, options, source)); } if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType))) { return(MultipleRowsCopy(table, options, source)); } var sqlBuilder = _dataProvider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var writerType = _connectionType.AssemblyEx().GetType("Npgsql.NpgsqlBinaryImporter", true); var fields = string.Join(", ", columns.Select(column => sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; var rowsCopied = new BulkCopyRowsCopied(); // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var currentCount = 0; var dc = (dynamic)connection; var key = new { Type = typeof(T), options.KeepIdentity, ed }; var rowWriter = (Action <MappingSchema, object, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd( key, _ => BuildRowWriter <T>(writerType, columns, dataConnection.MappingSchema)); var writer = dc.BeginBinaryImport(copyCommand); // https://github.com/npgsql/npgsql/issues/1646 // npgsql 4.0 will revert logic by removing explicit Cancel() and add explicit Complete() var hasCancel = writer.GetType().GetMethod("Cancel") != null; var hasComplete = writer.GetType().GetMethod("Complete") != null; try { foreach (var item in source) { rowWriter(dataConnection.MappingSchema, writer, columns, item); currentCount++; rowsCopied.RowsCopied++; if (currentCount >= batchSize) { if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { if (hasCancel) { writer.Cancel(); } break; } } if (hasComplete) { writer.Complete(); } writer.Dispose(); writer = dc.BeginBinaryImport(copyCommand); currentCount = 0; } } if (!rowsCopied.Abort && hasComplete) { writer.Complete(); } } catch when(hasCancel) { writer.Cancel(); throw; } finally { writer.Dispose(); } return(rowsCopied); }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( [JetBrains.Annotations.NotNull] DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source) { if (dataConnection == null) { throw new ArgumentNullException("dataConnection"); } var connection = dataConnection.Connection; if (connection == null) { return(MultipleRowsCopy(dataConnection, options, source)); } if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType))) { return(MultipleRowsCopy(dataConnection, options, source)); } var transaction = dataConnection.Transaction; var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var rc = new BulkCopyRowsCopied(); if (_bulkCopyCreator == null) { var clientNamespace = _dataProvider.ConnectionNamespace; var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopy", false); _bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyOptions", false); var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false); var transactionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaTransaction", false); if (bulkCopyType != null) { _bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, _bulkCopyOptionType, transactionType); _columnMappingCreator = CreateColumnMappingCreator(columnMappingType); } } if (_bulkCopyCreator == null) { return(MultipleRowsCopy(dataConnection, options, source)); } int hanaOptions = 0; //default; if (options.KeepIdentity == true) { // instead of adding new option in HANA 2 provider to a free bit to preserve compatibility, // SAP reused value, assigned to TableLock before if (Enum.GetNames(_bulkCopyOptionType).Any(_ => _ == KeepIdentityOptionName)) { hanaOptions = hanaOptions | (int)Enum.Parse(_bulkCopyOptionType, KeepIdentityOptionName); } else { throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by your SAP HANA provider version"); } } using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction)) { dynamic dbc = bc; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { if (_bulkCopySubscriber == null) { _bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied"); } dbc.NotifyAfter = options.NotifyAfter; _bulkCopySubscriber(bc, arg => { dynamic darg = arg; rc.RowsCopied = darg.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { darg.Abort = true; } }); } if (options.MaxBatchSize.HasValue) { dbc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } var sqlBuilder = _dataProvider.CreateSqlBuilder(); var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, descriptor); dbc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName)); } var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source); TraceAction( dataConnection, () => "INSERT BULK " + tableName + Environment.NewLine, () => { dbc.WriteToServer(rd); return(rd.Count); }); if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } }
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>( ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { if (table.DataContext is DataConnection dataConnection && dataConnection.Transaction == null && _provider.Adapter.BulkCopy != null) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection != null) { var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList(); var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema); var rd = new BulkCopyReader(dataConnection, columns, source); var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default; var rc = new BulkCopyRowsCopied(); var tableName = GetTableName(sb, options, table); if (options.UseInternalTransaction == true) { sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction; } using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt)) { var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ? options.MaxBatchSize.Value : options.NotifyAfter; if (notifyAfter != 0 && options.RowsCopiedCallback != null) { bc.NotifyAfter = options.NotifyAfter; bc.OracleRowsCopied += (sender, args) => { rc.RowsCopied = args.RowsCopied; options.RowsCopiedCallback(rc); if (rc.Abort) { args.Abort = true; } }; } if (options.MaxBatchSize.HasValue) { bc.BatchSize = options.MaxBatchSize.Value; } if (options.BulkCopyTimeout.HasValue) { bc.BulkCopyTimeout = options.BulkCopyTimeout.Value; } bc.DestinationTableName = tableName; for (var i = 0; i < columns.Count; i++) { bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName)); } TraceAction( dataConnection, () => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, () => { bc.WriteToServer(rd); return(rd.Count); }); } if (rc.RowsCopied != rd.Count) { rc.RowsCopied = rd.Count; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rc); } } return(rc); } } return(MultipleRowsCopy(table, options, source)); }