protected virtual BulkCopyRowsCopied RowByRowCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source)
{
// This limitation could be lifted later for some providers that supports identity insert if we will get such request
// It will require support from DataConnection.Insert
if (options.KeepIdentity == true)
{
throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by {nameof(BulkCopyType)}.{nameof(BulkCopyType.RowByRow)} mode");
}
var rowsCopied = new BulkCopyRowsCopied();
foreach (var item in source)
{
dataConnection.Insert(item, options.TableName, options.DatabaseName, options.SchemaName);
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
break;
}
}
}
return(rowsCopied);
}
protected virtual BulkCopyRowsCopied RowByRowCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source)
{
var rowsCopied = new BulkCopyRowsCopied();
foreach (var item in source)
{
dataConnection.Insert(item, options.TableName, options.DatabaseName, options.SchemaName);
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
break;
}
}
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (table.DataContext is DataConnection dataConnection)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
var transaction = dataConnection.Transaction;
if (connection != null && transaction != null)
{
transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema);
}
if (connection != null && (dataConnection.Transaction == null || transaction != null))
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = new BulkCopyReader(dataConnection, columns, source);
var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.SqlRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField)));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (!(table?.DataContext is DataConnection dataConnection))
{
throw new ArgumentNullException(nameof(dataConnection));
}
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
var transaction = dataConnection.Transaction;
if (connection != null && transaction != null)
{
transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema);
}
if (connection != null && (dataConnection.Transaction == null || transaction != null))
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rc = new BulkCopyRowsCopied();
var hanaOptions = SapHanaProviderAdapter.HanaBulkCopyOptions.Default;
if (options.KeepIdentity == true)
{
hanaOptions |= SapHanaProviderAdapter.HanaBulkCopyOptions.KeepIdentity;
}
using (var bc = _provider.Adapter.CreateBulkCopy(connection, hanaOptions, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.HanaRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema);
var tableName = GetTableName(sqlBuilder, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, columns[i].ColumnName));
}
var rd = new BulkCopyReader(dataConnection, columns, source);
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken)
where T : notnull
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection == null)
{
return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext));
}
var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema);
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length];
for (var i = 0; i < columns.Length; i++)
{
var npgsqlType = _provider.GetNativeType(columns[i].DbType, true);
if (npgsqlType == null)
{
var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null;
if (columnType == null || columnType.Type.DataType == DataType.Undefined)
{
columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType);
}
var sb = new System.Text.StringBuilder();
sqlBuilder.BuildTypeName(sb, columnType);
npgsqlType = _provider.GetNativeType(sb.ToString(), true);
}
if (npgsqlType == null)
{
throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping.");
}
npgsqlTypes[i] = npgsqlType.Value;
}
var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
if (!writer.SupportsAsync)
{
// seems to be missing one of the required async methods; fallback to sync importer
var enumerator = source.GetAsyncEnumerator(cancellationToken);
await using (enumerator.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext))
{
return(ProviderSpecificCopySyncImpl(dataConnection, options, EnumerableHelper.AsyncToSyncEnumerable(enumerator), connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, writer));
}
}
var rowsCopied = new BulkCopyRowsCopied();
var currentCount = 0;
try
{
await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext))
{
await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
for (var i = 0; i < columns.Length; i++)
{
await writer.WriteAsync(columns[i].GetValue(item !), npgsqlTypes[i], cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
break;
}
}
if (currentCount >= batchSize)
{
await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
await TraceActionAsync(
dataConnection,
() => "INSERT ASYNC BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
async() => {
var ret = await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
return((int)rowsCopied.RowsCopied);
}).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
finally
{
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (!(table?.DataContext is DataConnection dataConnection))
{
throw new ArgumentNullException(nameof(dataConnection));
}
if (dataConnection.Connection is DbConnection dbConnection)
{
if (Proxy.GetUnderlyingObject(dbConnection) is SqlConnection connection)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _dataProvider.CreateSqlBuilder();
var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source);
var sqlopt = SqlBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SqlBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SqlBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SqlBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SqlBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SqlBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SqlBulkCopyOptions.UseInternalTransaction;
}
using (var bc = new SqlBulkCopy(connection, sqlopt, (SqlTransaction)dataConnection.Transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.SqlRowsCopied += (sender, e) =>
{
rc.RowsCopied = e.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
e.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var tableName = GetTableName(sqlBuilder, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(new SqlBulkCopyColumnMapping(
i,
sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField).ToString()));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", bc.ColumnMappings.Cast <SqlBulkCopyColumnMapping>().Select(x => x.DestinationColumn)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyInternal <T>(
ProviderConnections providerConnections,
ITable <T> table,
BulkCopyOptions options,
Func <List <Mapping.ColumnDescriptor>, BulkCopyReader <T> > createDataReader,
bool runAsync,
CancellationToken cancellationToken)
where T : notnull
{
var dataConnection = providerConnections.DataConnection;
var connection = providerConnections.ProviderConnection;
var transaction = providerConnections.ProviderTransaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = createDataReader(columns);
var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.SqlRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField)));
}
await TraceActionAsync(
dataConnection,
() => (runAsync ? "INSERT ASYNC BULK " : "INSERT BULK ") + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine,
async() => {
if (runAsync)
{
await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
else
{
bc.WriteToServer(rd);
}
return(rd.Count);
}, true).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyInternal <T>(
ProviderConnections providerConnections,
ITable <T> table,
BulkCopyOptions options,
IAsyncEnumerable <T> source,
CancellationToken cancellationToken)
{
var dataConnection = providerConnections.DataConnection;
var connection = providerConnections.ProviderConnection;
var transaction = providerConnections.ProviderTransaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rc = new BulkCopyRowsCopied();
var bc = _provider.Adapter.BulkCopy !.Create(connection, transaction);
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.MySqlRowsCopied += (sender, args) =>
{
rc.RowsCopied += args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = GetTableName(sb, options, table);
for (var i = 0; i < columns.Count; i++)
{
bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
// emulate missing BatchSize property
// this is needed, because MySql fails on big batches, so users should be able to limit batch size
var batches = EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue);
await foreach (var batch in batches.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext))
{
var rd = new BulkCopyReader <T>(dataConnection, columns, batch, cancellationToken);
await TraceActionAsync(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
async() => {
if (bc.CanWriteToServerAsync2)
{
await bc.WriteToServerAsync2(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
else
if (bc.CanWriteToServerAsync)
{
await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
else
{
bc.WriteToServer(rd);
}
return(rd.Count);
}).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
rc.RowsCopied += rd.Count;
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
return(rc);
}
protected BulkCopyRowsCopied IDSProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source,
DataConnection dataConnection,
IDbConnection connection,
InformixProviderAdapter.BulkCopyAdapter bulkCopy)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = new BulkCopyReader(dataConnection, columns, source);
var sqlopt = InformixProviderAdapter.IfxBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.KeepIdentity == true)
{
sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.TableLock;
}
using (var bc = bulkCopy.Create(connection, sqlopt))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.IfxRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField)));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (table.DataContext is DataConnection dataConnection && _provider.Adapter.BulkCopy != null)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
// for run in transaction see
// https://stackoverflow.com/questions/57675379
// provider will call sp_oledb_columns which creates temp table
var transaction = dataConnection.Transaction;
if (connection != null && transaction != null)
{
transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema);
}
if (connection != null && (dataConnection.Transaction == null || transaction != null))
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = new BulkCopyReader(dataConnection, columns, source);
var sqlopt = SybaseProviderAdapter.AseBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SybaseProviderAdapter.AseBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.AseRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
// do not convert table and column names to valid sql name, as sybase bulk copy implementation
// doesn't understand escaped names (facepalm)
// which means it will probably fail when escaping required anyways...
bc.DestinationTableName = GetDestinationTableName(sb, options, table);
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(columns[i].ColumnName, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (!(table?.DataContext is DataConnection dataConnection))
{
throw new ArgumentNullException(nameof(dataConnection));
}
if (dataConnection.Transaction == null)
{
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema);
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
if (_bulkCopyCreator == null)
{
var bulkCopyType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopy", false);
var bulkCopyOptionType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyOptions", false);
var columnMappingType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyColumnMapping", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator != null)
{
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source);
var rc = new BulkCopyRowsCopied();
var bcOptions = 0; // Default
if (options.KeepIdentity == true)
{
bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4,
}
if (options.TableLock == true)
{
bcOptions |= 2;
}
using (var bc = _bulkCopyCreator(Proxy.GetUnderlyingObject((DbConnection)dataConnection.Connection), bcOptions))
{
dynamic dbc = bc;
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ?
options.MaxBatchSize.Value : options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2RowsCopied");
}
dbc.NotifyAfter = notifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied MultipleRowsCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source)
{
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, descriptor);
var iszOS = _dataProvider.Version == DB2Version.zOS;
var rowsCopied = new BulkCopyRowsCopied();
var sb = new StringBuilder();
var buildValue = BasicSqlBuilder.GetBuildValue(sqlBuilder, sb);
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToArray();
var pname = sqlBuilder.Convert("p", ConvertType.NameToQueryParameter).ToString();
sb
.AppendFormat("INSERT INTO {0}", tableName).AppendLine()
.Append("(");
foreach (var column in columns)
{
sb
.AppendLine()
.Append("\t")
.Append(sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField))
.Append(",");
}
sb.Length--;
sb
.AppendLine()
.Append(")");
if (!iszOS)
{
sb
.AppendLine()
.Append("VALUES");
}
var headerLen = sb.Length;
var currentCount = 0;
var batchSize = options.MaxBatchSize ?? 1000;
if (batchSize <= 0)
{
batchSize = 1000;
}
var parms = new List <DataParameter>();
var pidx = 0;
foreach (var item in source)
{
sb
.AppendLine()
.Append(iszOS ? "SELECT " : "(");
foreach (var column in columns)
{
var value = column.GetValue(item);
if (value == null)
{
sb.Append("NULL");
}
else
{
switch (Type.GetTypeCode(value.GetType()))
{
case TypeCode.DBNull:
sb.Append("NULL");
break;
case TypeCode.String:
var isString = false;
switch (column.DataType)
{
case DataType.NVarChar:
case DataType.Char:
case DataType.VarChar:
case DataType.NChar:
case DataType.Undefined:
isString = true;
break;
}
if (isString)
{
goto case TypeCode.Int32;
}
goto default;
case TypeCode.Boolean:
case TypeCode.Char:
case TypeCode.SByte:
case TypeCode.Byte:
case TypeCode.Int16:
case TypeCode.UInt16:
case TypeCode.Int32:
case TypeCode.UInt32:
case TypeCode.Int64:
case TypeCode.UInt64:
case TypeCode.Single:
case TypeCode.Double:
case TypeCode.Decimal:
case TypeCode.DateTime:
//SetParameter(dataParam, "", column.DataType, value);
buildValue(value);
break;
default:
var name = pname + ++pidx;
sb.Append(name);
parms.Add(new DataParameter("p" + pidx, value, column.DataType));
break;
}
}
sb.Append(",");
}
sb.Length--;
sb.Append(iszOS ? " FROM SYSIBM.SYSDUMMY1 UNION ALL" : "),");
rowsCopied.RowsCopied++;
currentCount++;
if (currentCount >= batchSize || parms.Count > 100000 || sb.Length > 100000)
{
if (iszOS)
{
sb.Length -= " UNION ALL".Length;
}
else
{
sb.Length--;
}
dataConnection.Execute(sb.AppendLine().ToString(), parms.ToArray());
if (options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
return(rowsCopied);
}
}
parms.Clear();
pidx = 0;
currentCount = 0;
sb.Length = headerLen;
}
}
if (currentCount > 0)
{
if (iszOS)
{
sb.Length -= " UNION ALL".Length;
}
else
{
sb.Length--;
}
dataConnection.Execute(sb.ToString(), parms.ToArray());
sb.Length = headerLen;
if (options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
return(rowsCopied);
}
private BulkCopyRowsCopied ProviderSpecificCopyImpl_AccessClient <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source,
DataConnection dataConnection,
IDbConnection connection,
DB2iSeriesAccessClientProviderAdapter adapter,
Action <DataConnection, Func <string>, Func <int> > traceAction)
{
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var rc = new BulkCopyRowsCopied();
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema);
var tableName = GetTableName(sqlBuilder, options, table);
var columnNames = columns.Select(x => x.ColumnName);
var fields = string.Join(", ", columnNames);
var parameters = string.Join(", ", columnNames.Select(x => "@" + x));
var sql = $"INSERT INTO {tableName} ({fields}) VALUES({parameters})";
var batchSize = Math.Min(options.MaxBatchSize ?? MAX_ACCESS_CLIENT_BATCH_SIZE, MAX_ACCESS_CLIENT_BATCH_SIZE);
var sw = Stopwatch.StartNew();
using (var cmd = connection.CreateCommand())
{
cmd.CommandText = sql;
adapter.DeriveParameters(cmd);
var columnDataTypes = cmd.Parameters.Cast <IDbDataParameter>()
.Select(adapter.GetDbType)
.Select(adapter.GetDataType)
.ToList();
//*LOB and XML types not supported, fallback to multiple rows
if (columnDataTypes.Any(x => x.In(DataType.Blob, DataType.Text, DataType.NText, DataType.Xml)))
{
return(MultipleRowsCopy(table, options, source));
}
var columnDbDataTypes = columns
.Select((x, i) => x
.GetDbDataType(true)
.WithDataType(columnDataTypes[i]))
.ToList();
var count = 0;
var notificationBatchCount = 0;
var bufferEmpty = false;
while (rd.Read())
{
var i = 0;
bufferEmpty = false;
foreach (IDbDataParameter parameter in cmd.Parameters)
{
dataConnection.DataProvider.SetParameter(
dataConnection,
parameter,
columns[i].ColumnName,
columnDbDataTypes[i],
rd.GetValue(i++));
}
try
{
adapter.AddBatch(cmd);
}
//In case columns can't be mapped fall back to multiple rows
catch
{
return(MultipleRowsCopy(table, options, source));
}
count++;
notificationBatchCount++;
if (count % batchSize == 0)
{
execute();
if (rc.Abort)
{
return(rc);
}
bufferEmpty = true;
}
}
if (!bufferEmpty)
{
execute();
if (rc.Abort)
{
return(rc);
}
}
void execute()
{
cmd.ExecuteNonQuery();
rc.RowsCopied = count;
if (sw.Elapsed.TotalSeconds > options.BulkCopyTimeout)
{
traceAction(
dataConnection,
() => $"INSERT BULK {tableName} TIMEOUT AFTER {sw.Elapsed.TotalSeconds} SECONDS" + Environment.NewLine,
() => (int)rc.RowsCopied);
rc.Abort = true;
return;
}
if (options.NotifyAfter > 0 && notificationBatchCount > options.NotifyAfter)
{
options.RowsCopiedCallback?.Invoke(rc);
notificationBatchCount = 0;
}
if (rc.Abort)
{
traceAction(
dataConnection,
() => $"INSERT BULK {tableName} ABORTED BY USER" + Environment.NewLine,
() => (int)rc.RowsCopied);
return;
}
}
}
return(rc);
}
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken)
where T : notnull
{
var connection = _provider.TryGetProviderConnection(dataConnection, dataConnection.Connection);
if (connection == null)
{
return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext));
}
var sqlBuilder = (PostgreSQLSqlBuilder)_provider.CreateSqlBuilder(table.DataContext.MappingSchema);
var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var(npgsqlTypes, dbTypes, columnTypes) = BuildTypes(_provider.Adapter, sqlBuilder, columns);
var writer = _provider.Adapter.BeginBinaryImportAsync != null
? await _provider.Adapter.BeginBinaryImportAsync(connection, copyCommand, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)
: _provider.Adapter.BeginBinaryImport(connection, copyCommand);
if (!writer.SupportsAsync)
{
// seems to be missing one of the required async methods; fallback to sync importer
var enumerator = source.GetAsyncEnumerator(cancellationToken);
await using (enumerator.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext))
{
return(ProviderSpecificCopySyncImpl(dataConnection, options, EnumerableHelper.AsyncToSyncEnumerable(enumerator), connection, tableName, columns, columnTypes, npgsqlTypes, dbTypes, copyCommand, batchSize, writer));
}
}
var rowsCopied = new BulkCopyRowsCopied();
var currentCount = 0;
try
{
await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext))
{
await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
for (var i = 0; i < columns.Length; i++)
{
if (npgsqlTypes[i] != null)
{
await writer.WriteAsync(_provider.NormalizeTimeStamp(columns[i].GetProviderValue(item !), columnTypes[i]), npgsqlTypes[i] !.Value, cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
else
{
await writer.WriteAsync(_provider.NormalizeTimeStamp(columns[i].GetProviderValue(item !), columnTypes[i]), dbTypes[i] !, cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
}
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
break;
}
}
if (currentCount >= batchSize)
{
await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
writer = _provider.Adapter.BeginBinaryImportAsync != null
? await _provider.Adapter.BeginBinaryImportAsync(connection, copyCommand, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)
: _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
await TraceActionAsync(
dataConnection,
() => $"INSERT ASYNC BULK {tableName}({string.Join(", ", columns.Select(x => x.ColumnName))}){Environment.NewLine}",
async() => {
var ret = await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
return((int)rowsCopied.RowsCopied);
}).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
finally
{
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
var connection = dataConnection.Connection;
if (connection == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOf(_connectionType)))
{
return(MultipleRowsCopy(dataConnection, options, source));
}
var transaction = dataConnection.Transaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rc = new BulkCopyRowsCopied();
if (_bulkCopyCreator == null)
{
var clientNamespace = _dataProvider.ConnectionNamespace;
var bulkCopyType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopy", false);
var bulkCopyOptionType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopyOptions", false);
var columnMappingType = _connectionType.Assembly.GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false);
var transactionType = _connectionType.Assembly.GetType(clientNamespace + ".HanaTransaction", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType, transactionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
const int hanaOptions = 0; //default;
using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction))
{
dynamic dbc = bc;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied");
}
dbc.NotifyAfter = options.NotifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.MaxBatchSize.HasValue)
{
dbc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, descriptor);
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
var rd = new BulkCopyReader(_dataProvider, columns, source);
TraceAction(
dataConnection,
"INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (table.DataContext is DataConnection dataConnection && dataConnection.Transaction == null && _provider.Adapter.BulkCopy != null)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection != null)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
var tableName = GetTableName(sb, options, table);
if (options.UseInternalTransaction == true)
{
sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt))
{
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue
? options.MaxBatchSize.Value
: options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.OracleRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(ITable <T> table, BulkCopyOptions options, IEnumerable <T> source)
{
if (table.DataContext is DataConnection dataConnection)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection == null)
{
return(MultipleRowsCopy(table, options, source));
}
var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema);
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
var rowsCopied = new BulkCopyRowsCopied();
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var currentCount = 0;
var key = new { Type = typeof(T), options.KeepIdentity, ed };
var rowWriter = (Action <MappingSchema, NpgsqlProviderAdapter.NpgsqlBinaryImporter, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd(
key,
_ => _provider.Adapter.CreateBinaryImportRowWriter <T>(_provider, sqlBuilder, columns, dataConnection.MappingSchema));
var useComplete = _provider.Adapter.BinaryImporterHasComplete;
var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
try
{
foreach (var item in source)
{
rowWriter(dataConnection.MappingSchema, writer, columns, item);
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
if (!useComplete)
{
writer.Cancel();
}
break;
}
}
if (currentCount >= batchSize)
{
if (useComplete)
{
writer.Complete();
}
writer.Dispose();
writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => {
if (useComplete)
{
writer.Complete();
}
return((int)rowsCopied.RowsCopied);
});
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
catch when(!useComplete)
{
writer.Cancel();
throw;
}
finally
{
writer.Dispose();
}
return(rowsCopied);
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
if (dataConnection.Transaction == null)
{
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, descriptor);
if (_bulkCopyCreator == null)
{
throw new NotImplementedException("Not able to do bulk copy in DB2iSeries Provider.");
}
if (_bulkCopyCreator != null)
{
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source);
var rc = new BulkCopyRowsCopied();
var bcOptions = 0; // Default
if (options.KeepIdentity == true)
{
bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4,
}
if (options.TableLock == true)
{
bcOptions |= 2;
}
using (var bc = _bulkCopyCreator(dataConnection.Connection, bcOptions))
{
dynamic dbc = bc;
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ?
options.MaxBatchSize.Value : options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2iSeriesRowsCopied");
}
dbc.NotifyAfter = notifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
dbc.WriteToServer(rd);
}
rc.RowsCopied = rd.Count;
return(rc);
}
}
return(MultipleRowsCopy(dataConnection, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
// database name is not a part of table FQN in oracle
var serverName = options.ServerName ?? table.ServerName;
if (table.DataContext is DataConnection dataConnection && _provider.Adapter.BulkCopy != null &&
serverName == null)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection != null)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
// ODP.NET doesn't bulk copy doesn't work if columns that require escaping:
// - if escaping applied, pre-flight validation fails as it performs uppercase comparison and quotes make it fail with
// InvalidOperationException: Column mapping is invalid
// - if escaping not applied - if fails as expected on server, because it treats passed name as uppercased name
// and gives "ORA-00904: "STRINGVALUE": invalid identifier" error
// That's quite common error in bulk copy implementation error by providers...
var supported = true;
foreach (var column in columns)
{
if (column.ColumnName != sb.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))
{
// fallback to sql-based copy
// TODO: we should add support for by-ordinal column mapping to workaround it
supported = false;
break;
}
}
if (supported)
{
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
var tableName = sb.ConvertInline(options.TableName ?? table.TableName, ConvertType.NameToQueryTable);
var schemaName = options.SchemaName ?? table.SchemaName;
if (schemaName != null)
{
schemaName = sb.ConvertInline(schemaName, ConvertType.NameToSchema);
}
if (options.UseInternalTransaction == true)
{
sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt))
{
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue
? options.MaxBatchSize.Value
: options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.OracleRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
bc.DestinationTableName = tableName;
bc.DestinationSchemaName = schemaName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + (schemaName == null ? tableName : schemaName + "." + tableName) + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
}
return(MultipleRowsCopy(table, options, source));
}
internal static BulkCopyRowsCopied ProviderSpecificCopyImpl <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source,
DataConnection dataConnection,
DbConnection connection,
DB2ProviderAdapter.BulkCopyAdapter bulkCopy,
Action <DataConnection, Func <string>, Func <int> > traceAction)
where T : notnull
{
var descriptor = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var rc = new BulkCopyRowsCopied();
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(table.DataContext.MappingSchema);
var tableName = GetTableName(sqlBuilder, options, table);
var bcOptions = DB2BulkCopyOptions.Default;
if (options.KeepIdentity == true)
{
bcOptions |= DB2BulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
bcOptions |= DB2BulkCopyOptions.TableLock;
}
using (var bc = bulkCopy.Create(connection, bcOptions))
{
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ?
options.MaxBatchSize.Value : options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = notifyAfter;
bc.DB2RowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sqlBuilder.ConvertInline(columns[i].ColumnName, SqlProvider.ConvertType.NameToQueryField)));
}
traceAction(
dataConnection,
() => "INSERT BULK " + tableName + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException(nameof(dataConnection));
}
var connection = dataConnection.Connection;
if (connection == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType)))
{
return(MultipleRowsCopy(dataConnection, options, source));
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, ed);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var writerType = _connectionType.AssemblyEx().GetType("Npgsql.NpgsqlBinaryImporter", true);
var fields = string.Join(", ", columns.Select(column => sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
var rowsCopied = new BulkCopyRowsCopied();
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var currentCount = 0;
var dc = (dynamic)connection;
var key = new { Type = typeof(T), options.KeepIdentity, ed };
var rowWriter = (Action <MappingSchema, object, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd(
key,
_ => BuildRowWriter <T>(writerType, columns, dataConnection.MappingSchema));
var writer = dc.BeginBinaryImport(copyCommand);
// https://github.com/npgsql/npgsql/issues/1646
// npgsql 4.0 will revert logic by removing explicit Cancel() and add explicit Complete()
var hasCancel = writer.GetType().GetMethod("Cancel") != null;
var hasComplete = writer.GetType().GetMethod("Complete") != null;
try
{
foreach (var item in source)
{
rowWriter(dataConnection.MappingSchema, writer, columns, item);
currentCount++;
rowsCopied.RowsCopied++;
if (currentCount >= batchSize)
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
if (hasCancel)
{
writer.Cancel();
}
break;
}
}
if (hasComplete)
{
writer.Complete();
}
writer.Dispose();
writer = dc.BeginBinaryImport(copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort && hasComplete)
{
writer.Complete();
}
}
catch when(hasCancel)
{
writer.Cancel();
throw;
}
finally
{
writer.Dispose();
}
return(rowsCopied);
}
private async Task<BulkCopyRowsCopied> ProviderSpecificCopyInternal<T>(
ProviderConnections providerConnections,
ITable<T> table,
BulkCopyOptions options,
Func<List<Mapping.ColumnDescriptor>, BulkCopyReader<T>> createDataReader,
bool runAsync,
CancellationToken cancellationToken)
where T : notnull
{
var dataConnection = providerConnections.DataConnection;
var connection = providerConnections.ProviderConnection;
var transaction = providerConnections.ProviderTransaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rc = new BulkCopyRowsCopied();
var hanaOptions = SapHanaProviderAdapter.HanaBulkCopyOptions.Default;
if (options.KeepIdentity == true) hanaOptions |= SapHanaProviderAdapter.HanaBulkCopyOptions.KeepIdentity;
using (var bc = _provider.Adapter.CreateBulkCopy(connection, hanaOptions, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.HanaRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
args.Abort = true;
};
}
if (options.MaxBatchSize.HasValue)
bc.BatchSize = options.MaxBatchSize.Value;
if (options.BulkCopyTimeout.HasValue)
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout)
bc.BulkCopyTimeout = connection.ConnectionTimeout;
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder(dataConnection.MappingSchema);
var tableName = GetTableName(sqlBuilder, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, columns[i].ColumnName));
var rd = createDataReader(columns);
await TraceActionAsync(
dataConnection,
() => (runAsync && bc.CanWriteToServerAsync ? "INSERT ASYNC BULK " : "INSERT BULK ") + tableName + Environment.NewLine,
async () => {
if (runAsync && bc.CanWriteToServerAsync)
await bc.WriteToServerAsync(rd, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
else
bc.WriteToServer(rd);
return rd.Count;
}).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
options.RowsCopiedCallback(rc);
}
return rc;
}
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
var connection = dataConnection.Connection as SqlConnection;
if (connection != null)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _dataProvider.CreateSqlBuilder();
var rd = new BulkCopyReader(_dataProvider, columns, source);
var sqlopt = SqlBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SqlBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SqlBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SqlBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SqlBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SqlBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SqlBulkCopyOptions.UseInternalTransaction;
}
using (var bc = new SqlBulkCopy(connection, sqlopt, (SqlTransaction)dataConnection.Transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.SqlRowsCopied += (sender, e) =>
{
rc.RowsCopied = e.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
e.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, descriptor);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(new SqlBulkCopyColumnMapping(
i,
sb.Convert(columns[i].ColumnName, ConvertType.NameToQueryField).ToString()));
}
bc.WriteToServer(rd);
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
return(MultipleRowsCopy(dataConnection, options, source));
}
private BulkCopyRowsCopied ProviderSpecificCopySyncImpl <T>(
DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source,
IDbConnection connection,
string tableName,
ColumnDescriptor[] columns,
NpgsqlProviderAdapter.NpgsqlDbType[] npgsqlTypes,
string copyCommand,
int batchSize,
NpgsqlProviderAdapter.NpgsqlBinaryImporter writer)
{
var currentCount = 0;
var rowsCopied = new BulkCopyRowsCopied();
try
{
foreach (var item in source)
{
writer.StartRow();
for (var i = 0; i < columns.Length; i++)
{
writer.Write(columns[i].GetValue(item !), npgsqlTypes[i]);
}
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
if (!writer.HasComplete && !writer.HasComplete5)
{
writer.Cancel();
}
break;
}
}
if (currentCount >= batchSize)
{
if (writer.HasComplete)
{
writer.Complete();
}
else if (writer.HasComplete5)
{
writer.Complete5();
}
writer.Dispose();
writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() =>
{
if (writer.HasComplete)
{
writer.Complete();
}
else if (writer.HasComplete5)
{
writer.Complete5();
}
return((int)rowsCopied.RowsCopied);
});
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
catch when(!writer.HasComplete && !writer.HasComplete5)
{
writer.Cancel();
throw;
}
finally
{
writer.Dispose();
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
var connection = dataConnection.Connection;
if (connection == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType)))
{
return(MultipleRowsCopy(dataConnection, options, source));
}
var transaction = dataConnection.Transaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rc = new BulkCopyRowsCopied();
if (_bulkCopyCreator == null)
{
var clientNamespace = _dataProvider.ConnectionNamespace;
var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopy", false);
_bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyOptions", false);
var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false);
var transactionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaTransaction", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, _bulkCopyOptionType, transactionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
int hanaOptions = 0; //default;
if (options.KeepIdentity == true)
{
// instead of adding new option in HANA 2 provider to a free bit to preserve compatibility,
// SAP reused value, assigned to TableLock before
if (Enum.GetNames(_bulkCopyOptionType).Any(_ => _ == KeepIdentityOptionName))
{
hanaOptions = hanaOptions | (int)Enum.Parse(_bulkCopyOptionType, KeepIdentityOptionName);
}
else
{
throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by your SAP HANA provider version");
}
}
using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction))
{
dynamic dbc = bc;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied");
}
dbc.NotifyAfter = options.NotifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.MaxBatchSize.HasValue)
{
dbc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, descriptor);
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source);
TraceAction(
dataConnection,
"INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, descriptor);
if (dataConnection.Transaction == null)
{
if (_bulkCopyCreator == null)
{
var clientNamespace = ((OracleDataProvider)dataConnection.DataProvider).AssemblyName + ".Client.";
var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopy", false);
var bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopyOptions", false);
var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + "OracleBulkCopyColumnMapping", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator != null)
{
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToList();
var rd = new BulkCopyReader(_dataProvider, columns, source);
var rc = new BulkCopyRowsCopied();
var bcOptions = 0; // Default
if (options.UseInternalTransaction == true)
{
bcOptions |= 1; // UseInternalTransaction = 1,
}
using (var bc = _bulkCopyCreator(dataConnection.Connection, bcOptions))
{
dynamic dbc = bc;
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ?
options.MaxBatchSize.Value : options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "OracleRowsCopied");
}
dbc.NotifyAfter = notifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.MaxBatchSize.HasValue)
{
dbc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
"INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
options.BulkCopyType = BulkCopyType.MultipleRows;
return(MultipleRowsCopy(dataConnection, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (_provider.Adapter.BulkCopy != null && table.DataContext is DataConnection dataConnection)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
var transaction = dataConnection.Transaction;
if (connection != null && transaction != null)
{
transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema);
}
if (connection != null && (dataConnection.Transaction == null || transaction != null))
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rc = new BulkCopyRowsCopied();
var bc = _provider.Adapter.BulkCopy.Create(connection, transaction);
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.MySqlRowsCopied += (sender, args) =>
{
rc.RowsCopied += args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = GetTableName(sb, options, table);
for (var i = 0; i < columns.Count; i++)
{
bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
// emulate missing BatchSize property
// this is needed, because MySql fails on big batches, so users should be able to limit batch size
foreach (var batch in EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue))
{
var rd = new BulkCopyReader(dataConnection, columns, batch);
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
rc.RowsCopied += rd.Count;
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
