private BulkCopyRowsCopied ProviderSpecificCopySyncImpl <T>(
DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source,
IDbConnection connection,
string tableName,
ColumnDescriptor[] columns,
NpgsqlProviderAdapter.NpgsqlDbType[] npgsqlTypes,
string copyCommand,
int batchSize,
bool useComplete,
NpgsqlProviderAdapter.NpgsqlBinaryImporter writer)
{
var currentCount = 0;
var rowsCopied = new BulkCopyRowsCopied();
try
{
foreach (var item in source)
{
writer.StartRow();
for (var i = 0; i < columns.Length; i++)
{
writer.Write(columns[i].GetValue(item !), npgsqlTypes[i]);
}
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
if (!useComplete)
{
writer.Cancel();
}
break;
}
}
if (currentCount >= batchSize)
{
if (useComplete)
{
writer.Complete();
}
writer.Dispose();
writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() =>
{
if (useComplete)
{
writer.Complete();
}
return((int)rowsCopied.RowsCopied);
});
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
catch when(!useComplete)
{
writer.Cancel();
throw;
}
finally
{
writer.Dispose();
}
return(rowsCopied);
}
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IEnumerable <T> source, CancellationToken cancellationToken)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection == null)
{
return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext));
}
var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema);
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length];
for (var i = 0; i < columns.Length; i++)
{
var npgsqlType = _provider.GetNativeType(columns[i].DbType, true);
if (npgsqlType == null)
{
var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null;
if (columnType == null || columnType.Type.DataType == DataType.Undefined)
{
columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType);
}
var sb = new System.Text.StringBuilder();
sqlBuilder.BuildTypeName(sb, columnType);
npgsqlType = _provider.GetNativeType(sb.ToString(), true);
}
if (npgsqlType == null)
{
throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping.");
}
npgsqlTypes[i] = npgsqlType.Value;
}
var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
if (!writer.SupportsAsync)
{
// seems to be missing one of the required async methods; fallback to sync importer
return(ProviderSpecificCopySyncImpl(dataConnection, options, source, connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, true, writer));
}
var rowsCopied = new BulkCopyRowsCopied();
var currentCount = 0;
try
{
foreach (var item in source)
{
await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
for (var i = 0; i < columns.Length; i++)
{
await writer.WriteAsync(columns[i].GetValue(item !), npgsqlTypes[i], cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
currentCount++;
rowsCopied.RowsCopied++;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
break;
}
}
if (currentCount >= batchSize)
{
await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort)
{
await TraceActionAsync(
dataConnection,
() => "INSERT ASYNC BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
async() => {
var ret = await writer.CompleteAsync(cancellationToken)
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
return((int)rowsCopied.RowsCopied);
}).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
finally
{
await writer.DisposeAsync()
.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext);
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (!(table?.DataContext is DataConnection dataConnection))
{
throw new ArgumentNullException(nameof(dataConnection));
}
if (dataConnection.Transaction == null)
{
var sqlBuilder = dataConnection.DataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
if (_bulkCopyCreator == null)
{
var bulkCopyType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopy", false);
var bulkCopyOptionType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyOptions", false);
var columnMappingType = _connectionType.AssemblyEx().GetType("IBM.Data.DB2.DB2BulkCopyColumnMapping", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = CreateBulkCopyCreator(_connectionType, bulkCopyType, bulkCopyOptionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator != null)
{
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rd = new BulkCopyReader(dataConnection.DataProvider, dataConnection.MappingSchema, columns, source);
var rc = new BulkCopyRowsCopied();
var bcOptions = 0; // Default
if (options.KeepIdentity == true)
{
bcOptions |= 1; // KeepIdentity = 1, TableLock = 2, Truncate = 4,
}
if (options.TableLock == true)
{
bcOptions |= 2;
}
using (var bc = _bulkCopyCreator(Proxy.GetUnderlyingObject((DbConnection)dataConnection.Connection), bcOptions))
{
dynamic dbc = bc;
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue ?
options.MaxBatchSize.Value : options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "DB2RowsCopied");
}
dbc.NotifyAfter = notifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
// database name is not a part of table FQN in oracle
var serverName = options.ServerName ?? table.ServerName;
if (table.TryGetDataConnection(out var dataConnection) && _provider.Adapter.BulkCopy != null &&
serverName == null)
{
var connection = _provider.TryGetProviderConnection(dataConnection, dataConnection.Connection);
if (connection != null)
{
var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema);
// ODP.NET doesn't bulk copy doesn't work if columns that require escaping:
// - if escaping applied, pre-flight validation fails as it performs uppercase comparison and quotes make it fail with
// InvalidOperationException: Column mapping is invalid
// - if escaping not applied - if fails as expected on server, because it treats passed name as uppercased name
// and gives "ORA-00904: "STRINGVALUE": invalid identifier" error
// That's quite common error in bulk copy implementation error by providers...
var supported = true;
foreach (var column in columns)
{
if (column.ColumnName != sb.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))
{
// fallback to sql-based copy
// TODO: we should add support for by-ordinal column mapping to workaround it
supported = false;
break;
}
}
if (supported)
{
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var sqlopt = OracleProviderAdapter.BulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
var tableName = sb.ConvertInline(options.TableName ?? table.TableName, ConvertType.NameToQueryTable);
var schemaName = options.SchemaName ?? table.SchemaName;
if (schemaName != null)
{
schemaName = sb.ConvertInline(schemaName, ConvertType.NameToSchema);
}
if (options.UseInternalTransaction == true)
{
sqlopt |= OracleProviderAdapter.BulkCopyOptions.UseInternalTransaction;
}
if (options.CheckConstraints == true)
{
sqlopt |= OracleProviderAdapter.BulkCopyOptions.KeepConstraints;
}
if (options.FireTriggers != true)
{
sqlopt |= OracleProviderAdapter.BulkCopyOptions.DisableTriggers;
}
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue
? options.MaxBatchSize.Value
: options.NotifyAfter;
using (var bc = _provider.Adapter.BulkCopy.Create(
connection,
sqlopt,
tableName,
schemaName,
notifyAfter != 0 && options.RowsCopiedCallback != null ? notifyAfter : null,
options.RowsCopiedCallback,
rc,
options.MaxBatchSize,
options.BulkCopyTimeout ?? (Configuration.Data.BulkCopyUseConnectionCommandTimeout ? connection.ConnectionTimeout : null)))
{
for (var i = 0; i < columns.Count; i++)
{
bc.AddColumn(i, columns[i]);
}
//
TraceAction(
dataConnection,
() => "INSERT BULK " + (schemaName == null ? tableName : schemaName + "." + tableName) + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine,
() => { bc.Execute(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (_provider.Adapter.BulkCopy != null && table.DataContext is DataConnection dataConnection)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
var transaction = dataConnection.Transaction;
if (connection != null && transaction != null)
{
transaction = _provider.TryGetProviderTransaction(transaction, dataConnection.MappingSchema);
}
if (connection != null && (dataConnection.Transaction == null || transaction != null))
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rc = new BulkCopyRowsCopied();
var bc = _provider.Adapter.BulkCopy.Create(connection, transaction);
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.MySqlRowsCopied += (sender, args) =>
{
rc.RowsCopied += args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = GetTableName(sb, options, table);
for (var i = 0; i < columns.Count; i++)
{
bc.AddColumnMapping(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
// emulate missing BatchSize property
// this is needed, because MySql fails on big batches, so users should be able to limit batch size
foreach (var batch in EnumerableHelper.Batch(source, options.MaxBatchSize ?? int.MaxValue))
{
var rd = new BulkCopyReader(dataConnection, columns, batch);
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
rc.RowsCopied += rd.Count;
}
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}
protected override BulkCopyRowsCopied MultipleRowsCopy <T>(
DataConnection dataConnection, BulkCopyOptions options, IEnumerable <T> source)
{
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, descriptor);
var sb = new StringBuilder();
var buildValue = BasicSqlBuilder.GetBuildValue(sqlBuilder, sb);
var columns = descriptor.Columns.Where(c => !c.SkipOnInsert).ToArray();
var pname = sqlBuilder.Convert("p", ConvertType.NameToQueryParameter).ToString();
var rowsCopied = new BulkCopyRowsCopied();
sb
.AppendFormat("INSERT INTO {0}", tableName).AppendLine()
.Append("(");
foreach (var column in columns)
{
sb
.AppendLine()
.Append("\t")
.Append(sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField))
.Append(",");
}
sb.Length--;
sb
.AppendLine()
.Append(")");
sb
.AppendLine()
.Append("VALUES");
var headerLen = sb.Length;
var currentCount = 0;
var batchSize = options.MaxBatchSize ?? 1000;
if (batchSize <= 0)
{
batchSize = 1000;
}
var parms = new List <DataParameter>();
var pidx = 0;
foreach (var item in source)
{
sb
.AppendLine()
.Append("(");
foreach (var column in columns)
{
var value = column.GetValue(item);
if (value == null)
{
sb.Append("NULL");
}
else
{
switch (Type.GetTypeCode(value.GetType()))
{
case TypeCode.DBNull:
sb.Append("NULL");
break;
case TypeCode.String:
var isString = false;
switch (column.DataType)
{
case DataType.NVarChar:
case DataType.Char:
case DataType.VarChar:
case DataType.NChar:
case DataType.Undefined:
isString = true;
break;
}
if (isString)
{
goto case TypeCode.Int32;
}
goto default;
case TypeCode.Boolean:
case TypeCode.Char:
case TypeCode.SByte:
case TypeCode.Byte:
case TypeCode.Int16:
case TypeCode.UInt16:
case TypeCode.Int32:
case TypeCode.UInt32:
case TypeCode.Int64:
case TypeCode.UInt64:
case TypeCode.Single:
case TypeCode.Double:
case TypeCode.Decimal:
case TypeCode.DateTime:
//SetParameter(dataParam, "", column.DataType, value);
buildValue(value);
break;
default:
var name = pname + ++pidx;
sb.Append(name);
parms.Add(new DataParameter("p" + pidx, value, column.DataType));
break;
}
}
sb.Append(",");
}
sb.Length--;
sb.Append("),");
rowsCopied.RowsCopied++;
currentCount++;
if (currentCount >= batchSize || parms.Count > 100000 || sb.Length > 100000)
{
sb.Length--;
dataConnection.Execute(sb.AppendLine().ToString(), parms.ToArray());
if (options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
return(rowsCopied);
}
}
parms.Clear();
pidx = 0;
currentCount = 0;
sb.Length = headerLen;
}
}
if (currentCount > 0)
{
sb.Length--;
dataConnection.Execute(sb.ToString(), parms.ToArray());
sb.Length = headerLen;
if (options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rowsCopied);
}
}
return(rowsCopied);
}
protected BulkCopyRowsCopied IDSProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source,
DataConnection dataConnection,
IDbConnection connection,
InformixProviderAdapter.BulkCopyAdapter bulkCopy)
where T : notnull
{
var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema);
var rd = new BulkCopyReader <T>(dataConnection, columns, source);
var sqlopt = InformixProviderAdapter.IfxBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.KeepIdentity == true)
{
sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= InformixProviderAdapter.IfxBulkCopyOptions.TableLock;
}
using (var bc = bulkCopy.Create(connection, sqlopt))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.IfxRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(bulkCopy.CreateColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField)));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
private BulkCopyRowsCopied ProviderSpecificCopyInternal <T>(
ProviderConnections providerConnections,
ITable <T> table,
BulkCopyOptions options,
Func <List <Mapping.ColumnDescriptor>, BulkCopyReader <T> > createDataReader)
where T : notnull
{
var dataConnection = providerConnections.DataConnection;
var connection = providerConnections.ProviderConnection;
var transaction = providerConnections.ProviderTransaction;
var ed = table.DataContext.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(table.DataContext.MappingSchema);
var rd = createDataReader(columns);
var sqlopt = SqlServerProviderAdapter.SqlBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
if (options.CheckConstraints == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.CheckConstraints;
}
if (options.KeepIdentity == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepIdentity;
}
if (options.TableLock == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.TableLock;
}
if (options.KeepNulls == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.KeepNulls;
}
if (options.FireTriggers == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.FireTriggers;
}
if (options.UseInternalTransaction == true)
{
sqlopt |= SqlServerProviderAdapter.SqlBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.CreateBulkCopy(connection, sqlopt, transaction))
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.SqlRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
else if (Common.Configuration.Data.BulkCopyUseConnectionCommandTimeout)
{
bc.BulkCopyTimeout = connection.ConnectionTimeout;
}
var tableName = GetTableName(sb, options, table);
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.CreateBulkCopyColumnMapping(i, sb.ConvertInline(columns[i].ColumnName, ConvertType.NameToQueryField)));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + ")" + Environment.NewLine,
() => {
bc.WriteToServer(rd);
return(rd.Count);
});
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(ITable <T> table, BulkCopyOptions options, IEnumerable <T> source)
{
if (!(table?.DataContext is DataConnection dataConnection))
{
throw new ArgumentNullException(nameof(dataConnection));
}
var connection = dataConnection.Connection;
if (connection == null)
{
return(MultipleRowsCopy(table, options, source));
}
if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType)))
{
return(MultipleRowsCopy(table, options, source));
}
var sqlBuilder = _dataProvider.CreateSqlBuilder(dataConnection.MappingSchema);
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, table);
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray();
var writerType = _connectionType.AssemblyEx().GetType("Npgsql.NpgsqlBinaryImporter", true);
var fields = string.Join(", ", columns.Select(column => sqlBuilder.Convert(column.ColumnName, ConvertType.NameToQueryField)));
var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)";
var rowsCopied = new BulkCopyRowsCopied();
// batch size numbers not based on any strong grounds as I didn't found any recommendations for it
var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000);
var currentCount = 0;
var dc = (dynamic)connection;
var key = new { Type = typeof(T), options.KeepIdentity, ed };
var rowWriter = (Action <MappingSchema, object, ColumnDescriptor[], T>)_rowWriterCache.GetOrAdd(
key,
_ => BuildRowWriter <T>(writerType, columns, dataConnection.MappingSchema));
var writer = dc.BeginBinaryImport(copyCommand);
// https://github.com/npgsql/npgsql/issues/1646
// npgsql 4.0 will revert logic by removing explicit Cancel() and add explicit Complete()
var hasCancel = writer.GetType().GetMethod("Cancel") != null;
var hasComplete = writer.GetType().GetMethod("Complete") != null;
try
{
foreach (var item in source)
{
rowWriter(dataConnection.MappingSchema, writer, columns, item);
currentCount++;
rowsCopied.RowsCopied++;
if (currentCount >= batchSize)
{
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0)
{
options.RowsCopiedCallback(rowsCopied);
if (rowsCopied.Abort)
{
if (hasCancel)
{
writer.Cancel();
}
break;
}
}
if (hasComplete)
{
writer.Complete();
}
writer.Dispose();
writer = dc.BeginBinaryImport(copyCommand);
currentCount = 0;
}
}
if (!rowsCopied.Abort && hasComplete)
{
writer.Complete();
}
}
catch when(hasCancel)
{
writer.Cancel();
throw;
}
finally
{
writer.Dispose();
}
return(rowsCopied);
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
[JetBrains.Annotations.NotNull] DataConnection dataConnection,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (dataConnection == null)
{
throw new ArgumentNullException("dataConnection");
}
var connection = dataConnection.Connection;
if (connection == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
if (!(connection.GetType() == _connectionType || connection.GetType().IsSubclassOfEx(_connectionType)))
{
return(MultipleRowsCopy(dataConnection, options, source));
}
var transaction = dataConnection.Transaction;
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var rc = new BulkCopyRowsCopied();
if (_bulkCopyCreator == null)
{
var clientNamespace = _dataProvider.ConnectionNamespace;
var bulkCopyType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopy", false);
_bulkCopyOptionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyOptions", false);
var columnMappingType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaBulkCopyColumnMapping", false);
var transactionType = _connectionType.AssemblyEx().GetType(clientNamespace + ".HanaTransaction", false);
if (bulkCopyType != null)
{
_bulkCopyCreator = SapHanaCreateBulkCopyCreator(_connectionType, bulkCopyType, _bulkCopyOptionType, transactionType);
_columnMappingCreator = CreateColumnMappingCreator(columnMappingType);
}
}
if (_bulkCopyCreator == null)
{
return(MultipleRowsCopy(dataConnection, options, source));
}
int hanaOptions = 0; //default;
if (options.KeepIdentity == true)
{
// instead of adding new option in HANA 2 provider to a free bit to preserve compatibility,
// SAP reused value, assigned to TableLock before
if (Enum.GetNames(_bulkCopyOptionType).Any(_ => _ == KeepIdentityOptionName))
{
hanaOptions = hanaOptions | (int)Enum.Parse(_bulkCopyOptionType, KeepIdentityOptionName);
}
else
{
throw new LinqToDBException($"{nameof(BulkCopyOptions)}.{nameof(BulkCopyOptions.KeepIdentity)} = true is not supported by your SAP HANA provider version");
}
}
using (var bc = _bulkCopyCreator(connection, hanaOptions, transaction))
{
dynamic dbc = bc;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
if (_bulkCopySubscriber == null)
{
_bulkCopySubscriber = CreateBulkCopySubscriber(bc, "HannaRowsCopied");
}
dbc.NotifyAfter = options.NotifyAfter;
_bulkCopySubscriber(bc, arg =>
{
dynamic darg = arg;
rc.RowsCopied = darg.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
darg.Abort = true;
}
});
}
if (options.MaxBatchSize.HasValue)
{
dbc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
dbc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
var sqlBuilder = _dataProvider.CreateSqlBuilder();
var descriptor = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var tableName = GetTableName(sqlBuilder, options, descriptor);
dbc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
dbc.ColumnMappings.Add((dynamic)_columnMappingCreator(i, columns[i].ColumnName));
}
var rd = new BulkCopyReader(_dataProvider, dataConnection.MappingSchema, columns, source);
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + Environment.NewLine,
() => { dbc.WriteToServer(rd); return(rd.Count); });
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
protected override BulkCopyRowsCopied ProviderSpecificCopy <T>(
ITable <T> table,
BulkCopyOptions options,
IEnumerable <T> source)
{
if (table.DataContext is DataConnection dataConnection && dataConnection.Transaction == null && _provider.Adapter.BulkCopy != null)
{
var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema);
if (connection != null)
{
var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T));
var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToList();
var sb = _provider.CreateSqlBuilder(dataConnection.MappingSchema);
var rd = new BulkCopyReader(dataConnection, columns, source);
var sqlopt = OracleProviderAdapter.OracleBulkCopyOptions.Default;
var rc = new BulkCopyRowsCopied();
var tableName = GetTableName(sb, options, table);
if (options.UseInternalTransaction == true)
{
sqlopt |= OracleProviderAdapter.OracleBulkCopyOptions.UseInternalTransaction;
}
using (var bc = _provider.Adapter.BulkCopy.Create(connection, sqlopt))
{
var notifyAfter = options.NotifyAfter == 0 && options.MaxBatchSize.HasValue
? options.MaxBatchSize.Value
: options.NotifyAfter;
if (notifyAfter != 0 && options.RowsCopiedCallback != null)
{
bc.NotifyAfter = options.NotifyAfter;
bc.OracleRowsCopied += (sender, args) =>
{
rc.RowsCopied = args.RowsCopied;
options.RowsCopiedCallback(rc);
if (rc.Abort)
{
args.Abort = true;
}
};
}
if (options.MaxBatchSize.HasValue)
{
bc.BatchSize = options.MaxBatchSize.Value;
}
if (options.BulkCopyTimeout.HasValue)
{
bc.BulkCopyTimeout = options.BulkCopyTimeout.Value;
}
bc.DestinationTableName = tableName;
for (var i = 0; i < columns.Count; i++)
{
bc.ColumnMappings.Add(_provider.Adapter.BulkCopy.CreateColumnMapping(i, columns[i].ColumnName));
}
TraceAction(
dataConnection,
() => "INSERT BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine,
() => { bc.WriteToServer(rd); return(rd.Count); });
}
if (rc.RowsCopied != rd.Count)
{
rc.RowsCopied = rd.Count;
if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null)
{
options.RowsCopiedCallback(rc);
}
}
return(rc);
}
}
return(MultipleRowsCopy(table, options, source));
}