private BulkCopyRowsCopied ProviderSpecificCopyImpl <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IEnumerable <T> source) { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection == null) { return(MultipleRowsCopy(table, options, source)); } var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length]; for (var i = 0; i < columns.Length; i++) { var npgsqlType = _provider.GetNativeType(columns[i].DbType, true); if (npgsqlType == null) { var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null; if (columnType == null || columnType.Type.DataType == DataType.Undefined) { columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType); } var sb = new System.Text.StringBuilder(); sqlBuilder.BuildTypeName(sb, columnType); npgsqlType = _provider.GetNativeType(sb.ToString(), true); } if (npgsqlType == null) { throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping."); } npgsqlTypes[i] = npgsqlType.Value; } var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var useComplete = _provider.Adapter.BinaryImporterHasComplete; var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); return(ProviderSpecificCopySyncImpl(dataConnection, options, source, connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, useComplete, writer)); }
private async Task <BulkCopyRowsCopied> ProviderSpecificCopyImplAsync <T>(DataConnection dataConnection, ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken) where T : notnull { var connection = _provider.TryGetProviderConnection(dataConnection.Connection, dataConnection.MappingSchema); if (connection == null) { return(await MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)); } var sqlBuilder = (BasicSqlBuilder)_provider.CreateSqlBuilder(dataConnection.MappingSchema); var ed = dataConnection.MappingSchema.GetEntityDescriptor(typeof(T)); var tableName = GetTableName(sqlBuilder, options, table); var columns = ed.Columns.Where(c => !c.SkipOnInsert || options.KeepIdentity == true && c.IsIdentity).ToArray(); var fields = string.Join(", ", columns.Select(column => sqlBuilder.ConvertInline(column.ColumnName, ConvertType.NameToQueryField))); var copyCommand = $"COPY {tableName} ({fields}) FROM STDIN (FORMAT BINARY)"; // batch size numbers not based on any strong grounds as I didn't found any recommendations for it var batchSize = Math.Max(10, options.MaxBatchSize ?? 10000); var npgsqlTypes = new NpgsqlProviderAdapter.NpgsqlDbType[columns.Length]; for (var i = 0; i < columns.Length; i++) { var npgsqlType = _provider.GetNativeType(columns[i].DbType, true); if (npgsqlType == null) { var columnType = columns[i].DataType != DataType.Undefined ? new SqlQuery.SqlDataType(columns[i]) : null; if (columnType == null || columnType.Type.DataType == DataType.Undefined) { columnType = columns[i].MappingSchema.GetDataType(columns[i].StorageType); } var sb = new System.Text.StringBuilder(); sqlBuilder.BuildTypeName(sb, columnType); npgsqlType = _provider.GetNativeType(sb.ToString(), true); } if (npgsqlType == null) { throw new LinqToDBException($"Cannot guess PostgreSQL type for column {columns[i].ColumnName}. Specify type explicitly in column mapping."); } npgsqlTypes[i] = npgsqlType.Value; } var writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); if (!writer.SupportsAsync) { // seems to be missing one of the required async methods; fallback to sync importer var enumerator = source.GetAsyncEnumerator(cancellationToken); await using (enumerator.ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { return(ProviderSpecificCopySyncImpl(dataConnection, options, EnumerableHelper.AsyncToSyncEnumerable(enumerator), connection, tableName, columns, npgsqlTypes, copyCommand, batchSize, writer)); } } var rowsCopied = new BulkCopyRowsCopied(); var currentCount = 0; try { await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext)) { await writer.StartRowAsync(cancellationToken).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); for (var i = 0; i < columns.Length; i++) { await writer.WriteAsync(columns[i].GetValue(item !), npgsqlTypes[i], cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } currentCount++; rowsCopied.RowsCopied++; if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null && rowsCopied.RowsCopied % options.NotifyAfter == 0) { options.RowsCopiedCallback(rowsCopied); if (rowsCopied.Abort) { break; } } if (currentCount >= batchSize) { await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); writer = _provider.Adapter.BeginBinaryImport(connection, copyCommand); currentCount = 0; } } if (!rowsCopied.Abort) { await TraceActionAsync( dataConnection, () => "INSERT ASYNC BULK " + tableName + "(" + string.Join(", ", columns.Select(x => x.ColumnName)) + Environment.NewLine, async() => { var ret = await writer.CompleteAsync(cancellationToken) .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); return((int)rowsCopied.RowsCopied); }).ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } if (options.NotifyAfter != 0 && options.RowsCopiedCallback != null) { options.RowsCopiedCallback(rowsCopied); } } finally { await writer.DisposeAsync() .ConfigureAwait(Common.Configuration.ContinueOnCapturedContext); } return(rowsCopied); }