/// <summary> /// Runs sql bulk insert using custom IDataReader /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities"></param> /// <param name="transaction"></param> public override async Task RunAsync <T>(IEnumerable <T> entities, HanaTransaction transaction) { var sqlBulkCopyOptions = ToHanaBulkCopyOptions(Options.BulkCopyOptions); var keepIdentity = (HanaBulkCopyOptions.KeepIdentity & sqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { using (var bulkCopy = new HanaBulkCopy(transaction.Connection, sqlBulkCopyOptions, transaction)) { bulkCopy.BulkCopyTimeout = Options.TimeOut; bulkCopy.BatchSize = Options.BatchSize; bulkCopy.DestinationTableName = string.Format("[{0}].[{1}]", reader.SchemaName, reader.TableName); bulkCopy.NotifyAfter = Options.NotifyAfter; if (Options.Callback != null) { bulkCopy.HanaRowsCopied += (sender, args) => { Options.Callback.Invoke(sender, new RowsCopiedEventArgs(args.RowsCopied)); }; } foreach (var kvp in reader.Cols) { if (kvp.Value.IsIdentity && !keepIdentity) { continue; } bulkCopy.ColumnMappings.Add(kvp.Value.ColumnName, kvp.Value.ColumnName); } await bulkCopy.WriteToServerAsync(reader); } } }
public static void BulkInsertWithTransaction <T>(IEnumerable <T> entityCollection, DbContext context) { var provider = new EfSqlBulkInsertProviderWithMappedDataReader(); provider.SetContext(context); SqlTransaction transaction = context.Database.CurrentTransaction?.UnderlyingTransaction as SqlTransaction; using (var mappedDataReader = new MappedDataReader <T>(entityCollection, provider)) { var databaseConnection = context.Database.Connection as SqlConnection ?? throw new InvalidOperationException("Can't cast Context.Database.Connection to type 'SqlConnection'"); using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy(databaseConnection, SqlBulkCopyOptions.Default, transaction)) { sqlBulkCopy.DestinationTableName = $"[{mappedDataReader.SchemaName}].[{mappedDataReader.TableName}]"; using (var enumerator = mappedDataReader.Cols.GetEnumerator()) { while (enumerator.MoveNext()) { KeyValuePair <int, IPropertyMap> current = enumerator.Current; sqlBulkCopy.ColumnMappings.Add(current.Value.ColumnName, current.Value.ColumnName); } } sqlBulkCopy.WriteToServer(mappedDataReader); } } }
/// <summary> /// Runs sql bulk insert using custom IDataReader /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities"></param> /// <param name="transaction"></param> public override void Run <T>(IEnumerable <T> entities, SqlTransaction transaction) { var keepIdentity = (SqlBulkCopyOptions.KeepIdentity & Options.SqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { using (var sqlBulkCopy = new SqlBulkCopy(transaction.Connection, Options.SqlBulkCopyOptions, transaction)) { sqlBulkCopy.BulkCopyTimeout = Options.TimeOut; sqlBulkCopy.BatchSize = Options.BatchSize; sqlBulkCopy.DestinationTableName = string.Format("[{0}].[{1}]", reader.SchemaName, reader.TableName); #if !NET40 sqlBulkCopy.EnableStreaming = Options.EnableStreaming; #endif sqlBulkCopy.NotifyAfter = Options.NotifyAfter; if (Options.Callback != null) { sqlBulkCopy.SqlRowsCopied += Options.Callback; } foreach (var kvp in reader.Cols) { if (kvp.Value.IsIdentity && !keepIdentity) { continue; } sqlBulkCopy.ColumnMappings.Add(kvp.Value.ColumnName, kvp.Value.ColumnName); } sqlBulkCopy.WriteToServer(reader); } } }
/// <summary> /// Runs sql bulk insert using custom IDataReader /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities"></param> /// <param name="transaction"></param> public override async Task RunAsync <T>(IEnumerable <T> entities, SqlTransaction transaction) { var keepIdentity = (SqlBulkCopyOptions.KeepIdentity & this.Options.SqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { using (var sqlBulkCopy = new SqlBulkCopy(transaction.Connection, this.Options.SqlBulkCopyOptions, transaction)) { sqlBulkCopy.BulkCopyTimeout = this.Options.TimeOut; sqlBulkCopy.BatchSize = this.Options.BatchSize; sqlBulkCopy.DestinationTableName = $"[{reader.SchemaName}].[{reader.TableName}]"; sqlBulkCopy.EnableStreaming = this.Options.EnableStreaming; sqlBulkCopy.NotifyAfter = this.Options.NotifyAfter; if (this.Options.Callback != null) { sqlBulkCopy.SqlRowsCopied += this.Options.Callback; } foreach (var kvp in reader.Cols) { if (kvp.Value.IsIdentity && !keepIdentity) { continue; } sqlBulkCopy.ColumnMappings.Add(kvp.Value.ColumnName, kvp.Value.ColumnName); } await sqlBulkCopy.WriteToServerAsync(reader); } } }
public override void Run <T>(IEnumerable <T> entities, SqlTransaction transaction, BulkInsertOptions options) { var flag = (SqlBulkCopyOptions.KeepIdentity & options.SqlBulkCopyOptions) > SqlBulkCopyOptions.Default; using (var mappedDataReader = new MappedDataReader <T>(entities, this)) { using (var sqlBulkCopy = new SqlBulkCopy(transaction.Connection, options.SqlBulkCopyOptions, transaction)) { sqlBulkCopy.BulkCopyTimeout = options.TimeOut; sqlBulkCopy.BatchSize = options.BatchSize; sqlBulkCopy.DestinationTableName = $"[{ mappedDataReader.SchemaName}].[{ mappedDataReader.TableName}]"; sqlBulkCopy.EnableStreaming = options.EnableStreaming; sqlBulkCopy.NotifyAfter = options.NotifyAfter; if (options.Callback != null) { sqlBulkCopy.SqlRowsCopied += options.Callback; } foreach (var col in mappedDataReader.Cols) { if (!col.Value.IsIdentity || flag) { sqlBulkCopy.ColumnMappings.Add(col.Value.ColumnName, col.Value.ColumnName); } } sqlBulkCopy.WriteToServer(mappedDataReader); } } }
public void ComplexTypeReader() { var user = new TestUser { Contact = new Contact { Address = new Address { City = "Tallinn", Country = "Estonia"}, PhoneNumber = "1234567"}, FirstName = "Max", LastName = "Lego", Id = Guid.NewGuid() }; var emptyUser = new TestUser(); using (var ctx = new TestContext()) { var tableMapping = ctx.Db<TestUser>(); var tableMappings = new Dictionary<Type, IEntityMap> { {typeof (TestUser), tableMapping} }; using (var reader = new MappedDataReader<TestUser>(new[] { user, emptyUser }, tableMappings)) { Assert.AreEqual(9, reader.FieldCount); while (reader.Read()) { for (int i = 0; i < reader.FieldCount; ++i) { Console.WriteLine("{0}: {1}", i, reader.GetValue(i)); } } } } }
public void ComplexTypeReader() { var user = new TestUser { Contact = new Contact { Address = new Address { City = "Tallinn", Country = "Estonia" }, PhoneNumber = "1234567" }, FirstName = "Max", LastName = "Lego", Id = Guid.NewGuid() }; var emptyUser = new TestUser(); using (var ctx = new TestBaseContext()) { using (var reader = new MappedDataReader <TestUser>(new[] { user, emptyUser }, GetDummyProvider(ctx))) { Assert.AreEqual(11, reader.FieldCount); while (reader.Read()) { for (int i = 0; i < reader.FieldCount; ++i) { Console.WriteLine("{0}: {1}", i, reader.GetValue(i)); } } } } }
/* * public override void Run<T>(IEnumerable<T> entities, BulkInsertOptions options) * { * throw new System.NotImplementedException(); * } */ public override void Run <T>(IEnumerable <T> entities, SqlTransaction transaction, SqlBulkCopyOptions options, int batchSize) { var baseType = typeof(T); var allTypes = baseType.GetDerivedTypes(true); var neededMappings = allTypes.ToDictionary(x => x, x => Context.Db(x)); var keepIdentity = (SqlBulkCopyOptions.KeepIdentity & options) > 0; using (var reader = new MappedDataReader <T>(entities, neededMappings, keepIdentity)) { using (var sqlBulkCopy = new SqlBulkCopy(transaction.Connection, options, transaction)) { sqlBulkCopy.BatchSize = batchSize; sqlBulkCopy.DestinationTableName = string.Format("[{0}].[{1}]", reader.SchemaName, reader.TableName); //sqlBulkCopy.DestinationTableName = reader.TableName; #if !NET40 //sqlBulkCopy.EnableStreaming = true; #endif foreach (var kvp in reader.Mappings) { sqlBulkCopy.ColumnMappings.Add(kvp.Key, kvp.Value); } sqlBulkCopy.WriteToServer(reader); } } }
public void Performance() { var sw = new Stopwatch(); var swv = new Stopwatch(); using (var ctx = new TestBaseContext()) { ctx.Database.Initialize(false); sw.Restart(); using (var reader = new MappedDataReader <Page>(CreatePages(1000000), GetDummyProvider(ctx))) { Console.WriteLine("Construct {0}ms", sw.Elapsed.TotalMilliseconds); while (reader.Read()) { foreach (var col in reader.Cols) { swv.Start(); var value = reader.GetValue(col.Key); swv.Stop(); } } } sw.Stop(); Console.WriteLine("Elapsed {0}ms. Getting values took {1}ms", sw.Elapsed.TotalMilliseconds, swv.Elapsed.TotalMilliseconds); } }
public override void Run <T>(IEnumerable <T> entities, MySqlTransaction transaction, BulkInsertOptions options) { var flag = (SqlBulkCopyOptions.KeepIdentity & options.SqlBulkCopyOptions) > SqlBulkCopyOptions.Default; using (var mappedDataReader = new MappedDataReader <T>(entities, this)) { using (transaction) { var mySqlCommand = new StringBuilder($"INSERT INTO {mappedDataReader.TableName} ("); for (var i = 0; i < mappedDataReader.Cols.Count; i++) { mySqlCommand.Append($"{mappedDataReader.Cols[i]}"); if (i < mappedDataReader.Cols.Count) { mySqlCommand.Append(","); } } mySqlCommand.Append(") VALUES"); do { //mappedDataReader. } while (mappedDataReader.NextResult()); } } }
/// <summary> /// Runs sql bulk insert using custom IDataReader /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities"></param> /// <param name="transaction"></param> public override void Run <T>(IEnumerable <T> entities, HanaTransaction transaction) { var sqlBulkCopyOptions = ToHanaBulkCopyOptions(Options.BulkCopyOptions); var keepIdentity = (HanaBulkCopyOptions.KeepIdentity & sqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { using (var bulkCopy = new HanaBulkCopy(transaction.Connection, sqlBulkCopyOptions, transaction)) { bulkCopy.BulkCopyTimeout = Options.TimeOut; bulkCopy.BatchSize = Options.BatchSize; bulkCopy.DestinationTableName = string.Format("\"{0}\".\"{1}\"", reader.SchemaName, reader.TableName); bulkCopy.NotifyAfter = Options.NotifyAfter; if (Options.Callback != null) { bulkCopy.HanaRowsCopied += (sender, args) => { Options.Callback.Invoke(sender, new RowsCopiedEventArgs(args.RowsCopied)); }; } var table = new DataTable(reader.TableName); foreach (var kvp in reader.Cols) { var dataType = kvp.Value.Type; if (dataType.IsConstructedGenericType) { dataType = dataType.GenericTypeArguments[0]; } table.Columns.Add(kvp.Value.ColumnName, dataType); if (kvp.Value.IsIdentity && !keepIdentity) { continue; } bulkCopy.ColumnMappings.Add(kvp.Value.ColumnName, kvp.Value.ColumnName); } foreach (var x in entities) { var row = table.NewRow(); foreach (var p in reader.Cols.Values) { row[p.ColumnName] = p.Selector.DynamicInvoke(x) ?? DBNull.Value; } table.Rows.Add(row); } ; bulkCopy.WriteToServer(table); } } }
public void SimpleTableReader() { using (var ctx = new TestBaseContext()) { using (var reader = new MappedDataReader <Page>(new[] { new Page { Title = "test" } }, GetDummyProvider(ctx))) { Assert.AreEqual(6, reader.FieldCount); } } }
public void SimpleTableReader() { using (var ctx = new TestContext()) { var tableMapping = ctx.Db<Page>(); var tableMappings = new Dictionary<Type, IEntityMap> { {typeof (Page), tableMapping} }; using (var reader = new MappedDataReader<Page>(new[] {new Page { Title = "test"}}, tableMappings)) { Assert.AreEqual(6, reader.FieldCount); } } }
public void SimpleTableReader() { using (var ctx = new TestContext()) { var tableMapping = ctx.Db <Page>(); var tableMappings = new Dictionary <Type, IEntityMap> { { typeof(Page), tableMapping } }; using (var reader = new MappedDataReader <Page>(new[] { new Page { Title = "test" } }, tableMappings)) { Assert.AreEqual(6, reader.FieldCount); } } }
/// <summary> /// Runs sql bulk insert using custom IDataReader /// </summary> /// <typeparam name="T"></typeparam> /// <param name="entities"></param> /// <param name="transaction"></param> public override void Run <T>(IEnumerable <T> entities, SqlTransaction transaction) { var provider = Locator.Current.Resolve <IMappingProvider>(); var entiyMap = provider != null?provider.GetEntityMap(typeof(T), Context) : null; if (entiyMap == null) { return; } var keepIdentity = (SqlBulkCopyOptions.KeepIdentity & Options.SqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, entiyMap)) { using (var sqlBulkCopy = new SqlBulkCopy(transaction.Connection, Options.SqlBulkCopyOptions, transaction)) { sqlBulkCopy.BulkCopyTimeout = Options.TimeOut; sqlBulkCopy.BatchSize = Options.BatchSize; sqlBulkCopy.DestinationTableName = entiyMap.TableName; #if !NET40 sqlBulkCopy.EnableStreaming = Options.EnableStreaming; #endif sqlBulkCopy.NotifyAfter = Options.NotifyAfter; if (Options.Callback != null) { sqlBulkCopy.SqlRowsCopied += Options.Callback; } foreach (var kvp in reader.Cols) { if (/*kvp.Value.IsIdentity &&*/ !keepIdentity) { continue; } sqlBulkCopy.ColumnMappings.Add(kvp.Value.ColumnName, kvp.Value.ColumnName); } sqlBulkCopy.WriteToServer(reader); } } }
public void ComplexTypeReader() { var user = new TestUser { Contact = new Contact { Address = new Address { City = "Tallinn", Country = "Estonia" }, PhoneNumber = "1234567" }, FirstName = "Max", LastName = "Lego", Id = Guid.NewGuid() }; var emptyUser = new TestUser(); using (var ctx = new TestContext()) { var tableMapping = ctx.Db <TestUser>(); var tableMappings = new Dictionary <Type, IEntityMap> { { typeof(TestUser), tableMapping } }; using (var reader = new MappedDataReader <TestUser>(new[] { user, emptyUser }, tableMappings)) { Assert.AreEqual(9, reader.FieldCount); while (reader.Read()) { for (int i = 0; i < reader.FieldCount; ++i) { Console.WriteLine("{0}: {1}", i, reader.GetValue(i)); } } } } }
private void Run <T>(IEnumerable <T> entities, MySqlConnection connection, MySqlTransaction transaction) { bool keepIdentity = (BulkCopyOptions.KeepIdentity & Options.BulkCopyOptions) > 0; bool keepNulls = (BulkCopyOptions.KeepNulls & Options.BulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { var tableEngine = GetTableEngine(connection.Database, reader.TableName, connection); var columns = reader.Cols .Where(x => !x.Value.Computed && (!x.Value.IsIdentity || keepIdentity)) .ToArray(); // INSERT INTO [TableName] (column list) var insert = new StringBuilder() .Append($" INSERT INTO {reader.TableName} ") .Append("(") .Append(string.Join(",", columns.Select(col => col.Value.ColumnName))) .Append(")") .Append(" VALUES") .ToString(); int i = 0; long rowsCopied = 0; var rows = new List <string>(); while (reader.Read()) { var values = new List <string>(); foreach (var col in columns) { var value = reader.GetValue(col.Key); var type = col.Value.Type; AddParameter(type, value, values); } rows.Add("(" + string.Join(",", values) + ")"); i++; if (i == Options.BatchSize || i == Options.NotifyAfter) { using (var cmd = CreateCommand(CreateInsertBatchText(insert, rows, tableEngine), connection, transaction)) cmd.ExecuteNonQueryAsync(); if (Options.Callback != null) { int batches = Options.BatchSize / Options.NotifyAfter; rowsCopied += i; Options.Callback(this, new RowsCopiedEventArgs(rowsCopied)); } i = 0; rows.Clear(); } } if (rows.Any()) { using (var cmd = CreateCommand(CreateInsertBatchText(insert, rows, tableEngine), connection, transaction)) cmd.ExecuteNonQuery(); } } }
private void Run<T>(IEnumerable<T> entities, MySqlConnection connection, MySqlTransaction transaction) { if (!entities.Any()) return; bool keepIdentity = (BulkCopyOptions.KeepIdentity & Options.BulkCopyOptions) > 0; bool keepNulls = (BulkCopyOptions.KeepNulls & Options.BulkCopyOptions) > 0; using (var reader = new MappedDataReader<T>(entities, this)) { var tableEngine = GetTableEngine(connection.Database, reader.TableName, connection); var keyColumns = reader.Cols .Where(x => !x.Value.Computed && x.Value.IsPk) .ToArray(); var columns = reader.Cols .Where(x => !x.Value.Computed && !x.Value.IsPk && (!x.Value.IsIdentity || keepIdentity)) .ToArray(); // UPDATE [TableName] (column list) var updateHeader = $"UPDATE `{reader.TableName}` SET "; var updateBuilder = new StringBuilder(); int i = 0; long rowsCopied = 0; while (reader.Read()) { updateBuilder.Append(updateHeader); foreach (var col in columns) { var value = reader.GetValue(col.Key); var type = col.Value.Type; updateBuilder.AppendFormat("{0}={1},", col.Value.ColumnName, AddParameter(type, value)); } updateBuilder.Remove(updateBuilder.Length - 1, 1);//remove last , updateBuilder.Append(" WHERE "); foreach(var col in keyColumns) { var value = reader.GetValue(col.Key); var type = col.Value.Type; updateBuilder.AppendFormat("{0}={1} AND ", col.Value.ColumnName, AddParameter(type, value)); } updateBuilder.Remove(updateBuilder.Length - 5, 5);//remove last AND updateBuilder.Append(";"); i++; if (i == Options.BatchSize || i == Options.NotifyAfter) { using (var cmd = CreateCommand(CreateUpdateBatchText(updateBuilder, tableEngine), connection, transaction)) cmd.ExecuteNonQueryAsync(); if (Options.Callback != null) { int batches = Options.BatchSize / Options.NotifyAfter; rowsCopied += i; Options.Callback(this, new RowsCopiedEventArgs(rowsCopied)); } i = 0; } } if (i>0) { using (var cmd = CreateCommand(CreateUpdateBatchText(updateBuilder, tableEngine), connection, transaction)) cmd.ExecuteNonQuery(); } } }
public override void Run <T>(IEnumerable <T> entities, MySqlTransaction transaction, BulkInsertOptions options) { var keepIdentity = (SqlBulkCopyOptions.KeepIdentity & options.SqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { var csvPath = AppDomain.CurrentDomain.BaseDirectory + System.Guid.NewGuid().ToString() + ".csv"; Dictionary <int, IPropertyMap> propertyMaps = reader.Cols .Where(x => !x.Value.IsIdentity || keepIdentity) .ToDictionary(x => x.Key, x => x.Value); var csv = new StringBuilder(); while (reader.Read()) { foreach (var kvp in propertyMaps) { var value = reader.GetValue(kvp.Key); if (value == null) { csv .Append("null") .Append(FieldTerminator); } else { // todo - escape "'" csv .AppendFormat("{0}", value) .Append(FieldTerminator); } } csv.Append(LineTerminator); } // todo - save csv File.WriteAllText(csvPath, csv.ToString()); // upload csv var mySqlBulkLoader = new MySqlBulkLoader(transaction.Connection); //mySqlBulkLoader.TableName = string.Format("[{0}].[{1}]", reader.SchemaName, reader.TableName); mySqlBulkLoader.TableName = reader.TableName; mySqlBulkLoader.FieldTerminator = FieldTerminator; mySqlBulkLoader.LineTerminator = LineTerminator; mySqlBulkLoader.FileName = csvPath; foreach (var kvp in propertyMaps) { mySqlBulkLoader.Columns.Add(kvp.Value.ColumnName); } int count = mySqlBulkLoader.Load(); try { File.Delete(csvPath); } catch (Exception exception) { } } }
private void Run <T>(IEnumerable <T> entities, SqlCeConnection connection, SqlCeTransaction transaction) { bool runIdentityScripts; bool keepIdentity = runIdentityScripts = (SqlBulkCopyOptions.KeepIdentity & Options.SqlBulkCopyOptions) > 0; var keepNulls = (SqlBulkCopyOptions.KeepNulls & Options.SqlBulkCopyOptions) > 0; using (var reader = new MappedDataReader <T>(entities, this)) { var identityCols = reader.Cols.Values.Where(x => x.IsIdentity).ToArray(); if (identityCols.Length != 1 || !IsValidIdentityType(identityCols[0].Type)) { runIdentityScripts = false; } if (keepIdentity && runIdentityScripts) { SetIdentityInsert(connection, transaction, reader.TableName, true); } var colInfos = ColInfos(connection, reader) .Values .Where(x => !x.IsIdentity || keepIdentity) .ToArray(); using (var cmd = CreateCommand(reader.TableName, connection, transaction)) { cmd.CommandType = CommandType.TableDirect; using (var rs = cmd.ExecuteResultSet(ResultSetOptions.Updatable)) { var rec = rs.CreateRecord(); int i = 0; long rowsCopied = 0; while (reader.Read()) { foreach (var colInfo in colInfos) { var value = reader.GetValue(colInfo.ReaderKey); if (value == null && keepNulls) { rec.SetValue(colInfo.OrdinalPosition, DBNull.Value); } else { rec.SetValue(colInfo.OrdinalPosition, value); } } rs.Insert(rec); ++i; if (i == Options.NotifyAfter && Options.Callback != null) { rowsCopied += i; Options.Callback(this, new SqlRowsCopiedEventArgs(rowsCopied)); i = 0; } } } } if (keepIdentity && runIdentityScripts) { SetIdentityInsert(connection, transaction, reader.TableName, false); } } }
private static Dictionary <string, ColInfo> ColInfos <T>(SqlCeConnection sqlCeConnection, MappedDataReader <T> reader) { var dtColumns = sqlCeConnection.GetSchema("Columns"); var colInfos = new Dictionary <string, ColInfo>(); foreach (DataRow row in dtColumns.Rows) { var tableName = (string)row.ItemArray[2]; if (tableName != reader.TableName) { continue; } var columnName = (string)row.ItemArray[3]; var ordinal = (int)row.ItemArray[4] - 1; colInfos[columnName] = new ColInfo { OrdinalPosition = ordinal }; } foreach (var kvp in reader.Cols) { var colName = kvp.Value.ColumnName; if (colInfos.ContainsKey(colName)) { colInfos[colName].ReaderKey = kvp.Key; colInfos[colName].IsIdentity = kvp.Value.IsIdentity; } } return(colInfos); }