Beispiel #1
0
 protected virtual async Task OnPostBulkInsert(SqlConnection sqlConnection, TargetTable target, T[] insertedData)
 {
     if (m_postBulkInsert != null)
     {
         await m_postBulkInsert(sqlConnection, m_targetTable, insertedData).ConfigureAwait(false);
     }
 }
Beispiel #2
0
        public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
            : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
        {
            m_longConnection = new SqlConnection(targetTable.ConnectionString);
            m_longConnection.Open();

            m_transaction = m_longConnection.BeginTransaction();
        }
Beispiel #3
0
        /// <summary>
        /// Constructs an instance of DbBulkInserter
        /// </summary>
        /// <param name="targetTable">Information about the database target and mapping label</param>
        /// <param name="options">Options to use for this dataflow</param>
        /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
        /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
        /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
        public DbBulkInserterBase(TargetTable targetTable,
                                  DataflowOptions options,
                                  int bulkSize = 4096 * 2,
                                  string dbBulkInserterName = null,
                                  PostBulkInsertDelegate <T> postBulkInsert = null)
            : base(options)
        {
            this.m_targetTable  = targetTable;
            this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);

            this.m_bulkSize           = bulkSize;
            this.m_dbBulkInserterName = dbBulkInserterName;
            this.m_postBulkInsert     = postBulkInsert;
            this.m_batchBlock         = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());

            var bulkInsertOption = options.ToExecutionBlockOption();

            //Action block deal with array references
            if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
            {
                bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
            }

            this.m_logger      = Utils.GetNamespaceLogger();
            this.m_actionBlock = new ActionBlock <T[]>(
                async array =>
            {
                m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
                try
                {
                    await this.DumpToDBAsync(array, targetTable);
                }
                catch (Exception e)
                {
                    m_logger.Error($"{this.FullName} failed bulk-inserting {array.Length} {typeof(T).Name} to db table {targetTable.TableName}", e);
                    throw;
                }
                m_logger.Debug(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
            }
                , bulkInsertOption);
            this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);


            this.RegisterChild(this.m_batchBlock);
            this.RegisterChild(this.m_actionBlock);

            this.m_timer = new Timer(
                state =>
            {
                this.TriggerBatch();
            }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
        }
Beispiel #4
0
 public TypeAccessor(TargetTable target)
 {
     m_destLabel            = target.DestLabel;
     m_connectionString     = target.ConnectionString;
     m_destinationTablename = string.IsNullOrWhiteSpace(target.TableName)
         ? typeof(T).Name
         : target.TableName;
     m_schemaTable      = new Lazy <DataTable>(this.GetSchemaTable);
     m_properties       = new Dictionary <int, Func <T, object> >();
     m_dbColumnMappings = new List <DBColumnMapping>();
     m_classLogger      = LogManager.GetLogger(this.GetType().Namespace + "." + this.GetType().GetFriendlyName());
     CreateTypeVisitor();
 }
Beispiel #5
0
        protected async virtual Task DumpToDBAsync(T[] data, TargetTable targetTable)
        {
            m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));

            using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data))
            {
                using (var conn = new SqlConnection(targetTable.ConnectionString))
                {
                    await conn.OpenAsync().ConfigureAwait(false);

                    var transaction = conn.BeginTransaction();
                    try
                    {
                        using (var bulkCopy = new SqlBulkCopy(conn, SqlBulkCopyOptions.TableLock, transaction))
                        {
                            foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings)
                            {
                                bulkCopy.ColumnMappings.Add(map);
                            }

                            bulkCopy.DestinationTableName = targetTable.TableName;
                            bulkCopy.BulkCopyTimeout      = (int)TimeSpan.FromMinutes(30).TotalMilliseconds;
                            bulkCopy.BatchSize            = m_bulkSize;

                            // Write from the source to the destination.
                            await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false);
                        }
                    }
                    catch (Exception e)
                    {
                        if (e is NullReferenceException)
                        {
                            m_logger.ErrorFormat(
                                "{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.", this.FullName);
                        }

                        m_logger.ErrorFormat("{0} Bulk insertion failed. Rolling back all changes...", this.FullName, e);
                        transaction.Rollback();
                        m_logger.InfoFormat("{0} Changes successfully rolled back", this.FullName);

                        //As this is an unrecoverable exception, rethrow it
                        throw new AggregateException(e);
                    }

                    transaction.Commit();
                    await this.OnPostBulkInsert(conn, targetTable, data).ConfigureAwait(false);
                }
            }

            m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
        }
Beispiel #6
0
        protected override async Task DumpToDBAsync(T[] data, TargetTable targetTable)
        {
            m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));

            using (var bulkReader = new BulkDataReader <T>(m_typeAccessor, data))
            {
                try
                {
                    using (var bulkCopy = new SqlBulkCopy(m_longConnection, SqlBulkCopyOptions.TableLock, m_transaction))
                    {
                        foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings)
                        {
                            bulkCopy.ColumnMappings.Add(map);
                        }

                        bulkCopy.DestinationTableName = targetTable.TableName;
                        bulkCopy.BulkCopyTimeout      = (int)TimeSpan.FromMinutes(30).TotalMilliseconds;
                        bulkCopy.BatchSize            = m_bulkSize;

                        // Write from the source to the destination.
                        await bulkCopy.WriteToServerAsync(bulkReader).ConfigureAwait(false);
                    }
                }
                catch (Exception e)
                {
                    if (e is NullReferenceException)
                    {
                        m_logger.ErrorFormat(
                            "{0} NullReferenceException occurred in bulk insertion. This is probably caused by forgetting assigning value to a [NoNullCheck] attribute when constructing your object.",
                            this.FullName);
                    }

                    m_logger.ErrorFormat("{0} Bulk insertion error in the first place", e, this.FullName);

                    //As this is an unrecoverable exception, rethrow it
                    throw new AggregateException(e);
                }

                await this.OnPostBulkInsert(m_longConnection, targetTable, data).ConfigureAwait(false);
            }

            m_logger.Info(h => h("{3} bulk-inserted {0} {1} to db table {2}", data.Length, typeof(T).Name, targetTable.TableName, this.FullName));
        }
Beispiel #7
0
        /// <summary>
        /// Constructs an instance of DbBulkInserter
        /// </summary>
        /// <param name="targetTable">Information about the database target and mapping label</param>
        /// <param name="options">Options to use for this dataflow</param>
        /// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
        /// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
        /// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
        public DbBulkInserterBase(TargetTable targetTable,
                                  DataflowOptions options,
                                  int bulkSize = 4096 * 2,
                                  string dbBulkInserterName = null,
                                  PostBulkInsertDelegate <T> postBulkInsert = null)
            : base(options)
        {
            this.m_targetTable  = targetTable;
            this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);

            this.m_bulkSize           = bulkSize;
            this.m_dbBulkInserterName = dbBulkInserterName;
            this.m_postBulkInsert     = postBulkInsert;
            this.m_batchBlock         = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());

            var bulkInsertOption = options.ToExecutionBlockOption();

            //Action block deal with array references
            if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
            {
                bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
            }

            this.m_actionBlock = new ActionBlock <T[]>(array => this.DumpToDBAsync(array, targetTable), bulkInsertOption);
            this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);
            this.m_logger = Utils.GetNamespaceLogger();

            this.RegisterChild(this.m_batchBlock);
            this.RegisterChild(this.m_actionBlock);

            this.m_timer = new Timer(
                state =>
            {
                this.TriggerBatch();
            }, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
        }
 protected bool Equals(TargetTable other)
 {
     return string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel);
 }
Beispiel #9
0
 /// <summary>
 /// If the typeAccessor exists, just return it; else create a new one with parameter: destLabel, connectionString,
 /// dataTableName
 /// </summary>
 public static TypeAccessor <T> GetAccessorForTable(TargetTable target)
 {
     return(s_accessors.GetOrAdd(target, t => new Lazy <TypeAccessor <T> >(() => new TypeAccessor <T>(t))).Value);
 }
Beispiel #10
0
 public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
     : base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
 {
 }
Beispiel #11
0
 protected abstract Task DumpToDBAsync(T[] data, TargetTable targetTable);
Beispiel #12
0
 protected bool Equals(TargetTable other)
 {
     return(string.Equals(this.ConnectionString, other.ConnectionString) && string.Equals(this.TableName, other.TableName) && string.Equals(this.DestLabel, other.DestLabel));
 }