Beispiel #1
0
        /// <summary>备份单表数据</summary>
        /// <remarks>
        /// 最大支持21亿行
        /// </remarks>
        /// <param name="table">数据表</param>
        /// <param name="stream">目标数据流</param>
        /// <param name="progress">进度回调,参数为已处理行数和当前页表</param>
        /// <returns></returns>
        public Int32 Backup(IDataTable table, Stream stream, Action <Int32, DbTable> progress = null)
        {
            var writeFile = new WriteFileActor
            {
                Stream = stream,

                // 最多同时堆积数
                BoundedCapacity = 4,
            };
            //writeFile.Start();

            // 自增
            var id = table.Columns.FirstOrDefault(e => e.Identity);

            // 主键
            if (id == null)
            {
                var pks = table.PrimaryKeys;
                if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt())
                {
                    id = pks[0];
                }
            }

            var sb = new SelectBuilder
            {
                Table = Db.FormatTableName(table.TableName)
            };

            // 总行数
            writeFile.Total = SelectCount(sb);
            WriteLog("备份[{0}/{1}]开始,共[{2:n0}]行", table, ConnName, writeFile.Total);

            var row      = 0L;
            var pageSize = 10_000;
            var total    = 0;
            var sw       = Stopwatch.StartNew();

            while (total < writeFile.Total)
            {
                var sql = "";
                // 分割数据页,自增或分页
                if (id != null)
                {
                    sb.Where = $"{id.ColumnName}>={row}";
                    sql      = PageSplit(sb, 0, pageSize);
                }
                else
                {
                    sql = PageSplit(sb, row, pageSize);
                }

                // 查询数据
                var dt = Session.Query(sql, null);
                if (dt == null)
                {
                    break;
                }

                var count = dt.Rows.Count;
                WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, row, count);
                if (count == 0)
                {
                    break;
                }

                // 进度报告
                progress?.Invoke((Int32)row, dt);

                // 消费数据
                writeFile.Tell(dt);

                // 下一页
                total += count;
                //if (count < pageSize) break;

                // 自增分割时,取最后一行
                if (id != null)
                {
                    row = dt.Get <Int64>(count - 1, id.ColumnName) + 1;
                }
                else
                {
                    row += pageSize;
                }
            }

            // 通知写入完成
            writeFile.Stop(-1);

            sw.Stop();
            var ms = sw.Elapsed.TotalMilliseconds;

            WriteLog("备份[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);

            // 返回总行数
            return(total);
        }
Beispiel #2
0
        /// <summary>备份单表数据</summary>
        /// <remarks>
        /// 最大支持21亿行
        /// </remarks>
        /// <param name="table">数据表</param>
        /// <param name="stream">目标数据流</param>
        /// <param name="progress">进度回调,参数为已处理行数和当前页表</param>
        /// <returns></returns>
        public Int32 Backup(IDataTable table, Stream stream, Action <Int64, DbTable> progress = null)
        {
            var writeFile = new WriteFileActor
            {
                Stream = stream,

                // 最多同时堆积数
                BoundedCapacity = 4,
            };

            // 自增
            var id = table.Columns.FirstOrDefault(e => e.Identity);

            if (id == null)
            {
                var pks = table.PrimaryKeys;
                if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt())
                {
                    id = pks[0];
                }
            }
            var tableName = Db.FormatName(table);
            var sb        = new SelectBuilder {
                Table = tableName
            };

            // 总行数
            writeFile.Total = SelectCount(sb);
            WriteLog("备份[{0}/{1}]开始,共[{2:n0}]行", table, ConnName, writeFile.Total);

            IExtracter <DbTable> extracer = new PagingExtracter(this, tableName);

            if (id != null)
            {
                extracer = new IdExtracter(this, tableName, id.ColumnName);
            }

            var sw    = Stopwatch.StartNew();
            var total = 0;

            foreach (var dt in extracer.Fetch())
            {
                var count = dt.Rows.Count;
                WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, extracer.Row, count);
                if (count == 0)
                {
                    break;
                }

                // 进度报告
                progress?.Invoke(extracer.Row, dt);

                // 消费数据
                writeFile.Tell(dt);

                total += count;
            }

            // 通知写入完成
            writeFile.Stop(-1);

            sw.Stop();
            var ms = sw.Elapsed.TotalMilliseconds;

            WriteLog("备份[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);

            // 返回总行数
            return(total);
        }
Beispiel #3
0
        /// <summary>备份单表数据</summary>
        /// <remarks>
        /// 最大支持21亿行
        /// </remarks>
        /// <param name="table">数据表</param>
        /// <param name="stream">目标数据流</param>
        /// <param name="progress">进度回调,参数为已处理行数和当前页表</param>
        /// <returns></returns>
        public Int32 Backup(String table, Stream stream, Action <Int32, DbTable> progress = null)
        {
            var writeFile = new WriteFileActor
            {
                Stream = stream,

                // 最多同时堆积数页
                BoundedCapacity = 4,
            };
            //writeFile.Start();

            var sb = new SelectBuilder
            {
                Table = Db.FormatTableName(table)
            };

            // 总行数
            writeFile.Total = SelectCount(sb);
            WriteLog("备份[{0}/{1}]开始,共[{2:n0}]行", table, ConnName, writeFile.Total);

            var row      = 0;
            var pageSize = 10_000;
            var total    = 0;
            var sw       = Stopwatch.StartNew();

            while (true)
            {
                // 分页
                var sb2 = PageSplit(sb, row, pageSize);

                // 查询数据
                var dt = Session.Query(sb2.ToString(), null);
                if (dt == null)
                {
                    break;
                }

                var count = dt.Rows.Count;
                WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, row, count);

                // 进度报告
                progress?.Invoke(row, dt);

                // 消费数据
                writeFile.Tell(dt);

                // 下一页
                total += count;
                if (count < pageSize)
                {
                    break;
                }
                row += pageSize;
            }

            // 通知写入完成
            writeFile.Stop(-1);

            sw.Stop();
            var ms = sw.Elapsed.TotalMilliseconds;

            WriteLog("备份[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);

            // 返回总行数
            return(total);
        }
Beispiel #4
0
        /// <summary>备份单表数据,抽取数据和写入文件双线程</summary>
        /// <remarks>
        /// 最大支持21亿行
        /// </remarks>
        /// <param name="table">数据表</param>
        /// <param name="stream">目标数据流</param>
        /// <returns></returns>
        public virtual Int32 Backup(IDataTable table, Stream stream)
        {
            using var span = Tracer?.NewSpan("db:Backup", table.Name);

            // 并行写入文件,提升吞吐
            var writeFile = new WriteFileActor
            {
                Stream = stream,

                // 最多同时堆积数
                BoundedCapacity = 4,
                TracerParent    = span,
                Tracer          = Tracer,
                Log             = Log,
            };

            var tableName = Dal.Db.FormatName(table);
            var sb        = new SelectBuilder {
                Table = tableName
            };
            var connName = Dal.ConnName;

            var extracer = GetExtracter(table);

            // 总行数
            writeFile.Total = Dal.SelectCount(sb);
            WriteLog("备份[{0}/{1}]开始,共[{2:n0}]行,抽取器{3}", table, connName, writeFile.Total, extracer);

            // 临时关闭日志
            var old = Dal.Db.ShowSQL;

            Dal.Db.ShowSQL      = false;
            Dal.Session.ShowSQL = false;

            var total = 0;
            var sw    = Stopwatch.StartNew();

            try
            {
                foreach (var dt in extracer.Fetch())
                {
                    var row   = extracer.Row;
                    var count = dt.Rows.Count;
                    WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, connName, row, count);
                    if (count == 0)
                    {
                        break;
                    }

                    // 字段名更换为属性名
                    for (var i = 0; i < dt.Columns.Length; i++)
                    {
                        var dc = table.GetColumn(dt.Columns[i]);
                        if (dc != null)
                        {
                            dt.Columns[i] = dc.Name;
                        }
                    }

                    // 进度报告、消费数据
                    OnProcess(table, row, dt, writeFile);

                    total += count;
                }

                // 通知写入完成
                writeFile.Stop(-1);
            }
            catch (Exception ex)
            {
                span?.SetError(ex, table);
                throw;
            }
            finally
            {
                Dal.Db.ShowSQL      = old;
                Dal.Session.ShowSQL = old;
            }

            sw.Stop();
            var ms = sw.Elapsed.TotalMilliseconds;

            WriteLog("备份[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table, connName, total, ms, total * 1000L / ms);

            // 返回总行数
            return(total);
        }