/// <summary>从数据流恢复数据</summary> /// <param name="stream">数据流</param> /// <param name="table">数据表</param> /// <param name="progress">进度回调,参数为已处理行数和当前页表</param> /// <returns></returns> public Int32 Restore(Stream stream, IDataTable table, Action <Int32, DbTable> progress = null) { var writeDb = new WriteDbActor { Table = table, Dal = this, // 最多同时堆积数页 BoundedCapacity = 4, }; //writeDb.Start(); var sw = Stopwatch.StartNew(); // 二进制读写器 var bn = new Binary { EncodeInt = true, Stream = stream, }; var dt = new DbTable(); dt.ReadHeader(bn); WriteLog("恢复[{0}/{1}]开始,共[{2:n0}]行", table.Name, ConnName, dt.Total); // 输出日志 var cs = dt.Columns; var ts = dt.Types; WriteLog("字段[{0}]:{1}", cs.Length, cs.Join()); WriteLog("类型[{0}]:{1}", ts.Length, ts.Join(",", e => e.Name)); var row = 0; var pageSize = 10_000; var total = 0; while (true) { // 读取数据 var count = dt.ReadData(bn, Math.Min(dt.Total - row, pageSize)); var rs = dt.Rows; if (rs == null || rs.Count == 0) { break; } WriteLog("恢复[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, rs.Count); // 进度报告 progress?.Invoke(row, dt); // 批量写入数据库。克隆对象,避免被修改 writeDb.Tell(dt.Clone()); // 下一页 total += count; if (count < pageSize) { break; } row += pageSize; } // 通知写入完成 writeDb.Stop(-1); sw.Stop(); var ms = sw.Elapsed.TotalMilliseconds; WriteLog("恢复[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms); // 返回总行数 return(total); }
/// <summary>同步单表数据</summary> /// <remarks> /// 把数据同一张表同步到另一个库 /// </remarks> /// <param name="table">数据表</param> /// <param name="connName">目标连接名</param> /// <param name="syncSchema">同步架构</param> /// <param name="progress">进度回调,参数为已处理行数和当前页表</param> /// <returns></returns> public Int32 Sync(IDataTable table, String connName, Boolean syncSchema = true, Action <Int32, DbTable> progress = null) { var dal = connName.IsNullOrEmpty() ? null : Create(connName); var writeDb = new WriteDbActor { Table = table, Dal = dal, // 最多同时堆积数页 BoundedCapacity = 4, }; // 自增 var id = table.Columns.FirstOrDefault(e => e.Identity); // 主键 if (id == null) { var pks = table.PrimaryKeys; if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt()) { id = pks[0]; } } var sw = Stopwatch.StartNew(); // 表结构 if (syncSchema) { dal.SetTables(table); } var sb = new SelectBuilder { Table = Db.FormatTableName(table.TableName) }; var row = 0L; var pageSize = 10_000; var total = 0; while (true) { var sql = ""; // 分割数据页,自增或分页 if (id != null) { sb.Where = $"{id.ColumnName}>={row}"; sql = PageSplit(sb, 0, pageSize); } else { sql = PageSplit(sb, row, pageSize); } // 查询数据 var dt = Session.Query(sql, null); if (dt == null) { break; } var count = dt.Rows.Count; WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, count); // 进度报告 progress?.Invoke((Int32)row, dt); // 消费数据 writeDb.Tell(dt); // 下一页 total += count; //if (count < pageSize) break; // 自增分割时,取最后一行 if (id != null) { row = dt.Get <Int64>(count - 1, id.ColumnName) + 1; } else { row += pageSize; } } // 通知写入完成 writeDb.Stop(-1); sw.Stop(); var ms = sw.Elapsed.TotalMilliseconds; WriteLog("同步[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms); // 返回总行数 return(total); }
/// <summary>同步单表数据</summary> /// <remarks> /// 把数据同一张表同步到另一个库 /// </remarks> /// <param name="table">数据表</param> /// <param name="connName">目标连接名</param> /// <param name="syncSchema">同步架构</param> /// <param name="progress">进度回调,参数为已处理行数和当前页表</param> /// <returns></returns> public Int32 Sync(IDataTable table, String connName, Boolean syncSchema = true, Action <Int32, DbTable> progress = null) { var dal = connName.IsNullOrEmpty() ? null : Create(connName); var writeDb = new WriteDbActor { Table = table, Dal = dal, // 最多同时堆积数页 BoundedCapacity = 4, }; var sw = Stopwatch.StartNew(); // 表结构 if (syncSchema) { dal.SetTables(table); } var sb = new SelectBuilder { Table = Db.FormatTableName(table.TableName) }; var row = 0; var pageSize = 10_000; var total = 0; while (true) { // 分页 var sb2 = PageSplit(sb, row, pageSize); // 查询数据 var dt = Session.Query(sb2.ToString(), null); if (dt == null) { break; } var count = dt.Rows.Count; WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, count); // 进度报告 progress?.Invoke(row, dt); // 消费数据 writeDb.Tell(dt); // 下一页 total += count; if (count < pageSize) { break; } row += pageSize; } // 通知写入完成 writeDb.Stop(-1); sw.Stop(); var ms = sw.Elapsed.TotalMilliseconds; WriteLog("同步[{0}/{1}]完成,共[{2:n0}]行,耗时{3:n0}ms,速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms); // 返回总行数 return(total); }