/// <summary>同步单表数据</summary>
/// <remarks>
/// 把数据同一张表同步到另一个库
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="connName">目标连接名</param>
/// <param name="syncSchema">同步架构</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Sync(IDataTable table, String connName, Boolean syncSchema = true, Action <Int32, DbTable> progress = null)
{
var dal = connName.IsNullOrEmpty() ? null : Create(connName);
var writeDb = new WriteDbActor
{
Table = table,
Dal = dal,
// 最多同时堆积数页
BoundedCapacity = 4,
};
// 自增
var id = table.Columns.FirstOrDefault(e => e.Identity);
// 主键
if (id == null)
{
var pks = table.PrimaryKeys;
if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt())
{
id = pks[0];
}
}
var sw = Stopwatch.StartNew();
// 表结构
if (syncSchema)
{
dal.SetTables(table);
}
var sb = new SelectBuilder
{
Table = Db.FormatTableName(table.TableName)
};
var row = 0L;
var pageSize = 10_000;
var total = 0;
while (true)
{
var sql = "";
// 分割数据页，自增或分页
if (id != null)
{
sb.Where = $"{id.ColumnName}>={row}";
sql = PageSplit(sb, 0, pageSize);
}
else
{
sql = PageSplit(sb, row, pageSize);
}
// 查询数据
var dt = Session.Query(sql, null);
if (dt == null)
{
break;
}
var count = dt.Rows.Count;
WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, count);
// 进度报告
progress?.Invoke((Int32)row, dt);
// 消费数据
writeDb.Tell(dt);
// 下一页
total += count;
//if (count < pageSize) break;
// 自增分割时，取最后一行
if (id != null)
{
row = dt.Get <Int64>(count - 1, id.ColumnName) + 1;
}
else
{
row += pageSize;
}
}
// 通知写入完成
writeDb.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("同步[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>从数据流恢复数据</summary>
/// <param name="stream">数据流</param>
/// <param name="table">数据表</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Restore(Stream stream, IDataTable table, Action <Int32, DbTable> progress = null)
{
var writeDb = new WriteDbActor
{
Table = table,
Dal = this,
// 最多同时堆积数页
BoundedCapacity = 4,
};
//writeDb.Start();
var sw = Stopwatch.StartNew();
// 二进制读写器
var bn = new Binary
{
EncodeInt = true,
Stream = stream,
};
var dt = new DbTable();
dt.ReadHeader(bn);
WriteLog("恢复[{0}/{1}]开始，共[{2:n0}]行", table.Name, ConnName, dt.Total);
// 输出日志
var cs = dt.Columns;
var ts = dt.Types;
WriteLog("字段[{0}]：{1}", cs.Length, cs.Join());
WriteLog("类型[{0}]：{1}", ts.Length, ts.Join(",", e => e.Name));
var row = 0;
var pageSize = 10_000;
var total = 0;
while (true)
{
// 读取数据
var count = dt.ReadData(bn, Math.Min(dt.Total - row, pageSize));
var rs = dt.Rows;
if (rs == null || rs.Count == 0)
{
break;
}
WriteLog("恢复[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, rs.Count);
// 进度报告
progress?.Invoke(row, dt);
// 批量写入数据库。克隆对象，避免被修改
writeDb.Tell(dt.Clone());
// 下一页
total += count;
if (count < pageSize)
{
break;
}
row += pageSize;
}
// 通知写入完成
writeDb.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("恢复[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>同步单表数据</summary>
/// <remarks>
/// 把数据同一张表同步到另一个库
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="connName">目标连接名</param>
/// <param name="syncSchema">同步架构</param>
/// <returns></returns>
public virtual Int32 Sync(IDataTable table, String connName, Boolean syncSchema = true)
{
if (connName.IsNullOrEmpty())
{
throw new ArgumentNullException(nameof(connName));
}
if (table == null)
{
throw new ArgumentNullException(nameof(table));
}
using var span = Tracer?.NewSpan("db:Sync", $"{table.Name}->{connName}");
var dal = DAL.Create(connName);
var writeDb = new WriteDbActor
{
Table = table,
Dal = dal,
IgnorePageError = IgnorePageError,
Log = Log,
Tracer = Tracer,
// 最多同时堆积数页
BoundedCapacity = 4,
TracerParent = span,
};
var extracer = GetExtracter(table);
// 临时关闭日志
var old = Dal.Db.ShowSQL;
Dal.Db.ShowSQL = false;
Dal.Session.ShowSQL = false;
var total = 0;
var sw = Stopwatch.StartNew();
try
{
// 表结构
if (syncSchema)
{
dal.SetTables(table);
}
foreach (var dt in extracer.Fetch())
{
var row = extracer.Row;
var count = dt.Rows.Count;
WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, Dal.ConnName, row, count);
// 进度报告、消费数据
OnProcess(table, row, dt, writeDb);
total += count;
}
// 通知写入完成
writeDb.Stop(-1);
}
catch (Exception ex)
{
span?.SetError(ex, table);
throw;
}
finally
{
Dal.Db.ShowSQL = old;
Dal.Session.ShowSQL = old;
}
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("同步[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table.Name, Dal.ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>同步单表数据</summary>
/// <remarks>
/// 把数据同一张表同步到另一个库
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="connName">目标连接名</param>
/// <param name="syncSchema">同步架构</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Sync(IDataTable table, String connName, Boolean syncSchema = true, Action <Int32, DbTable> progress = null)
{
var dal = connName.IsNullOrEmpty() ? null : Create(connName);
var writeDb = new WriteDbActor
{
Table = table,
Dal = dal,
// 最多同时堆积数页
BoundedCapacity = 4,
};
var sw = Stopwatch.StartNew();
// 表结构
if (syncSchema)
{
dal.SetTables(table);
}
var sb = new SelectBuilder
{
Table = Db.FormatTableName(table.TableName)
};
var row = 0;
var pageSize = 10_000;
var total = 0;
while (true)
{
// 分页
var sb2 = PageSplit(sb, row, pageSize);
// 查询数据
var dt = Session.Query(sb2.ToString(), null);
if (dt == null)
{
break;
}
var count = dt.Rows.Count;
WriteLog("同步[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, ConnName, row, count);
// 进度报告
progress?.Invoke(row, dt);
// 消费数据
writeDb.Tell(dt);
// 下一页
total += count;
if (count < pageSize)
{
break;
}
row += pageSize;
}
// 通知写入完成
writeDb.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("同步[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table.Name, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>从数据流恢复数据</summary>
/// <param name="stream">数据流</param>
/// <param name="table">数据表</param>
/// <returns></returns>
public virtual Int32 Restore(Stream stream, IDataTable table)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (table == null)
{
throw new ArgumentNullException(nameof(table));
}
using var span = Tracer?.NewSpan("db:Restore", table.Name);
var writeDb = new WriteDbActor
{
Table = table,
Dal = Dal,
IgnorePageError = IgnorePageError,
Log = Log,
Tracer = Tracer,
// 最多同时堆积数页
BoundedCapacity = 4,
TracerParent = span,
};
var connName = Dal.ConnName;
// 临时关闭日志
var old = Dal.Db.ShowSQL;
Dal.Db.ShowSQL = false;
Dal.Session.ShowSQL = false;
var total = 0;
var sw = Stopwatch.StartNew();
try
{
// 二进制读写器
var bn = new Binary
{
EncodeInt = true,
Stream = stream,
};
var dt = new DbTable();
dt.ReadHeader(bn);
WriteLog("恢复[{0}/{1}]开始，共[{2:n0}]行", table.Name, connName, dt.Total);
// 输出日志
var cs = dt.Columns;
var ts = dt.Types;
for (var i = 0; i < cs.Length; i++)
{
if (ts[i] == null || ts[i] == typeof(Object))
{
var dc = table.GetColumn(cs[i]);
if (dc != null)
{
ts[i] = dc.DataType;
}
}
}
WriteLog("字段[{0}]：{1}", cs.Length, cs.Join());
WriteLog("类型[{0}]：{1}", ts.Length, ts.Join(",", e => e?.Name));
var row = 0;
var pageSize = (Dal.Db as DbBase).BatchSize;
while (true)
{
// 读取数据
dt.ReadData(bn, Math.Min(dt.Total - row, pageSize));
var rs = dt.Rows;
if (rs == null || rs.Count == 0)
{
break;
}
WriteLog("恢复[{0}/{1}]数据 {2:n0} + {3:n0}", table.Name, connName, row, rs.Count);
// 进度报告，批量写入数据库
OnProcess(table, row, dt, writeDb);
// 下一页
total += rs.Count;
if (rs.Count < pageSize)
{
break;
}
row += pageSize;
}
// 通知写入完成
writeDb.Stop(-1);
}
catch (Exception ex)
{
span?.SetError(ex, null);
throw;
}
finally
{
Dal.Db.ShowSQL = old;
Dal.Session.ShowSQL = old;
}
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("恢复[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table.Name, connName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
