/// <summary>备份单表数据</summary>
/// <remarks>
/// 最大支持21亿行
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="stream">目标数据流</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Backup(IDataTable table, Stream stream, Action <Int32, DbTable> progress = null)
{
var writeFile = new WriteFileActor
{
Stream = stream,
// 最多同时堆积数
BoundedCapacity = 4,
};
//writeFile.Start();
// 自增
var id = table.Columns.FirstOrDefault(e => e.Identity);
// 主键
if (id == null)
{
var pks = table.PrimaryKeys;
if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt())
{
id = pks[0];
}
}
var sb = new SelectBuilder
{
Table = Db.FormatTableName(table.TableName)
};
// 总行数
writeFile.Total = SelectCount(sb);
WriteLog("备份[{0}/{1}]开始，共[{2:n0}]行", table, ConnName, writeFile.Total);
var row = 0L;
var pageSize = 10_000;
var total = 0;
var sw = Stopwatch.StartNew();
while (total < writeFile.Total)
{
var sql = "";
// 分割数据页，自增或分页
if (id != null)
{
sb.Where = $"{id.ColumnName}>={row}";
sql = PageSplit(sb, 0, pageSize);
}
else
{
sql = PageSplit(sb, row, pageSize);
}
// 查询数据
var dt = Session.Query(sql, null);
if (dt == null)
{
break;
}
var count = dt.Rows.Count;
WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, row, count);
if (count == 0)
{
break;
}
// 进度报告
progress?.Invoke((Int32)row, dt);
// 消费数据
writeFile.Tell(dt);
// 下一页
total += count;
//if (count < pageSize) break;
// 自增分割时，取最后一行
if (id != null)
{
row = dt.Get <Int64>(count - 1, id.ColumnName) + 1;
}
else
{
row += pageSize;
}
}
// 通知写入完成
writeFile.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("备份[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>备份单表数据</summary>
/// <remarks>
/// 最大支持21亿行
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="stream">目标数据流</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Backup(IDataTable table, Stream stream, Action <Int64, DbTable> progress = null)
{
var writeFile = new WriteFileActor
{
Stream = stream,
// 最多同时堆积数
BoundedCapacity = 4,
};
// 自增
var id = table.Columns.FirstOrDefault(e => e.Identity);
if (id == null)
{
var pks = table.PrimaryKeys;
if (pks != null && pks.Length == 1 && pks[0].DataType.IsInt())
{
id = pks[0];
}
}
var tableName = Db.FormatName(table);
var sb = new SelectBuilder {
Table = tableName
};
// 总行数
writeFile.Total = SelectCount(sb);
WriteLog("备份[{0}/{1}]开始，共[{2:n0}]行", table, ConnName, writeFile.Total);
IExtracter <DbTable> extracer = new PagingExtracter(this, tableName);
if (id != null)
{
extracer = new IdExtracter(this, tableName, id.ColumnName);
}
var sw = Stopwatch.StartNew();
var total = 0;
foreach (var dt in extracer.Fetch())
{
var count = dt.Rows.Count;
WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, extracer.Row, count);
if (count == 0)
{
break;
}
// 进度报告
progress?.Invoke(extracer.Row, dt);
// 消费数据
writeFile.Tell(dt);
total += count;
}
// 通知写入完成
writeFile.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("备份[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>备份单表数据</summary>
/// <remarks>
/// 最大支持21亿行
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="stream">目标数据流</param>
/// <param name="progress">进度回调，参数为已处理行数和当前页表</param>
/// <returns></returns>
public Int32 Backup(String table, Stream stream, Action <Int32, DbTable> progress = null)
{
var writeFile = new WriteFileActor
{
Stream = stream,
// 最多同时堆积数页
BoundedCapacity = 4,
};
//writeFile.Start();
var sb = new SelectBuilder
{
Table = Db.FormatTableName(table)
};
// 总行数
writeFile.Total = SelectCount(sb);
WriteLog("备份[{0}/{1}]开始，共[{2:n0}]行", table, ConnName, writeFile.Total);
var row = 0;
var pageSize = 10_000;
var total = 0;
var sw = Stopwatch.StartNew();
while (true)
{
// 分页
var sb2 = PageSplit(sb, row, pageSize);
// 查询数据
var dt = Session.Query(sb2.ToString(), null);
if (dt == null)
{
break;
}
var count = dt.Rows.Count;
WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, ConnName, row, count);
// 进度报告
progress?.Invoke(row, dt);
// 消费数据
writeFile.Tell(dt);
// 下一页
total += count;
if (count < pageSize)
{
break;
}
row += pageSize;
}
// 通知写入完成
writeFile.Stop(-1);
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("备份[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table, ConnName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
/// <summary>备份单表数据，抽取数据和写入文件双线程</summary>
/// <remarks>
/// 最大支持21亿行
/// </remarks>
/// <param name="table">数据表</param>
/// <param name="stream">目标数据流</param>
/// <returns></returns>
public virtual Int32 Backup(IDataTable table, Stream stream)
{
using var span = Tracer?.NewSpan("db:Backup", table.Name);
// 并行写入文件，提升吞吐
var writeFile = new WriteFileActor
{
Stream = stream,
// 最多同时堆积数
BoundedCapacity = 4,
TracerParent = span,
Tracer = Tracer,
Log = Log,
};
var tableName = Dal.Db.FormatName(table);
var sb = new SelectBuilder {
Table = tableName
};
var connName = Dal.ConnName;
var extracer = GetExtracter(table);
// 总行数
writeFile.Total = Dal.SelectCount(sb);
WriteLog("备份[{0}/{1}]开始，共[{2:n0}]行，抽取器{3}", table, connName, writeFile.Total, extracer);
// 临时关闭日志
var old = Dal.Db.ShowSQL;
Dal.Db.ShowSQL = false;
Dal.Session.ShowSQL = false;
var total = 0;
var sw = Stopwatch.StartNew();
try
{
foreach (var dt in extracer.Fetch())
{
var row = extracer.Row;
var count = dt.Rows.Count;
WriteLog("备份[{0}/{1}]数据 {2:n0} + {3:n0}", table, connName, row, count);
if (count == 0)
{
break;
}
// 字段名更换为属性名
for (var i = 0; i < dt.Columns.Length; i++)
{
var dc = table.GetColumn(dt.Columns[i]);
if (dc != null)
{
dt.Columns[i] = dc.Name;
}
}
// 进度报告、消费数据
OnProcess(table, row, dt, writeFile);
total += count;
}
// 通知写入完成
writeFile.Stop(-1);
}
catch (Exception ex)
{
span?.SetError(ex, table);
throw;
}
finally
{
Dal.Db.ShowSQL = old;
Dal.Session.ShowSQL = old;
}
sw.Stop();
var ms = sw.Elapsed.TotalMilliseconds;
WriteLog("备份[{0}/{1}]完成，共[{2:n0}]行，耗时{3:n0}ms，速度{4:n0}tps", table, connName, total, ms, total * 1000L / ms);
// 返回总行数
return(total);
}
