/// <summary>抽取一批数据</summary>
/// <param name="set">设置</param>
/// <returns></returns>
public virtual IList <IEntity> Fetch(IExtractSetting set)
{
if (Field == null)
{
throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的自增字段！");
}
if (set == null)
{
throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置");
}
var start = set.Row;
var size = set.BatchSize;
if (size <= 0)
{
size = 1000;
}
// 分批获取数据，如果没有取到，则结束
var list = FetchData(start, size);
// 取到数据，需要滑动窗口
if (list.Count > 0)
{
var last = (Int32)list.Last()[FieldName];
set.Row = last;
}
return(list);
}
public static void OnError(this IEnumerable <IETLModule> es, Object source, IExtractSetting set, Exception ex)
{
foreach (var item in es)
{
item.OnError(source, set, ex);
}
}
/// <summary>克隆一份设置参数</summary>
/// <param name="src"></param>
/// <returns></returns>
public static IExtractSetting Clone(this IExtractSetting src)
{
var set = new ExtractSetting();
set.Copy(src);
return(set);
}
/// <summary>每一轮启动时</summary>
/// <param name="set"></param>
/// <returns></returns>
protected override Boolean Init(IExtractSetting set)
{
// 如果目标表为空，则使用仅插入
var count = Target.Count;
InsertOnly = count == 0;
return(base.Init(set));
}
/// <summary>拷贝设置参数</summary>
/// <param name="src"></param>
/// <param name="set"></param>
public static IExtractSetting Copy(this IExtractSetting src, IExtractSetting set)
{
if (src == null | set == null)
{
return(src);
}
src.Start = set.Start;
src.End = set.End;
src.Row = set.Row;
src.Step = set.Step;
src.BatchSize = set.BatchSize;
//src.Enable = set.Enable;
return(src);
}
/// <summary>抽取一批数据</summary>
/// <param name="set">设置</param>
/// <returns></returns>
public virtual IList <IEntity> Fetch(IExtractSetting set)
{
if (Field == null)
{
throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的时间字段！");
}
if (set == null)
{
throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置");
}
// 验证时间段
var start = set.Start;
var end = set.End;
// 区间无效
if (start >= end)
{
return(null);
}
var size = set.BatchSize;
// 分批获取数据，如果没有取到，则结束
var list = FetchData(start, end, set.Row, size);
// 取到数据，需要滑动窗口
if (list.Count > 0)
{
set.Row += list.Count;
}
else
{
set.Row = 0;
}
return(list);
}
/// <summary>抽取一批数据</summary>
/// <param name="set">设置</param>
/// <returns></returns>
public virtual IList <IEntity> Fetch(IExtractSetting set)
{
if (Field == null)
{
throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的时间字段！");
}
if (set == null)
{
throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置");
}
// 验证时间段
var start = set.Start;
// 有步进且位于第一页时，重新计算最小时间
if (set.Step > 0 && set.Row == 0)
{
start = GetMinTime(start);
}
var now = DateTime.Now.AddSeconds(-set.Offset);
if (start >= now)
{
return(null);
}
// 结束时间，必须是小于当前时间的有效值
var end = set.Step <= 0 ? DateTime.MaxValue : start.AddSeconds(set.Step);
// 结束时间有效时，设定末端边界
if (set.End > DateTime.MinValue && set.End < DateTime.MaxValue && end > set.End)
{
end = set.End;
}
// 不能超过当前时间
if (set.Step > 0 && end > now)
{
end = now;
}
// 区间无效
if (start >= end)
{
return(null);
}
ActualStart = start;
ActualEnd = end;
var size = set.BatchSize;
// 如果批大小为零，表示不分批
//if (size <= 0) size = 1000;
// 分批获取数据，如果没有取到，则结束
var list = FetchData(start, end, set.Row, size);
// 取到数据，需要滑动窗口
if (list.Count > 0)
{
var last = (DateTime)list.Last()[FieldName];
// 有可能时间字段为空
if (last <= DateTime.MinValue)
{
last = list.Max(e => (DateTime)e[FieldName]);
}
// 有可能时间超出区间
if (last < start)
{
last = list.Max(e => (DateTime)e[FieldName]);
}
// 满一批，后续还有数据
if (list.Count >= size)
{
// 最大时间行数
var maxCount = list.Count(e => (DateTime)e[FieldName] == last);
// 以最后时间为起点，跳过若干行。注意可能产生连续分页的情况
if (last == set.Start)
{
set.Row = set.Row + maxCount;
}
else
{
set.Row = maxCount;
}
set.Start = last;
}
else
{
set.Start = last.AddSeconds(1);
set.Row = 0;
}
}
else if (set.Step > 0)
{
set.Start = end;
set.Row = 0;
}
return(list);
}
/// <summary>实例化</summary>
/// <param name="set"></param>
public ExtractSetting(IExtractSetting set)
{
this.Copy(set);
}
public static void OnFinished(this IEnumerable <IETLModule> es, IList <IEntity> list, IExtractSetting set, Int32 success, Double fetchCost, Double processCost)
{
foreach (var item in es)
{
item.OnFinished(list, set, success, fetchCost, processCost);
}
}
