/// <summary>抽取一批数据</summary> /// <param name="set">设置</param> /// <returns></returns> public virtual IList <IEntity> Fetch(IExtractSetting set) { if (Field == null) { throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的自增字段!"); } if (set == null) { throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置"); } var start = set.Row; var size = set.BatchSize; if (size <= 0) { size = 1000; } // 分批获取数据,如果没有取到,则结束 var list = FetchData(start, size); // 取到数据,需要滑动窗口 if (list.Count > 0) { var last = (Int32)list.Last()[FieldName]; set.Row = last; } return(list); }
public static void OnError(this IEnumerable <IETLModule> es, Object source, IExtractSetting set, Exception ex) { foreach (var item in es) { item.OnError(source, set, ex); } }
/// <summary>克隆一份设置参数</summary> /// <param name="src"></param> /// <returns></returns> public static IExtractSetting Clone(this IExtractSetting src) { var set = new ExtractSetting(); set.Copy(src); return(set); }
/// <summary>每一轮启动时</summary> /// <param name="set"></param> /// <returns></returns> protected override Boolean Init(IExtractSetting set) { // 如果目标表为空,则使用仅插入 var count = Target.Count; InsertOnly = count == 0; return(base.Init(set)); }
/// <summary>拷贝设置参数</summary> /// <param name="src"></param> /// <param name="set"></param> public static IExtractSetting Copy(this IExtractSetting src, IExtractSetting set) { if (src == null | set == null) { return(src); } src.Start = set.Start; src.End = set.End; src.Row = set.Row; src.Step = set.Step; src.BatchSize = set.BatchSize; //src.Enable = set.Enable; return(src); }
/// <summary>抽取一批数据</summary> /// <param name="set">设置</param> /// <returns></returns> public virtual IList <IEntity> Fetch(IExtractSetting set) { if (Field == null) { throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的时间字段!"); } if (set == null) { throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置"); } // 验证时间段 var start = set.Start; var end = set.End; // 区间无效 if (start >= end) { return(null); } var size = set.BatchSize; // 分批获取数据,如果没有取到,则结束 var list = FetchData(start, end, set.Row, size); // 取到数据,需要滑动窗口 if (list.Count > 0) { set.Row += list.Count; } else { set.Row = 0; } return(list); }
/// <summary>抽取一批数据</summary> /// <param name="set">设置</param> /// <returns></returns> public virtual IList <IEntity> Fetch(IExtractSetting set) { if (Field == null) { throw new ArgumentNullException(nameof(FieldName), "未指定用于顺序抽取数据的时间字段!"); } if (set == null) { throw new ArgumentNullException(nameof(set), "没有设置数据抽取配置"); } // 验证时间段 var start = set.Start; // 有步进且位于第一页时,重新计算最小时间 if (set.Step > 0 && set.Row == 0) { start = GetMinTime(start); } var now = DateTime.Now.AddSeconds(-set.Offset); if (start >= now) { return(null); } // 结束时间,必须是小于当前时间的有效值 var end = set.Step <= 0 ? DateTime.MaxValue : start.AddSeconds(set.Step); // 结束时间有效时,设定末端边界 if (set.End > DateTime.MinValue && set.End < DateTime.MaxValue && end > set.End) { end = set.End; } // 不能超过当前时间 if (set.Step > 0 && end > now) { end = now; } // 区间无效 if (start >= end) { return(null); } ActualStart = start; ActualEnd = end; var size = set.BatchSize; // 如果批大小为零,表示不分批 //if (size <= 0) size = 1000; // 分批获取数据,如果没有取到,则结束 var list = FetchData(start, end, set.Row, size); // 取到数据,需要滑动窗口 if (list.Count > 0) { var last = (DateTime)list.Last()[FieldName]; // 有可能时间字段为空 if (last <= DateTime.MinValue) { last = list.Max(e => (DateTime)e[FieldName]); } // 有可能时间超出区间 if (last < start) { last = list.Max(e => (DateTime)e[FieldName]); } // 满一批,后续还有数据 if (list.Count >= size) { // 最大时间行数 var maxCount = list.Count(e => (DateTime)e[FieldName] == last); // 以最后时间为起点,跳过若干行。注意可能产生连续分页的情况 if (last == set.Start) { set.Row = set.Row + maxCount; } else { set.Row = maxCount; } set.Start = last; } else { set.Start = last.AddSeconds(1); set.Row = 0; } } else if (set.Step > 0) { set.Start = end; set.Row = 0; } return(list); }
/// <summary>实例化</summary> /// <param name="set"></param> public ExtractSetting(IExtractSetting set) { this.Copy(set); }
public static void OnFinished(this IEnumerable <IETLModule> es, IList <IEntity> list, IExtractSetting set, Int32 success, Double fetchCost, Double processCost) { foreach (var item in es) { item.OnFinished(list, set, success, fetchCost, processCost); } }