public override bool Init(IEnumerable <IFreeDocument> datas) { base.Init(datas); process = GetProcesses(); func = etl.Aggregate(d => d, process, IsExecute); return(true); }
public override bool Init(IEnumerable <IFreeDocument> datas) { base.Init(datas); var process = GetProcesses().ToList(); func = process.Aggregate(isexecute: true); return(true); }
public override bool Init(IEnumerable <IFreeDocument> datas) { base.Init(datas); var process = GetProcesses().ToList(); func = process.Aggregate(isexecute: true, analyzer: this.Father.Analyzer); //TODO: BUG return(true); }
public void ExecuteDatas() { var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList(); EnumerableFunc func = d => d; var index = 0; if (GenerateMode == GenerateMode.串行模式) { var generator = etls.FirstOrDefault() as IColumnGenerator; if (generator == null) { return; } var realfunc3 = Aggregate(func, etls.Skip(1), true); var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(), d => { realfunc3(new List <IFreeDocument> { d }).ToList(); }, null, generator.GenerateCount() ?? (-1)); SysProcessManager.CurrentProcessTasks.Add(task); } else { var timer = new DispatcherTimer(); TemporaryTask paratask = null; var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF; if (tolistTransformer != null) { index = etls.IndexOf(tolistTransformer); var beforefunc = Aggregate(func, etls.Take(index), true); List <IFreeDocument> taskbuff = new List <IFreeDocument>(); paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>()) , d2 => { //TODO:这种分组方式可能会丢数据!! if (taskbuff.Count < tolistTransformer.GroupMount) { taskbuff.Add(d2); return; } var newtaskbuff = taskbuff.ToList(); taskbuff.Clear(); if (paratask.IsPause == false && SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount) { iswait = true; paratask.IsPause = true; } var countstr = d2.Query(tolistTransformer.MountColumn); var name = d2.Query(tolistTransformer.IDColumn); if (name == null) { name = "清洗任务"; } var rcount = -1; int.TryParse(countstr, out rcount); var afterfunc = Aggregate(func, etls.Skip(index + 1), true); var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { }, null, rcount, false); if (tolistTransformer.DisplayProgress) { ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task)); } task.Start(); }, d => timer.Stop(), -1, false); } else { var generator = etls.FirstOrDefault() as IColumnGenerator; if (generator == null) { return; } var realfunc3 = Aggregate(func, etls.Skip(1), true); paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(), d => { if (paratask.IsPause == false && SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount) { iswait = true; paratask.IsPause = true; } var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> { d }), d2 => { }, null, 1, false); ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task)); task.Start(); }, d => timer.Stop(), generator.GenerateCount() ?? (-1), false); } SysProcessManager.CurrentProcessTasks.Add(paratask); timer.Interval = TimeSpan.FromSeconds(3); timer.Tick += (s, e) => { if (paratask.IsCanceled) { timer.Stop(); return; } if (paratask.IsStart == false) { paratask.Start(); return; } if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount) { paratask.IsPause = false; iswait = false; } }; timer.Start(); } }
public EnumerableFunc Aggregate(EnumerableFunc func, IEnumerable <IColumnProcess> tools, bool isexecute) { return(tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute))); }
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute) { try { tool.SetExecute(isexecute); tool.Init(new List <IFreeDocument>()); } catch (Exception ex) { XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = d => { if (ge.IsMultiYield) { return(ge.TransformManyData(func1(d))); } var r = func1(d); return(r.Select(d2 => Transform(ge, d2))); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = d => func1(d).Concat(ge.Generate()); break; case MergeType.Cross: func = d => func1(d).Cross(ge.Generate); break; case MergeType.Merge: func = d => func1(d).MergeAll(ge.Generate()); break; case MergeType.Mix: func = d => func1(d).Mix(ge.Generate()); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = d => ge.Execute(func1(d)); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = d => func1(d).Skip((int)range.Skip).Take((int)range.Take); } else { var func1 = func; func = d => func1(d).Where(t.FilteData); } } return(func); }
public static EnumerableFunc Aggregate(this IEnumerable <IColumnProcess> tools, EnumerableFunc func = null, bool isexecute = false, Analyzer analyzer = null) { if (func == null) { func = d => d; } if (analyzer != null) { analyzer.Items.Clear(); } return(tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute, analyzer))); }
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer) { AnalyzeItem analyzeItem = null; analyzeItem = analyzer?.Set(tool); try { tool.SetExecute(isexecute); if (analyzeItem != null) { analyzeItem.HasInit = tool.Init(new List <IFreeDocument>()); } } catch (Exception ex) { if (analyzeItem != null) { analyzeItem.HasInit = false; } XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = source => { var source2 = func1(source).CountInput(analyzeItem); if (ge.IsMultiYield) { return(ge.TransformManyData(source2).CountOutput(analyzeItem)); } ; return(source2.Select(input => Transform(ge, input, analyzeItem)).CountOutput(analyzeItem)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem); break; case MergeType.Cross: func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem); break; case MergeType.Merge: func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem); break; case MergeType.Mix: func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take).CountOutput(analyzeItem); } else { var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem); } } return(func); }
public static IEnumerable <IFreeDocument> ConcatPlus(this IEnumerable <IFreeDocument> d, EnumerableFunc func1, IColumnGenerator ge) { IFreeDocument last = null; foreach (var r in func1(d)) { yield return(r); last = r; } foreach (var r in ge.Generate(last)) { yield return (r); } }
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer) { AnalyzeItem analyzeItem = null; analyzeItem = analyzer?.Set(tool); try { tool.SetExecute(isexecute); if (analyzeItem != null) { analyzeItem.HasInit = tool.Init(new List <IFreeDocument>()); } } catch (Exception ex) { if (analyzeItem != null) { analyzeItem.HasInit = false; } XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_209"), tool.Column, tool.TypeName, ex)); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = source => { var source2 = func1(source).CountInput(analyzeItem); if (ge.IsMultiYield) { return(ge.TransformManyData(source2, analyzeItem).CountOutput(analyzeItem)); } ; return(source2.Select(input => { var now = DateTime.Now; var result = Transform(ge, input, analyzeItem); if (analyzeItem != null) { analyzeItem.RunningTime = DateTime.Now - now; } return result; }).CountOutput(analyzeItem)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem); break; case MergeType.Cross: func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem); break; case MergeType.Merge: func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem); break; case MergeType.Mix: func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == GlobalHelper.Get("key_210")) { dynamic range = t; var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take) .CountOutput(analyzeItem); } else { var func1 = func; switch (t.FilterWorkMode) { case FilterWorkMode.PassWhenSuccess: func = source => func1(source.CountInput(analyzeItem)) .SkipWhile(t.FilteData) .CountOutput(analyzeItem); break; case FilterWorkMode.ByItem: func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem); break; case FilterWorkMode.StopWhenFail: func = source => func1(source.CountInput(analyzeItem)) .TakeWhile(t.FilteData) .CountOutput(analyzeItem); break; } } } return(func); }
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute) { try { tool.SetExecute(isexecute); tool.Init(new List<IFreeDocument>()); } catch (Exception ex) { XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return func; } if (!tool.Enabled) return func; if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = d => { if (ge.IsMultiYield) { return ge.TransformManyData(func1(d)); } var r = func1(d); return r.Select(d2 => Transform(ge, d2)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = d => func1(d).Concat(ge.Generate()); break; case MergeType.Cross: func = d => func1(d).Cross(ge.Generate); break; case MergeType.Merge: func = d => func1(d).MergeAll(ge.Generate()); break; case MergeType.Mix: func = d => func1(d).Mix(ge.Generate()); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = d => ge.Execute(func1(d)); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = d => func1(d).Skip((int) range.Skip).Take((int) range.Take); } else { var func1 = func; func = d => func1(d).Where(t.FilteData); } } return func; }
public EnumerableFunc Aggregate(EnumerableFunc func, IEnumerable<IColumnProcess> tools, bool isexecute) { return tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute)); }
public override bool Init(IEnumerable<IFreeDocument> datas) { base.Init(datas); process = GetProcesses(); func = etl.Aggregate(d => d, process, IsExecute); return true; }