public override bool Init(IEnumerable <IFreeDocument> datas)
{
base.Init(datas);
process = GetProcesses();
func = etl.Aggregate(d => d, process, IsExecute);
return(true);
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
base.Init(datas);
var process = GetProcesses().ToList();
func = process.Aggregate(isexecute: true);
return(true);
}
public override bool Init(IEnumerable <IFreeDocument> datas)
{
base.Init(datas);
var process = GetProcesses().ToList();
func = process.Aggregate(isexecute: true, analyzer: this.Father.Analyzer); //TODO: BUG
return(true);
}
public void ExecuteDatas()
{
var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList();
EnumerableFunc func = d => d;
var index = 0;
if (GenerateMode == GenerateMode.串行模式)
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(),
d => { realfunc3(new List <IFreeDocument> {
d
}).ToList(); }, null, generator.GenerateCount() ?? (-1));
SysProcessManager.CurrentProcessTasks.Add(task);
}
else
{
var timer = new DispatcherTimer();
TemporaryTask paratask = null;
var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF;
if (tolistTransformer != null)
{
index = etls.IndexOf(tolistTransformer);
var beforefunc = Aggregate(func, etls.Take(index), true);
List <IFreeDocument> taskbuff = new List <IFreeDocument>();
paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>())
,
d2 =>
{ //TODO:这种分组方式可能会丢数据！！
if (taskbuff.Count < tolistTransformer.GroupMount)
{
taskbuff.Add(d2);
return;
}
var newtaskbuff = taskbuff.ToList();
taskbuff.Clear();
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var countstr = d2.Query(tolistTransformer.MountColumn);
var name = d2.Query(tolistTransformer.IDColumn);
if (name == null)
{
name = "清洗任务";
}
var rcount = -1;
int.TryParse(countstr, out rcount);
var afterfunc = Aggregate(func, etls.Skip(index + 1), true);
var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { },
null, rcount, false);
if (tolistTransformer.DisplayProgress)
{
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
}
task.Start();
}, d => timer.Stop(), -1, false);
}
else
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(),
d =>
{
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> {
d
}),
d2 => { },
null, 1, false);
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
task.Start();
}, d => timer.Stop(), generator.GenerateCount() ?? (-1), false);
}
SysProcessManager.CurrentProcessTasks.Add(paratask);
timer.Interval = TimeSpan.FromSeconds(3);
timer.Tick += (s, e) =>
{
if (paratask.IsCanceled)
{
timer.Stop();
return;
}
if (paratask.IsStart == false)
{
paratask.Start();
return;
}
if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount)
{
paratask.IsPause = false;
iswait = false;
}
};
timer.Start();
}
}
public EnumerableFunc Aggregate(EnumerableFunc func, IEnumerable <IColumnProcess> tools, bool isexecute)
{
return(tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute)));
}
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute)
{
try
{
tool.SetExecute(isexecute);
tool.Init(new List <IFreeDocument>());
}
catch (Exception ex)
{
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = d =>
{
if (ge.IsMultiYield)
{
return(ge.TransformManyData(func1(d)));
}
var r = func1(d);
return(r.Select(d2 => Transform(ge, d2)));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = d => func1(d).Concat(ge.Generate());
break;
case MergeType.Cross:
func = d => func1(d).Cross(ge.Generate);
break;
case MergeType.Merge:
func = d => func1(d).MergeAll(ge.Generate());
break;
case MergeType.Mix:
func = d => func1(d).Mix(ge.Generate());
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = d => ge.Execute(func1(d));
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = d => func1(d).Skip((int)range.Skip).Take((int)range.Take);
}
else
{
var func1 = func;
func = d => func1(d).Where(t.FilteData);
}
}
return(func);
}
public static EnumerableFunc Aggregate(this IEnumerable <IColumnProcess> tools, EnumerableFunc func = null, bool isexecute = false, Analyzer analyzer = null)
{
if (func == null)
{
func = d => d;
}
if (analyzer != null)
{
analyzer.Items.Clear();
}
return(tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute, analyzer)));
}
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer)
{
AnalyzeItem analyzeItem = null;
analyzeItem = analyzer?.Set(tool);
try
{
tool.SetExecute(isexecute);
if (analyzeItem != null)
{
analyzeItem.HasInit = tool.Init(new List <IFreeDocument>());
}
}
catch (Exception ex)
{
if (analyzeItem != null)
{
analyzeItem.HasInit = false;
}
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = source =>
{
var source2 = func1(source).CountInput(analyzeItem);
if (ge.IsMultiYield)
{
return(ge.TransformManyData(source2).CountOutput(analyzeItem));
}
;
return(source2.Select(input => Transform(ge, input, analyzeItem)).CountOutput(analyzeItem));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem);
break;
case MergeType.Cross:
func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem);
break;
case MergeType.Merge:
func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem);
break;
case MergeType.Mix:
func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem);
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem);
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take).CountOutput(analyzeItem);
}
else
{
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem);
}
}
return(func);
}
public static IEnumerable <IFreeDocument> ConcatPlus(this IEnumerable <IFreeDocument> d, EnumerableFunc func1,
IColumnGenerator ge)
{
IFreeDocument last = null;
foreach (var r in func1(d))
{
yield return(r);
last = r;
}
foreach (var r in ge.Generate(last))
{
yield return
(r);
}
}
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute,
Analyzer analyzer)
{
AnalyzeItem analyzeItem = null;
analyzeItem = analyzer?.Set(tool);
try
{
tool.SetExecute(isexecute);
if (analyzeItem != null)
{
analyzeItem.HasInit = tool.Init(new List <IFreeDocument>());
}
}
catch (Exception ex)
{
if (analyzeItem != null)
{
analyzeItem.HasInit = false;
}
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_209"), tool.Column, tool.TypeName, ex));
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = source =>
{
var source2 = func1(source).CountInput(analyzeItem);
if (ge.IsMultiYield)
{
return(ge.TransformManyData(source2, analyzeItem).CountOutput(analyzeItem));
}
;
return(source2.Select(input =>
{
var now = DateTime.Now;
var result = Transform(ge, input, analyzeItem);
if (analyzeItem != null)
{
analyzeItem.RunningTime = DateTime.Now - now;
}
return result;
}).CountOutput(analyzeItem));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem);
break;
case MergeType.Cross:
func = source =>
func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem);
break;
case MergeType.Merge:
func = source =>
func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem);
break;
case MergeType.Mix:
func = source =>
func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem);
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem);
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == GlobalHelper.Get("key_210"))
{
dynamic range = t;
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take)
.CountOutput(analyzeItem);
}
else
{
var func1 = func;
switch (t.FilterWorkMode)
{
case FilterWorkMode.PassWhenSuccess:
func =
source =>
func1(source.CountInput(analyzeItem))
.SkipWhile(t.FilteData)
.CountOutput(analyzeItem);
break;
case FilterWorkMode.ByItem:
func =
source =>
func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem);
break;
case FilterWorkMode.StopWhenFail:
func =
source =>
func1(source.CountInput(analyzeItem))
.TakeWhile(t.FilteData)
.CountOutput(analyzeItem);
break;
}
}
}
return(func);
}
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute)
{
try
{
tool.SetExecute(isexecute);
tool.Init(new List<IFreeDocument>());
}
catch (Exception ex)
{
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return func;
}
if (!tool.Enabled)
return func;
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = d =>
{
if (ge.IsMultiYield)
{
return ge.TransformManyData(func1(d));
}
var r = func1(d);
return r.Select(d2 => Transform(ge, d2));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = d => func1(d).Concat(ge.Generate());
break;
case MergeType.Cross:
func = d => func1(d).Cross(ge.Generate);
break;
case MergeType.Merge:
func = d => func1(d).MergeAll(ge.Generate());
break;
case MergeType.Mix:
func = d => func1(d).Mix(ge.Generate());
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = d => ge.Execute(func1(d));
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = d => func1(d).Skip((int) range.Skip).Take((int) range.Take);
}
else
{
var func1 = func;
func = d => func1(d).Where(t.FilteData);
}
}
return func;
}
public EnumerableFunc Aggregate(EnumerableFunc func, IEnumerable<IColumnProcess> tools, bool isexecute)
{
return tools.Aggregate(func, (current, tool) => FuncAdd(tool, current, isexecute));
}
public override bool Init(IEnumerable<IFreeDocument> datas)
{
base.Init(datas);
process = GetProcesses();
func = etl.Aggregate(d => d, process, IsExecute);
return true;
}
