public static T GetTask <T>(this IColumnProcess process, string name) where T : class, IDataProcess
{
var moduleName = (typeof(T) == typeof(SmartETLTool)) ? GlobalHelper.Get("smartetl_name") : GlobalHelper.Get("smartcrawler_name");
if (String.IsNullOrEmpty(name))
{
return(null);
}
var process_name = process?.TypeName;
if (process_name != null && String.IsNullOrEmpty(name))
{
XLogSys.Print.Error(String.Format(GlobalHelper.Get("key_203"), process_name));
return(default(T));
}
var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager;
var module = processManager.GetTask <T>(name);
if (module == null)
{
XLogSys.Print.Error(String.Format(GlobalHelper.Get("not_find_module"), name, moduleName, process_name));
return(null);
}
return(module);
}
public void AddErrorLog(IFreeDocument item, Exception ex, IColumnProcess process)
{
if (string.IsNullOrEmpty(errorLogName))
{
return;
}
var param = item.Clone() as FreeDocument;
param["__SysObjectID"] = process.ObjectID;
param["__SysETL"] = (process as ToolBase)?.Father.Name;
param["__SysERROR"] = ex.Message;
param["__SysTime"] = DateTime.Now.ToString();
ControlExtended.UIInvoke(() =>
{
if (ConfigFile.GetConfig <DataMiningConfig>().IsAddErrorCollection)
{
if (errorCollection == null)
{
errorCollection = new DataCollection()
{
Name = errorLogName
};
DataManager.AddDataCollection(errorCollection);
}
errorCollection?.ComputeData.Add(param);
errorCollection?.OnPropertyChanged("Count");
}
else
{
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), process.Column, process.TypeName, ex));
}
});
}
public static string GenerateRemark(this IColumnProcess etl, bool addNew, IProcessManager manager)
{
var list = new List <string>();
if (addNew)
{
list.Add(GlobalHelper.FormatArgs("drag_desc", etl.TypeName, etl.Column));
}
var refs = etl.GetReference(manager).ToList();
if (refs.Any())
{
list.Add(GlobalHelper.FormatArgs("set_before",
",".Join(refs.Where(d => d != null).Select(d => string.Format("{0}:{1}", d.TypeName, d.Name)))));
}
list.Add(GenerateItemRemark(etl, new List <string>()
{
"ObjectID", "Enabled", "Column", "Remark", "Group", "Type", "OneOutput", "IsMultiYield", "IsDebugFilter"
}));
var attribute = AttributeHelper.GetCustomAttribute(etl.GetType());
if (attribute != null)
{
var tooldesc = GlobalHelper.Get(attribute.Description);
list.Add(GlobalHelper.Get(tooldesc.Split('\n').FirstOrDefault(d => string.IsNullOrWhiteSpace(d) == false)));
}
if (!string.IsNullOrWhiteSpace(etl.Remark))
{
list.Add(GlobalHelper.RandomFormatArgs("reason_desc", etl.Remark));
}
return(",".Join(list.Where(d => string.IsNullOrWhiteSpace(d) == false)));
}
public static T GetModule <T>(this IColumnProcess process, string name) where T : class
{
var moduleName = (typeof(T) == typeof(SmartETLTool)) ? "数据清洗" : "网页采集器";
if (string.IsNullOrEmpty(name))
{
XLogSys.Print.Error($"您没有填写“{process.TypeName}”的对应参数。");
return(default(T));
}
var processManager = MainDescription.MainFrm.PluginDictionary[_static_name] as IProcessManager;
var module =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T;
if (module != null)
{
return(module);
}
var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name);
if (task == null)
{
XLogSys.Print.Error($"没有找到名称为'{name}'的{moduleName}，请检查“{process.TypeName}”是否填写错误");
throw new NullReferenceException($"can't find a ETL Module named {name}");
}
ControlExtended.UIInvoke(() => { task.Load(false); });
module =
processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T;
return(module);
}
public AnalyzeItem Set(IColumnProcess process)
{
var item = new AnalyzeItem();
item.Process = process;
Items.Add(item);
return(item);
}
private void InsertModule(IColumnProcess tool)
{
if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count)
{
CurrentETLTools.Add(tool);
}
else
{
CurrentETLTools.Insert(ETLMount, tool);
}
}
public static Func <List <string> > GetAllETLNames(this IColumnProcess process)
{
var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager;
return(() => processManager.CurrentProcessCollections.Where(d => d is SmartETLTool)
.Select(d => d.Name)
.Concat(
processManager.CurrentProject.Tasks.Where(d => d.TaskType == "SmartETLTool")
.Select(d => d.Name)).
Distinct().ToList());
}
public static IEnumerable <AbstractProcessMethod> GetReference(this IColumnProcess obj, IProcessManager manager)
{
var etlbase = obj as ETLBase;
if (etlbase != null)
{
yield return(manager.GetTask <SmartETLTool>(etlbase.ETLSelector.SelectItem));
}
var crawler = obj as ResponseTF;
if (crawler != null)
{
yield return(crawler.GetTask <SmartCrawler>(crawler.CrawlerSelector.SelectItem));
}
}
public static Func <List <string> > GetAllCrawlerNames(this IColumnProcess process)
{
var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager;
return(delegate
{
var item =
processManager.CurrentProcessCollections.Where(d => d is SmartCrawler)
.Select(d => d.Name)
.Concat(
processManager.CurrentProject.Tasks.Where(d => d.TaskType == "SmartCrawler")
.Select(d => d.Name)).
Distinct().ToList();
return item;
});
}
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute)
{
try
{
tool.SetExecute(isexecute);
tool.Init(new List <IFreeDocument>());
}
catch (Exception ex)
{
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = d =>
{
if (ge.IsMultiYield)
{
return(ge.TransformManyData(func1(d)));
}
var r = func1(d);
return(r.Select(d2 => Transform(ge, d2)));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = d => func1(d).Concat(ge.Generate());
break;
case MergeType.Cross:
func = d => func1(d).Cross(ge.Generate);
break;
case MergeType.Merge:
func = d => func1(d).MergeAll(ge.Generate());
break;
case MergeType.Mix:
func = d => func1(d).Mix(ge.Generate());
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = d => ge.Execute(func1(d));
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = d => func1(d).Skip((int)range.Skip).Take((int)range.Take);
}
else
{
var func1 = func;
func = d => func1(d).Where(t.FilteData);
}
}
return(func);
}
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer)
{
AnalyzeItem analyzeItem = null;
analyzeItem = analyzer?.Set(tool);
try
{
tool.SetExecute(isexecute);
if (analyzeItem != null)
{
analyzeItem.HasInit = tool.Init(new List <IFreeDocument>());
}
}
catch (Exception ex)
{
if (analyzeItem != null)
{
analyzeItem.HasInit = false;
}
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = source =>
{
var source2 = func1(source).CountInput(analyzeItem);
if (ge.IsMultiYield)
{
return(ge.TransformManyData(source2).CountOutput(analyzeItem));
}
;
return(source2.Select(input => Transform(ge, input, analyzeItem)).CountOutput(analyzeItem));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem);
break;
case MergeType.Cross:
func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem);
break;
case MergeType.Merge:
func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem);
break;
case MergeType.Mix:
func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem);
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem);
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take).CountOutput(analyzeItem);
}
else
{
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem);
}
}
return(func);
}
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute,
Analyzer analyzer)
{
AnalyzeItem analyzeItem = null;
analyzeItem = analyzer?.Set(tool);
try
{
tool.SetExecute(isexecute);
if (analyzeItem != null)
{
analyzeItem.HasInit = tool.Init(new List <IFreeDocument>());
}
}
catch (Exception ex)
{
if (analyzeItem != null)
{
analyzeItem.HasInit = false;
}
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_209"), tool.Column, tool.TypeName, ex));
return(func);
}
if (!tool.Enabled)
{
return(func);
}
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = source =>
{
var source2 = func1(source).CountInput(analyzeItem);
if (ge.IsMultiYield)
{
return(ge.TransformManyData(source2, analyzeItem).CountOutput(analyzeItem));
}
;
return(source2.Select(input =>
{
var now = DateTime.Now;
var result = Transform(ge, input, analyzeItem);
if (analyzeItem != null)
{
analyzeItem.RunningTime = DateTime.Now - now;
}
return result;
}).CountOutput(analyzeItem));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem);
break;
case MergeType.Cross:
func = source =>
func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem);
break;
case MergeType.Merge:
func = source =>
func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem);
break;
case MergeType.Mix:
func = source =>
func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem);
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem);
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == GlobalHelper.Get("key_210"))
{
dynamic range = t;
var func1 = func;
func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take)
.CountOutput(analyzeItem);
}
else
{
var func1 = func;
switch (t.FilterWorkMode)
{
case FilterWorkMode.PassWhenSuccess:
func =
source =>
func1(source.CountInput(analyzeItem))
.SkipWhile(t.FilteData)
.CountOutput(analyzeItem);
break;
case FilterWorkMode.ByItem:
func =
source =>
func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem);
break;
case FilterWorkMode.StopWhenFail:
func =
source =>
func1(source.CountInput(analyzeItem))
.TakeWhile(t.FilteData)
.CountOutput(analyzeItem);
break;
}
}
}
return(func);
}
private void InsertModule(IColumnProcess tool)
{
if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count)
CurrentETLTools.Add(tool);
else
{
CurrentETLTools.Insert(ETLMount , tool);
}
}
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute)
{
try
{
tool.SetExecute(isexecute);
tool.Init(new List<IFreeDocument>());
}
catch (Exception ex)
{
XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常：{ex},请检查任务参数");
return func;
}
if (!tool.Enabled)
return func;
if (tool is IColumnDataTransformer)
{
var ge = tool as IColumnDataTransformer;
var func1 = func;
func = d =>
{
if (ge.IsMultiYield)
{
return ge.TransformManyData(func1(d));
}
var r = func1(d);
return r.Select(d2 => Transform(ge, d2));
};
}
if (tool is IColumnGenerator)
{
var ge = tool as IColumnGenerator;
var func1 = func;
switch (ge.MergeType)
{
case MergeType.Append:
func = d => func1(d).Concat(ge.Generate());
break;
case MergeType.Cross:
func = d => func1(d).Cross(ge.Generate);
break;
case MergeType.Merge:
func = d => func1(d).MergeAll(ge.Generate());
break;
case MergeType.Mix:
func = d => func1(d).Mix(ge.Generate());
break;
}
}
if (tool is IDataExecutor && isexecute)
{
var ge = tool as IDataExecutor;
var func1 = func;
func = d => ge.Execute(func1(d));
}
else if (tool is IColumnDataFilter)
{
var t = tool as IColumnDataFilter;
if (t.TypeName == "数量范围选择")
{
dynamic range = t;
var func1 = func;
func = d => func1(d).Skip((int) range.Skip).Take((int) range.Take);
}
else
{
var func1 = func;
func = d => func1(d).Where(t.FilteData);
}
}
return func;
}
