public static T GetTask <T>(this IColumnProcess process, string name) where T : class, IDataProcess { var moduleName = (typeof(T) == typeof(SmartETLTool)) ? GlobalHelper.Get("smartetl_name") : GlobalHelper.Get("smartcrawler_name"); if (String.IsNullOrEmpty(name)) { return(null); } var process_name = process?.TypeName; if (process_name != null && String.IsNullOrEmpty(name)) { XLogSys.Print.Error(String.Format(GlobalHelper.Get("key_203"), process_name)); return(default(T)); } var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager; var module = processManager.GetTask <T>(name); if (module == null) { XLogSys.Print.Error(String.Format(GlobalHelper.Get("not_find_module"), name, moduleName, process_name)); return(null); } return(module); }
public void AddErrorLog(IFreeDocument item, Exception ex, IColumnProcess process) { if (string.IsNullOrEmpty(errorLogName)) { return; } var param = item.Clone() as FreeDocument; param["__SysObjectID"] = process.ObjectID; param["__SysETL"] = (process as ToolBase)?.Father.Name; param["__SysERROR"] = ex.Message; param["__SysTime"] = DateTime.Now.ToString(); ControlExtended.UIInvoke(() => { if (ConfigFile.GetConfig <DataMiningConfig>().IsAddErrorCollection) { if (errorCollection == null) { errorCollection = new DataCollection() { Name = errorLogName }; DataManager.AddDataCollection(errorCollection); } errorCollection?.ComputeData.Add(param); errorCollection?.OnPropertyChanged("Count"); } else { XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), process.Column, process.TypeName, ex)); } }); }
public static string GenerateRemark(this IColumnProcess etl, bool addNew, IProcessManager manager) { var list = new List <string>(); if (addNew) { list.Add(GlobalHelper.FormatArgs("drag_desc", etl.TypeName, etl.Column)); } var refs = etl.GetReference(manager).ToList(); if (refs.Any()) { list.Add(GlobalHelper.FormatArgs("set_before", ",".Join(refs.Where(d => d != null).Select(d => string.Format("{0}:{1}", d.TypeName, d.Name))))); } list.Add(GenerateItemRemark(etl, new List <string>() { "ObjectID", "Enabled", "Column", "Remark", "Group", "Type", "OneOutput", "IsMultiYield", "IsDebugFilter" })); var attribute = AttributeHelper.GetCustomAttribute(etl.GetType()); if (attribute != null) { var tooldesc = GlobalHelper.Get(attribute.Description); list.Add(GlobalHelper.Get(tooldesc.Split('\n').FirstOrDefault(d => string.IsNullOrWhiteSpace(d) == false))); } if (!string.IsNullOrWhiteSpace(etl.Remark)) { list.Add(GlobalHelper.RandomFormatArgs("reason_desc", etl.Remark)); } return(",".Join(list.Where(d => string.IsNullOrWhiteSpace(d) == false))); }
public static T GetModule <T>(this IColumnProcess process, string name) where T : class { var moduleName = (typeof(T) == typeof(SmartETLTool)) ? "数据清洗" : "网页采集器"; if (string.IsNullOrEmpty(name)) { XLogSys.Print.Error($"您没有填写“{process.TypeName}”的对应参数。"); return(default(T)); } var processManager = MainDescription.MainFrm.PluginDictionary[_static_name] as IProcessManager; var module = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T; if (module != null) { return(module); } var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name); if (task == null) { XLogSys.Print.Error($"没有找到名称为'{name}'的{moduleName},请检查“{process.TypeName}”是否填写错误"); throw new NullReferenceException($"can't find a ETL Module named {name}"); } ControlExtended.UIInvoke(() => { task.Load(false); }); module = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T; return(module); }
public AnalyzeItem Set(IColumnProcess process) { var item = new AnalyzeItem(); item.Process = process; Items.Add(item); return(item); }
private void InsertModule(IColumnProcess tool) { if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count) { CurrentETLTools.Add(tool); } else { CurrentETLTools.Insert(ETLMount, tool); } }
public static Func <List <string> > GetAllETLNames(this IColumnProcess process) { var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager; return(() => processManager.CurrentProcessCollections.Where(d => d is SmartETLTool) .Select(d => d.Name) .Concat( processManager.CurrentProject.Tasks.Where(d => d.TaskType == "SmartETLTool") .Select(d => d.Name)). Distinct().ToList()); }
public static IEnumerable <AbstractProcessMethod> GetReference(this IColumnProcess obj, IProcessManager manager) { var etlbase = obj as ETLBase; if (etlbase != null) { yield return(manager.GetTask <SmartETLTool>(etlbase.ETLSelector.SelectItem)); } var crawler = obj as ResponseTF; if (crawler != null) { yield return(crawler.GetTask <SmartCrawler>(crawler.CrawlerSelector.SelectItem)); } }
public static Func <List <string> > GetAllCrawlerNames(this IColumnProcess process) { var processManager = MainDescription.MainFrm.PluginDictionary["DataProcessManager"] as IProcessManager; return(delegate { var item = processManager.CurrentProcessCollections.Where(d => d is SmartCrawler) .Select(d => d.Name) .Concat( processManager.CurrentProject.Tasks.Where(d => d.TaskType == "SmartCrawler") .Select(d => d.Name)). Distinct().ToList(); return item; }); }
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute) { try { tool.SetExecute(isexecute); tool.Init(new List <IFreeDocument>()); } catch (Exception ex) { XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = d => { if (ge.IsMultiYield) { return(ge.TransformManyData(func1(d))); } var r = func1(d); return(r.Select(d2 => Transform(ge, d2))); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = d => func1(d).Concat(ge.Generate()); break; case MergeType.Cross: func = d => func1(d).Cross(ge.Generate); break; case MergeType.Merge: func = d => func1(d).MergeAll(ge.Generate()); break; case MergeType.Mix: func = d => func1(d).Mix(ge.Generate()); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = d => ge.Execute(func1(d)); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = d => func1(d).Skip((int)range.Skip).Take((int)range.Take); } else { var func1 = func; func = d => func1(d).Where(t.FilteData); } } return(func); }
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer) { AnalyzeItem analyzeItem = null; analyzeItem = analyzer?.Set(tool); try { tool.SetExecute(isexecute); if (analyzeItem != null) { analyzeItem.HasInit = tool.Init(new List <IFreeDocument>()); } } catch (Exception ex) { if (analyzeItem != null) { analyzeItem.HasInit = false; } XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = source => { var source2 = func1(source).CountInput(analyzeItem); if (ge.IsMultiYield) { return(ge.TransformManyData(source2).CountOutput(analyzeItem)); } ; return(source2.Select(input => Transform(ge, input, analyzeItem)).CountOutput(analyzeItem)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem); break; case MergeType.Cross: func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem); break; case MergeType.Merge: func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem); break; case MergeType.Mix: func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take).CountOutput(analyzeItem); } else { var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem); } } return(func); }
public static EnumerableFunc FuncAdd(this IColumnProcess tool, EnumerableFunc func, bool isexecute, Analyzer analyzer) { AnalyzeItem analyzeItem = null; analyzeItem = analyzer?.Set(tool); try { tool.SetExecute(isexecute); if (analyzeItem != null) { analyzeItem.HasInit = tool.Init(new List <IFreeDocument>()); } } catch (Exception ex) { if (analyzeItem != null) { analyzeItem.HasInit = false; } XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_209"), tool.Column, tool.TypeName, ex)); return(func); } if (!tool.Enabled) { return(func); } if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = source => { var source2 = func1(source).CountInput(analyzeItem); if (ge.IsMultiYield) { return(ge.TransformManyData(source2, analyzeItem).CountOutput(analyzeItem)); } ; return(source2.Select(input => { var now = DateTime.Now; var result = Transform(ge, input, analyzeItem); if (analyzeItem != null) { analyzeItem.RunningTime = DateTime.Now - now; } return result; }).CountOutput(analyzeItem)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = source => source.CountInput(analyzeItem).ConcatPlus(func1, ge).CountOutput(analyzeItem); break; case MergeType.Cross: func = source => func1(source.CountInput(analyzeItem)).Cross(ge.Generate).CountOutput(analyzeItem); break; case MergeType.Merge: func = source => func1(source.CountInput(analyzeItem)).MergeAll(ge.Generate()).CountOutput(analyzeItem); break; case MergeType.Mix: func = source => func1(source.CountInput(analyzeItem)).Mix(ge.Generate()).CountOutput(analyzeItem); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = source => ge.Execute(func1(source.CountInput(analyzeItem))).CountOutput(analyzeItem); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == GlobalHelper.Get("key_210")) { dynamic range = t; var func1 = func; func = source => func1(source.CountInput(analyzeItem)).Skip((int)range.Skip).Take((int)range.Take) .CountOutput(analyzeItem); } else { var func1 = func; switch (t.FilterWorkMode) { case FilterWorkMode.PassWhenSuccess: func = source => func1(source.CountInput(analyzeItem)) .SkipWhile(t.FilteData) .CountOutput(analyzeItem); break; case FilterWorkMode.ByItem: func = source => func1(source.CountInput(analyzeItem)).Where(t.FilteData).CountOutput(analyzeItem); break; case FilterWorkMode.StopWhenFail: func = source => func1(source.CountInput(analyzeItem)) .TakeWhile(t.FilteData) .CountOutput(analyzeItem); break; } } } return(func); }
private void InsertModule(IColumnProcess tool) { if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count) CurrentETLTools.Add(tool); else { CurrentETLTools.Insert(ETLMount , tool); } }
private EnumerableFunc FuncAdd(IColumnProcess tool, EnumerableFunc func, bool isexecute) { try { tool.SetExecute(isexecute); tool.Init(new List<IFreeDocument>()); } catch (Exception ex) { XLogSys.Print.Error($"位于{tool.Column}列的{tool.TypeName}模块在初始化时出现异常:{ex},请检查任务参数"); return func; } if (!tool.Enabled) return func; if (tool is IColumnDataTransformer) { var ge = tool as IColumnDataTransformer; var func1 = func; func = d => { if (ge.IsMultiYield) { return ge.TransformManyData(func1(d)); } var r = func1(d); return r.Select(d2 => Transform(ge, d2)); }; } if (tool is IColumnGenerator) { var ge = tool as IColumnGenerator; var func1 = func; switch (ge.MergeType) { case MergeType.Append: func = d => func1(d).Concat(ge.Generate()); break; case MergeType.Cross: func = d => func1(d).Cross(ge.Generate); break; case MergeType.Merge: func = d => func1(d).MergeAll(ge.Generate()); break; case MergeType.Mix: func = d => func1(d).Mix(ge.Generate()); break; } } if (tool is IDataExecutor && isexecute) { var ge = tool as IDataExecutor; var func1 = func; func = d => ge.Execute(func1(d)); } else if (tool is IColumnDataFilter) { var t = tool as IColumnDataFilter; if (t.TypeName == "数量范围选择") { dynamic range = t; var func1 = func; func = d => func1(d).Skip((int) range.Skip).Take((int) range.Take); } else { var func1 = func; func = d => func1(d).Where(t.FilteData); } } return func; }