public void InitProcess(bool isexecute) { foreach (var item in CurrentETLTools.Where(d => d.Enabled)) { if (isexecute == false && item is IDataExecutor) { continue; } item.Init(new List <IFreeDocument>()); } }
private void InsertModule(IColumnProcess tool) { if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count) { CurrentETLTools.Add(tool); } else { CurrentETLTools.Insert(ETLMount, tool); } }
public override FreeDocument DictSerialize(Scenario scenario = Scenario.Database) { var dict = base.DictSerialize(scenario); dict.Add("MaxThreadCount", MaxThreadCount); dict.Add("GenerateMode", GenerateMode); dict.Add("SampleMount", SampleMount); dict.Children = new List <FreeDocument>(); dict.Children.AddRange(CurrentETLTools.Select(d => d.DictSerialize(scenario))); return(dict); }
private bool DropAction(string sender, object attr) { if (sender == "Drop") { var objs = attr as object[]; if (objs.Count() == 2) { var p = objs[0] as SmartGroup; var t = objs[1] as XFrmWorkAttribute; var item = PluginProvider.GetObjectInstance(t.MyType) as IColumnProcess; if (string.IsNullOrEmpty(p.Name) == false) { item.Column = p.Name; } InsertModule(item); ETLMount++; RefreshSamples(); } } if (sender == "Click") { var smart = attr as SmartGroup; if (smart != null) { attr = smart.ColumnInfo; } var window = PropertyGridFactory.GetPropertyWindow(attr); window.Closed += (s, e) => RefreshSamples(); window.ShowDialog(); } if (sender != "Delete") { return(true); } var a = attr as IColumnProcess; if (MessageBox.Show("确实要删除" + a.TypeName + "吗?", "提示信息", MessageBoxButton.OKCancel) != MessageBoxResult.OK) { return(true); } CurrentETLTools.Remove(a); RefreshSamples(); return(true); }
public override void DictDeserialize(IDictionary <string, object> dicts, Scenario scenario = Scenario.Database) { base.DictDeserialize(dicts, scenario); MaxThreadCount = dicts.Set("MaxThreadCount", MaxThreadCount); GenerateMode = dicts.Set("GenerateMode", GenerateMode); SampleMount = dicts.Set("SampleMount", SampleMount); var doc = dicts as FreeDocument; if (doc != null && doc.Children != null) { foreach (var child in doc.Children) { var name = child["Type"].ToString(); var process = PluginProvider.GetObjectByType <IColumnProcess>(name); if (process != null) { process.DictDeserialize(child); CurrentETLTools.Add(process); } } } }
private void InitUI() { var alltools = CurrentETLTools.Take(ETLMount).ToList(); if (generateFloatGrid) { var gridview = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表"); var r = gridview.SetCurrentView(Documents); if (ControlExtended.DockableManager == null) { return; } ControlExtended.DockableManager.AddDockAbleContent( FrmState.Custom, r, "样例数据"); generateFloatGrid = false; } else { var view = new GridView(); Dict.Clear(); var keys = new List <string> { "" }; var docKeys = Documents.GetKeys(null, SampleMount); keys.AddRange(docKeys); var tool = CurrentTool; foreach (var key in keys) { var col = new GridViewColumn { Header = key, DisplayMemberBinding = new Binding($"[{key}]"), Width = 155 }; view.Columns.Add(col); var group = new SmartGroup { Name = key, Value = alltools.Where(d => d.Column == key).ToList() }; group.PropertyChanged += (s, e) => { if (e.PropertyName == "Name") { var last = alltools.LastOrDefault() as IColumnDataTransformer; if (last != null && last.TypeName == "列名修改器" && last.NewColumn == key) { last.NewColumn = group.Name; } else { last = PluginProvider.GetObjectInstance("列名修改器") as IColumnDataTransformer; last.NewColumn = group.Name; last.Column = key; InsertModule(last); ETLMount++; OnPropertyChanged("ETLMount"); RefreshSamples(); } } }; Dict.Add(group ); } if (tool != null) { Dict.Where(d => d.Name == tool.Column).Execute(d => d.GroupType = GroupType.Input); var transformer = tool as IColumnDataTransformer; if (transformer != null) { var newcol = transformer.NewColumn.Split(' '); if (transformer.IsMultiYield) { Dict.Execute(d => d.GroupType = newcol.Contains(d.Name)? GroupType.Input:GroupType.Output); } else { Dict.Where(d => d.Name == transformer.NewColumn).Execute(d => d.GroupType = GroupType.Output);; } } } var nullgroup = Dict.FirstOrDefault(d => string.IsNullOrEmpty(d.Name)); nullgroup?.Value.AddRange( alltools.Where( d => Documents.GetKeys().Contains(d.Column) == false && string.IsNullOrEmpty(d.Column) == false)); if (MainDescription.IsUIForm && IsUISupport) { if (dataView != null) { dataView.View = view; } } } }
public void RefreshSamples(bool canGetDatas = true) { if (SysProcessManager == null) { return; } if (!mudoleHasInit) { return; } if (SysProcessManager.CurrentProcessTasks.Any(d => d.Publisher == this)) { XLogSys.Print.WarnFormat("{0}已经有任务在执行,请在执行完毕后再刷新,或取消该任务", Name); return; } if (dataView == null && MainDescription.IsUIForm && IsUISupport) { var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager; var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this); if (control != null) { if (control.View is IRemoteInvoke) { var invoke = control.View as IRemoteInvoke; invoke.RemoteFunc = DropAction; } dynamic dy = control.View; dataView = dy.DataList; scrollViewer = dy.ScrollViewer; alltoolList = dy.ETLToolList; alltoolList.MouseMove += (s, e) => { if (e.LeftButton == MouseButtonState.Pressed) { var attr = alltoolList.SelectedItem as XFrmWorkAttribute; if (attr == null) { return; } var data = new DataObject(typeof(XFrmWorkAttribute), attr); try { DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move); } catch (Exception ex) { } } }; } } Documents.Clear(); var alltools = CurrentETLTools.Take(ETLMount).ToList(); var hasInit = false; var func = Aggregate(d => d, alltools, false); if (!canGetDatas) { return; } var temptask = TemporaryTask.AddTempTask(Name + "_转换", func(new List <IFreeDocument>()).Take(SampleMount), data => { ControlExtended.UIInvoke(() => { Documents.Add((data)); if (hasInit == false && Documents.Count > 2) { InitUI(); hasInit = true; } }); }, d => { if (!hasInit) { InitUI(); hasInit = true; } } , SampleMount); temptask.Publisher = this; SysProcessManager.CurrentProcessTasks.Add(temptask); }
public void ExecuteDatas() { var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList(); EnumerableFunc func = d => d; var index = 0; if (GenerateMode == GenerateMode.串行模式) { var generator = etls.FirstOrDefault() as IColumnGenerator; if (generator == null) { return; } var realfunc3 = Aggregate(func, etls.Skip(1), true); var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(), d => { realfunc3(new List <IFreeDocument> { d }).ToList(); }, null, generator.GenerateCount() ?? (-1)); SysProcessManager.CurrentProcessTasks.Add(task); } else { var timer = new DispatcherTimer(); TemporaryTask paratask = null; var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF; if (tolistTransformer != null) { index = etls.IndexOf(tolistTransformer); var beforefunc = Aggregate(func, etls.Take(index), true); List <IFreeDocument> taskbuff = new List <IFreeDocument>(); paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>()) , d2 => { //TODO:这种分组方式可能会丢数据!! if (taskbuff.Count < tolistTransformer.GroupMount) { taskbuff.Add(d2); return; } var newtaskbuff = taskbuff.ToList(); taskbuff.Clear(); if (paratask.IsPause == false && SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount) { iswait = true; paratask.IsPause = true; } var countstr = d2.Query(tolistTransformer.MountColumn); var name = d2.Query(tolistTransformer.IDColumn); if (name == null) { name = "清洗任务"; } var rcount = -1; int.TryParse(countstr, out rcount); var afterfunc = Aggregate(func, etls.Skip(index + 1), true); var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { }, null, rcount, false); if (tolistTransformer.DisplayProgress) { ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task)); } task.Start(); }, d => timer.Stop(), -1, false); } else { var generator = etls.FirstOrDefault() as IColumnGenerator; if (generator == null) { return; } var realfunc3 = Aggregate(func, etls.Skip(1), true); paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(), d => { if (paratask.IsPause == false && SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount) { iswait = true; paratask.IsPause = true; } var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> { d }), d2 => { }, null, 1, false); ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task)); task.Start(); }, d => timer.Stop(), generator.GenerateCount() ?? (-1), false); } SysProcessManager.CurrentProcessTasks.Add(paratask); timer.Interval = TimeSpan.FromSeconds(3); timer.Tick += (s, e) => { if (paratask.IsCanceled) { timer.Stop(); return; } if (paratask.IsStart == false) { paratask.Start(); return; } if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount) { paratask.IsPause = false; iswait = false; } }; timer.Start(); } }
public void RefreshSamples(bool canGetDatas = true) { if (shouldUpdate == false) { return; } if (SysProcessManager == null) { return; } if (!mudoleHasInit) { return; } OnPropertyChanged("AllETLMount"); var tasks = SysProcessManager.CurrentProcessTasks.Where(d => d.Publisher == this).ToList(); if (tasks.Any()) { var str = $"{Name}已经有任务在执行,由于调整参数,是否要取消当前任务重新执行?\n 【取消】:【不再提醒】"; if (isErrorRemind == false) { XLogSys.Print.Warn($"{Name}已经有任务在执行,请在任务管理器中取消该任务后再刷新"); return; } if (!MainDescription.IsUIForm) { return; } var result = MessageBox.Show(str, "提示信息", MessageBoxButton.YesNoCancel); if (result == MessageBoxResult.Yes) { foreach (var item in tasks) { item.Remove(); } XLogSys.Print.Warn(str + " 已经取消"); } else if (result == MessageBoxResult.Cancel) { isErrorRemind = false; } else { return; } } if (dataView == null && MainDescription.IsUIForm && IsUISupport) { var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager; var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this); if (control != null) { if (control.View is IRemoteInvoke) { var invoke = control.View as IRemoteInvoke; invoke.RemoteFunc = DropAction; } dynamic dy = control.View; dataView = dy.DataList; scrollViewer = dy.ScrollViewer; alltoolList = dy.ETLToolList; currentToolList = dy.CurrentETLToolList; currentToolList.MouseDoubleClick += (s, e) => { if (e.ChangedButton != MouseButton.Left) { return; } var process = currentToolList.SelectedItem as IColumnProcess; if (process == null) { return; } var oldProp = process.UnsafeDictSerializePlus(); var window = PropertyGridFactory.GetPropertyWindow(process); window.Closed += (s2, e2) => { if ( (oldProp.IsEqual(process.UnsafeDictSerializePlus()) == false && IsAutoRefresh).SafeCheck ("检查模块参数是否修改", LogType.Debug)) { RefreshSamples(); } }; window.ShowDialog(); }; dragMgr = new ListViewDragDropManager <IColumnProcess>(currentToolList); dragMgr.ShowDragAdorner = true; alltoolList.MouseMove += (s, e) => { if (e.LeftButton == MouseButtonState.Pressed) { var attr = alltoolList.SelectedItem as XFrmWorkAttribute; if (attr == null) { return; } var data = new DataObject(typeof(XFrmWorkAttribute), attr); try { DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move); } catch (Exception ex) { } } }; } } if (dataView == null) { return; } Analyzer.Items.Clear(); var alltools = CurrentETLTools.Take(ETLMount).ToList(); var func = alltools.Aggregate(isexecute: false, analyzer: Analyzer); if (!canGetDatas) { return; } SmartGroupCollection.Clear(); Documents.Clear(); shouldUpdate = false; var i = 0; foreach (var currentEtlTool in CurrentETLTools) { (currentEtlTool).ETLIndex = i++; } shouldUpdate = true; if (!MainDescription.IsUIForm) { return; } all_columns.Clear(); dataView.Columns.Clear(); AddColumn("", alltools); var temptask = TemporaryTask.AddTempTask(Name + "_转换", func(new List <IFreeDocument>()).Take(SampleMount), data => { ControlExtended.UIInvoke(() => { foreach (var key in data.GetKeys().Where(d => all_columns.Contains(d) == false).OrderBy(d => d)) { AddColumn(key, alltools); DeleteColumn(""); all_columns.Add(key); } Documents.Add((data)); InitUI(); }); }, r => { var tool = CurrentTool; var outputCol = new List <string>(); var inputCol = new List <string>(); if (tool != null) { inputCol.Add(tool.Column); var transformer = tool as IColumnDataTransformer; if (transformer != null) { if (transformer is CrawlerTF) { var crawler = transformer as CrawlerTF; outputCol = crawler?.Crawler?.CrawlItems.Select(d => d.Name).ToList(); } else if (transformer is ETLBase) { var etl = transformer as ETLBase; var target = etl.GetModule <SmartETLTool>(etl.ETLSelector.SelectItem); outputCol = target?.Documents.GetKeys().ToList(); inputCol.AddRange(etl.MappingSet.Split(' ').Select(d => d.Split(':')[0])); } else { outputCol = transformer.NewColumn.Split(' ').ToList(); } SmartGroupCollection.Where(d => outputCol != null && outputCol.Contains(d.Name)) .Execute(d => d.GroupType = GroupType.Output); SmartGroupCollection.Where(d => inputCol.Contains(d.Name)) .Execute(d => d.GroupType = GroupType.Input); } } var firstOutCol = outputCol?.FirstOrDefault(); if (firstOutCol != null) { var index = all_columns.IndexOf(firstOutCol); if (index != -1 && ETLMount < AllETLMount) { scrollViewer.ScrollToHorizontalOffset(index * CellWidth); } } var nullgroup = SmartGroupCollection.FirstOrDefault(d => string.IsNullOrEmpty(d.Name)); nullgroup?.Value.AddRange( alltools.Where( d => Documents.GetKeys().Contains(d.Column) == false && string.IsNullOrEmpty(d.Column) == false)); nullgroup?.OnPropertyChanged("Value"); } , SampleMount); temptask.Publisher = this; temptask.IsSelected = true; SysProcessManager.CurrentProcessTasks.Add(temptask); }
private bool DropAction(string sender, object attr) { if (sender == "Drop") { var objs = attr as object[]; if (objs.Count() == 2) { var p = objs[0] as SmartGroup; var t = objs[1] as XFrmWorkAttribute; var item = PluginProvider.GetObjectInstance(t.MyType) as IColumnProcess; if (string.IsNullOrEmpty(p.Name) == false) { item.Column = p.Name; } item.Father = this; shouldUpdate = false; InsertModule(item); shouldUpdate = true; if (NeedConfig(item)) { var window = PropertyGridFactory.GetPropertyWindow(item); window.ShowDialog(); } ETLMount++; } } if (sender == "Click") { var smart = attr as SmartGroup; if (smart != null) { attr = smart.ColumnInfo; } var window = PropertyGridFactory.GetPropertyWindow(attr); var oldProp = attr.UnsafeDictSerializePlus(); window.Closed += (s, e) => { if (oldProp.IsEqual(attr.UnsafeDictSerializePlus()) == false && IsAutoRefresh) { RefreshSamples(); } }; window.ShowDialog(); } if (sender != "Delete") { return(true); } var a = attr as IColumnProcess; if (MessageBox.Show("确实要删除" + a.TypeName + "吗?", "提示信息", MessageBoxButton.OKCancel) != MessageBoxResult.OK) { return(true); } CurrentETLTools.Remove(a); if (IsAutoRefresh) { RefreshSamples(); } return(true); }