public void InitProcess(bool isexecute)
{
foreach (var item in CurrentETLTools.Where(d => d.Enabled))
{
if (isexecute == false && item is IDataExecutor)
{
continue;
}
item.Init(new List <IFreeDocument>());
}
}
private void InsertModule(IColumnProcess tool)
{
if (ETLMount < 1 || ETLMount >= CurrentETLTools.Count)
{
CurrentETLTools.Add(tool);
}
else
{
CurrentETLTools.Insert(ETLMount, tool);
}
}
public override FreeDocument DictSerialize(Scenario scenario = Scenario.Database)
{
var dict = base.DictSerialize(scenario);
dict.Add("MaxThreadCount", MaxThreadCount);
dict.Add("GenerateMode", GenerateMode);
dict.Add("SampleMount", SampleMount);
dict.Children = new List <FreeDocument>();
dict.Children.AddRange(CurrentETLTools.Select(d => d.DictSerialize(scenario)));
return(dict);
}
private bool DropAction(string sender, object attr)
{
if (sender == "Drop")
{
var objs = attr as object[];
if (objs.Count() == 2)
{
var p = objs[0] as SmartGroup;
var t = objs[1] as XFrmWorkAttribute;
var item = PluginProvider.GetObjectInstance(t.MyType) as IColumnProcess;
if (string.IsNullOrEmpty(p.Name) == false)
{
item.Column = p.Name;
}
InsertModule(item);
ETLMount++;
RefreshSamples();
}
}
if (sender == "Click")
{
var smart = attr as SmartGroup;
if (smart != null)
{
attr = smart.ColumnInfo;
}
var window = PropertyGridFactory.GetPropertyWindow(attr);
window.Closed += (s, e) => RefreshSamples();
window.ShowDialog();
}
if (sender != "Delete")
{
return(true);
}
var a = attr as IColumnProcess;
if (MessageBox.Show("确实要删除" + a.TypeName + "吗?", "提示信息", MessageBoxButton.OKCancel) !=
MessageBoxResult.OK)
{
return(true);
}
CurrentETLTools.Remove(a);
RefreshSamples();
return(true);
}
public override void DictDeserialize(IDictionary <string, object> dicts, Scenario scenario = Scenario.Database)
{
base.DictDeserialize(dicts, scenario);
MaxThreadCount = dicts.Set("MaxThreadCount", MaxThreadCount);
GenerateMode = dicts.Set("GenerateMode", GenerateMode);
SampleMount = dicts.Set("SampleMount", SampleMount);
var doc = dicts as FreeDocument;
if (doc != null && doc.Children != null)
{
foreach (var child in doc.Children)
{
var name = child["Type"].ToString();
var process = PluginProvider.GetObjectByType <IColumnProcess>(name);
if (process != null)
{
process.DictDeserialize(child);
CurrentETLTools.Add(process);
}
}
}
}
private void InitUI()
{
var alltools = CurrentETLTools.Take(ETLMount).ToList();
if (generateFloatGrid)
{
var gridview = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
var r = gridview.SetCurrentView(Documents);
if (ControlExtended.DockableManager == null)
{
return;
}
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, r, "样例数据");
generateFloatGrid = false;
}
else
{
var view = new GridView();
Dict.Clear();
var keys = new List <string> {
""
};
var docKeys = Documents.GetKeys(null, SampleMount);
keys.AddRange(docKeys);
var tool = CurrentTool;
foreach (var key in keys)
{
var col = new GridViewColumn
{
Header = key,
DisplayMemberBinding = new Binding($"[{key}]"),
Width = 155
};
view.Columns.Add(col);
var group = new SmartGroup
{
Name = key,
Value = alltools.Where(d => d.Column == key).ToList()
};
group.PropertyChanged += (s, e) =>
{
if (e.PropertyName == "Name")
{
var last = alltools.LastOrDefault() as IColumnDataTransformer;
if (last != null && last.TypeName == "列名修改器" && last.NewColumn == key)
{
last.NewColumn = group.Name;
}
else
{
last = PluginProvider.GetObjectInstance("列名修改器") as IColumnDataTransformer;
last.NewColumn = group.Name;
last.Column = key;
InsertModule(last);
ETLMount++;
OnPropertyChanged("ETLMount");
RefreshSamples();
}
}
};
Dict.Add(group
);
}
if (tool != null)
{
Dict.Where(d => d.Name == tool.Column).Execute(d => d.GroupType = GroupType.Input);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
var newcol = transformer.NewColumn.Split(' ');
if (transformer.IsMultiYield)
{
Dict.Execute(d => d.GroupType = newcol.Contains(d.Name)? GroupType.Input:GroupType.Output);
}
else
{
Dict.Where(d => d.Name == transformer.NewColumn).Execute(d => d.GroupType = GroupType.Output);;
}
}
}
var nullgroup = Dict.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
if (MainDescription.IsUIForm && IsUISupport)
{
if (dataView != null)
{
dataView.View = view;
}
}
}
}
public void RefreshSamples(bool canGetDatas = true)
{
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
if (SysProcessManager.CurrentProcessTasks.Any(d => d.Publisher == this))
{
XLogSys.Print.WarnFormat("{0}已经有任务在执行，请在执行完毕后再刷新，或取消该任务", Name);
return;
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
Documents.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var hasInit = false;
var func = Aggregate(d => d, alltools, false);
if (!canGetDatas)
{
return;
}
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
Documents.Add((data));
if (hasInit == false && Documents.Count > 2)
{
InitUI();
hasInit = true;
}
});
}, d =>
{
if (!hasInit)
{
InitUI();
hasInit = true;
}
}
, SampleMount);
temptask.Publisher = this;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}
public void ExecuteDatas()
{
var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList();
EnumerableFunc func = d => d;
var index = 0;
if (GenerateMode == GenerateMode.串行模式)
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(),
d => { realfunc3(new List <IFreeDocument> {
d
}).ToList(); }, null, generator.GenerateCount() ?? (-1));
SysProcessManager.CurrentProcessTasks.Add(task);
}
else
{
var timer = new DispatcherTimer();
TemporaryTask paratask = null;
var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF;
if (tolistTransformer != null)
{
index = etls.IndexOf(tolistTransformer);
var beforefunc = Aggregate(func, etls.Take(index), true);
List <IFreeDocument> taskbuff = new List <IFreeDocument>();
paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>())
,
d2 =>
{ //TODO:这种分组方式可能会丢数据！！
if (taskbuff.Count < tolistTransformer.GroupMount)
{
taskbuff.Add(d2);
return;
}
var newtaskbuff = taskbuff.ToList();
taskbuff.Clear();
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var countstr = d2.Query(tolistTransformer.MountColumn);
var name = d2.Query(tolistTransformer.IDColumn);
if (name == null)
{
name = "清洗任务";
}
var rcount = -1;
int.TryParse(countstr, out rcount);
var afterfunc = Aggregate(func, etls.Skip(index + 1), true);
var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { },
null, rcount, false);
if (tolistTransformer.DisplayProgress)
{
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
}
task.Start();
}, d => timer.Stop(), -1, false);
}
else
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(),
d =>
{
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> {
d
}),
d2 => { },
null, 1, false);
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
task.Start();
}, d => timer.Stop(), generator.GenerateCount() ?? (-1), false);
}
SysProcessManager.CurrentProcessTasks.Add(paratask);
timer.Interval = TimeSpan.FromSeconds(3);
timer.Tick += (s, e) =>
{
if (paratask.IsCanceled)
{
timer.Stop();
return;
}
if (paratask.IsStart == false)
{
paratask.Start();
return;
}
if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount)
{
paratask.IsPause = false;
iswait = false;
}
};
timer.Start();
}
}
public void RefreshSamples(bool canGetDatas = true)
{
if (shouldUpdate == false)
{
return;
}
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
OnPropertyChanged("AllETLMount");
var tasks = SysProcessManager.CurrentProcessTasks.Where(d => d.Publisher == this).ToList();
if (tasks.Any())
{
var str = $"{Name}已经有任务在执行，由于调整参数，是否要取消当前任务重新执行？\n 【取消】:【不再提醒】";
if (isErrorRemind == false)
{
XLogSys.Print.Warn($"{Name}已经有任务在执行，请在任务管理器中取消该任务后再刷新");
return;
}
if (!MainDescription.IsUIForm)
{
return;
}
var result =
MessageBox.Show(str, "提示信息", MessageBoxButton.YesNoCancel);
if (result == MessageBoxResult.Yes)
{
foreach (var item in tasks)
{
item.Remove();
}
XLogSys.Print.Warn(str + " 已经取消");
}
else if (result == MessageBoxResult.Cancel)
{
isErrorRemind = false;
}
else
{
return;
}
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
currentToolList = dy.CurrentETLToolList;
currentToolList.MouseDoubleClick += (s, e) =>
{
if (e.ChangedButton != MouseButton.Left)
{
return;
}
var process = currentToolList.SelectedItem as IColumnProcess;
if (process == null)
{
return;
}
var oldProp = process.UnsafeDictSerializePlus();
var window = PropertyGridFactory.GetPropertyWindow(process);
window.Closed += (s2, e2) =>
{
if (
(oldProp.IsEqual(process.UnsafeDictSerializePlus()) == false && IsAutoRefresh).SafeCheck
("检查模块参数是否修改", LogType.Debug))
{
RefreshSamples();
}
};
window.ShowDialog();
};
dragMgr = new ListViewDragDropManager <IColumnProcess>(currentToolList);
dragMgr.ShowDragAdorner = true;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
if (dataView == null)
{
return;
}
Analyzer.Items.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var func = alltools.Aggregate(isexecute: false, analyzer: Analyzer);
if (!canGetDatas)
{
return;
}
SmartGroupCollection.Clear();
Documents.Clear();
shouldUpdate = false;
var i = 0;
foreach (var currentEtlTool in CurrentETLTools)
{
(currentEtlTool).ETLIndex = i++;
}
shouldUpdate = true;
if (!MainDescription.IsUIForm)
{
return;
}
all_columns.Clear();
dataView.Columns.Clear();
AddColumn("", alltools);
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
foreach (var key in data.GetKeys().Where(d => all_columns.Contains(d) == false).OrderBy(d => d))
{
AddColumn(key, alltools);
DeleteColumn("");
all_columns.Add(key);
}
Documents.Add((data));
InitUI();
});
}, r =>
{
var tool = CurrentTool;
var outputCol = new List <string>();
var inputCol = new List <string>();
if (tool != null)
{
inputCol.Add(tool.Column);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
if (transformer is CrawlerTF)
{
var crawler = transformer as CrawlerTF;
outputCol = crawler?.Crawler?.CrawlItems.Select(d => d.Name).ToList();
}
else if (transformer is ETLBase)
{
var etl = transformer as ETLBase;
var target = etl.GetModule <SmartETLTool>(etl.ETLSelector.SelectItem);
outputCol = target?.Documents.GetKeys().ToList();
inputCol.AddRange(etl.MappingSet.Split(' ').Select(d => d.Split(':')[0]));
}
else
{
outputCol = transformer.NewColumn.Split(' ').ToList();
}
SmartGroupCollection.Where(d => outputCol != null && outputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Output);
SmartGroupCollection.Where(d => inputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Input);
}
}
var firstOutCol = outputCol?.FirstOrDefault();
if (firstOutCol != null)
{
var index = all_columns.IndexOf(firstOutCol);
if (index != -1 && ETLMount < AllETLMount)
{
scrollViewer.ScrollToHorizontalOffset(index * CellWidth);
}
}
var nullgroup = SmartGroupCollection.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
nullgroup?.OnPropertyChanged("Value");
}
, SampleMount);
temptask.Publisher = this;
temptask.IsSelected = true;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}
private bool DropAction(string sender, object attr)
{
if (sender == "Drop")
{
var objs = attr as object[];
if (objs.Count() == 2)
{
var p = objs[0] as SmartGroup;
var t = objs[1] as XFrmWorkAttribute;
var item = PluginProvider.GetObjectInstance(t.MyType) as IColumnProcess;
if (string.IsNullOrEmpty(p.Name) == false)
{
item.Column = p.Name;
}
item.Father = this;
shouldUpdate = false;
InsertModule(item);
shouldUpdate = true;
if (NeedConfig(item))
{
var window = PropertyGridFactory.GetPropertyWindow(item);
window.ShowDialog();
}
ETLMount++;
}
}
if (sender == "Click")
{
var smart = attr as SmartGroup;
if (smart != null)
{
attr = smart.ColumnInfo;
}
var window = PropertyGridFactory.GetPropertyWindow(attr);
var oldProp = attr.UnsafeDictSerializePlus();
window.Closed += (s, e) =>
{
if (oldProp.IsEqual(attr.UnsafeDictSerializePlus()) == false && IsAutoRefresh)
{
RefreshSamples();
}
};
window.ShowDialog();
}
if (sender != "Delete")
{
return(true);
}
var a = attr as IColumnProcess;
if (MessageBox.Show("确实要删除" + a.TypeName + "吗?", "提示信息", MessageBoxButton.OKCancel) !=
MessageBoxResult.OK)
{
return(true);
}
CurrentETLTools.Remove(a);
if (IsAutoRefresh)
{
RefreshSamples();
}
return(true);
}
