private void InitUI()
{
var alltools = CurrentETLTools.Take(ETLMount).ToList();
if (generateFloatGrid)
{
var gridview = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
var r = gridview.SetCurrentView(Documents);
if (ControlExtended.DockableManager == null)
{
return;
}
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, r, "样例数据");
generateFloatGrid = false;
}
else
{
var view = new GridView();
Dict.Clear();
var keys = new List <string> {
""
};
var docKeys = Documents.GetKeys(null, SampleMount);
keys.AddRange(docKeys);
var tool = CurrentTool;
foreach (var key in keys)
{
var col = new GridViewColumn
{
Header = key,
DisplayMemberBinding = new Binding($"[{key}]"),
Width = 155
};
view.Columns.Add(col);
var group = new SmartGroup
{
Name = key,
Value = alltools.Where(d => d.Column == key).ToList()
};
group.PropertyChanged += (s, e) =>
{
if (e.PropertyName == "Name")
{
var last = alltools.LastOrDefault() as IColumnDataTransformer;
if (last != null && last.TypeName == "列名修改器" && last.NewColumn == key)
{
last.NewColumn = group.Name;
}
else
{
last = PluginProvider.GetObjectInstance("列名修改器") as IColumnDataTransformer;
last.NewColumn = group.Name;
last.Column = key;
InsertModule(last);
ETLMount++;
OnPropertyChanged("ETLMount");
RefreshSamples();
}
}
};
Dict.Add(group
);
}
if (tool != null)
{
Dict.Where(d => d.Name == tool.Column).Execute(d => d.GroupType = GroupType.Input);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
var newcol = transformer.NewColumn.Split(' ');
if (transformer.IsMultiYield)
{
Dict.Execute(d => d.GroupType = newcol.Contains(d.Name)? GroupType.Input:GroupType.Output);
}
else
{
Dict.Where(d => d.Name == transformer.NewColumn).Execute(d => d.GroupType = GroupType.Output);;
}
}
}
var nullgroup = Dict.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
if (MainDescription.IsUIForm && IsUISupport)
{
if (dataView != null)
{
dataView.View = view;
}
}
}
}
public void ExecuteDatas()
{
var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList();
EnumerableFunc func = d => d;
var index = 0;
if (GenerateMode == GenerateMode.串行模式)
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(),
d => { realfunc3(new List <IFreeDocument> {
d
}).ToList(); }, null, generator.GenerateCount() ?? (-1));
SysProcessManager.CurrentProcessTasks.Add(task);
}
else
{
var timer = new DispatcherTimer();
TemporaryTask paratask = null;
var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF;
if (tolistTransformer != null)
{
index = etls.IndexOf(tolistTransformer);
var beforefunc = Aggregate(func, etls.Take(index), true);
List <IFreeDocument> taskbuff = new List <IFreeDocument>();
paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>())
,
d2 =>
{ //TODO:这种分组方式可能会丢数据！！
if (taskbuff.Count < tolistTransformer.GroupMount)
{
taskbuff.Add(d2);
return;
}
var newtaskbuff = taskbuff.ToList();
taskbuff.Clear();
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var countstr = d2.Query(tolistTransformer.MountColumn);
var name = d2.Query(tolistTransformer.IDColumn);
if (name == null)
{
name = "清洗任务";
}
var rcount = -1;
int.TryParse(countstr, out rcount);
var afterfunc = Aggregate(func, etls.Skip(index + 1), true);
var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { },
null, rcount, false);
if (tolistTransformer.DisplayProgress)
{
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
}
task.Start();
}, d => timer.Stop(), -1, false);
}
else
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(),
d =>
{
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> {
d
}),
d2 => { },
null, 1, false);
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
task.Start();
}, d => timer.Stop(), generator.GenerateCount() ?? (-1), false);
}
SysProcessManager.CurrentProcessTasks.Add(paratask);
timer.Interval = TimeSpan.FromSeconds(3);
timer.Tick += (s, e) =>
{
if (paratask.IsCanceled)
{
timer.Stop();
return;
}
if (paratask.IsStart == false)
{
paratask.Start();
return;
}
if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount)
{
paratask.IsPause = false;
iswait = false;
}
};
timer.Start();
}
}
public void RefreshSamples(bool canGetDatas = true)
{
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
if (SysProcessManager.CurrentProcessTasks.Any(d => d.Publisher == this))
{
XLogSys.Print.WarnFormat("{0}已经有任务在执行，请在执行完毕后再刷新，或取消该任务", Name);
return;
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
Documents.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var hasInit = false;
var func = Aggregate(d => d, alltools, false);
if (!canGetDatas)
{
return;
}
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
Documents.Add((data));
if (hasInit == false && Documents.Count > 2)
{
InitUI();
hasInit = true;
}
});
}, d =>
{
if (!hasInit)
{
InitUI();
hasInit = true;
}
}
, SampleMount);
temptask.Publisher = this;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}
public void RefreshSamples(bool canGetDatas = true)
{
if (shouldUpdate == false)
{
return;
}
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
OnPropertyChanged("AllETLMount");
var tasks = SysProcessManager.CurrentProcessTasks.Where(d => d.Publisher == this).ToList();
if (tasks.Any())
{
var str = $"{Name}已经有任务在执行，由于调整参数，是否要取消当前任务重新执行？\n 【取消】:【不再提醒】";
if (isErrorRemind == false)
{
XLogSys.Print.Warn($"{Name}已经有任务在执行，请在任务管理器中取消该任务后再刷新");
return;
}
if (!MainDescription.IsUIForm)
{
return;
}
var result =
MessageBox.Show(str, "提示信息", MessageBoxButton.YesNoCancel);
if (result == MessageBoxResult.Yes)
{
foreach (var item in tasks)
{
item.Remove();
}
XLogSys.Print.Warn(str + " 已经取消");
}
else if (result == MessageBoxResult.Cancel)
{
isErrorRemind = false;
}
else
{
return;
}
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
currentToolList = dy.CurrentETLToolList;
currentToolList.MouseDoubleClick += (s, e) =>
{
if (e.ChangedButton != MouseButton.Left)
{
return;
}
var process = currentToolList.SelectedItem as IColumnProcess;
if (process == null)
{
return;
}
var oldProp = process.UnsafeDictSerializePlus();
var window = PropertyGridFactory.GetPropertyWindow(process);
window.Closed += (s2, e2) =>
{
if (
(oldProp.IsEqual(process.UnsafeDictSerializePlus()) == false && IsAutoRefresh).SafeCheck
("检查模块参数是否修改", LogType.Debug))
{
RefreshSamples();
}
};
window.ShowDialog();
};
dragMgr = new ListViewDragDropManager <IColumnProcess>(currentToolList);
dragMgr.ShowDragAdorner = true;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
if (dataView == null)
{
return;
}
Analyzer.Items.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var func = alltools.Aggregate(isexecute: false, analyzer: Analyzer);
if (!canGetDatas)
{
return;
}
SmartGroupCollection.Clear();
Documents.Clear();
shouldUpdate = false;
var i = 0;
foreach (var currentEtlTool in CurrentETLTools)
{
(currentEtlTool).ETLIndex = i++;
}
shouldUpdate = true;
if (!MainDescription.IsUIForm)
{
return;
}
all_columns.Clear();
dataView.Columns.Clear();
AddColumn("", alltools);
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
foreach (var key in data.GetKeys().Where(d => all_columns.Contains(d) == false).OrderBy(d => d))
{
AddColumn(key, alltools);
DeleteColumn("");
all_columns.Add(key);
}
Documents.Add((data));
InitUI();
});
}, r =>
{
var tool = CurrentTool;
var outputCol = new List <string>();
var inputCol = new List <string>();
if (tool != null)
{
inputCol.Add(tool.Column);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
if (transformer is CrawlerTF)
{
var crawler = transformer as CrawlerTF;
outputCol = crawler?.Crawler?.CrawlItems.Select(d => d.Name).ToList();
}
else if (transformer is ETLBase)
{
var etl = transformer as ETLBase;
var target = etl.GetModule <SmartETLTool>(etl.ETLSelector.SelectItem);
outputCol = target?.Documents.GetKeys().ToList();
inputCol.AddRange(etl.MappingSet.Split(' ').Select(d => d.Split(':')[0]));
}
else
{
outputCol = transformer.NewColumn.Split(' ').ToList();
}
SmartGroupCollection.Where(d => outputCol != null && outputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Output);
SmartGroupCollection.Where(d => inputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Input);
}
}
var firstOutCol = outputCol?.FirstOrDefault();
if (firstOutCol != null)
{
var index = all_columns.IndexOf(firstOutCol);
if (index != -1 && ETLMount < AllETLMount)
{
scrollViewer.ScrollToHorizontalOffset(index * CellWidth);
}
}
var nullgroup = SmartGroupCollection.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
nullgroup?.OnPropertyChanged("Value");
}
, SampleMount);
temptask.Publisher = this;
temptask.IsSelected = true;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}