public IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
var doc = MappingDocument(document);
if (AddTask)
{
var name = doc[Column];
ControlExtended.UIInvoke(() =>
{
var task = TemporaryTask.AddTempTask("ETL" + name, func(new List <IFreeDocument> {
doc
}),
d => d.ToList());
processManager.CurrentProcessTasks.Add(task);
});
}
else
{
var r = func(new List <IFreeDocument> {
doc
}).ToList();
}
yield return(document);
}
}
public void FeelLucky()
{
isBusy = true;
var crawTargets = new List <XPathAnalyzer.CrawTarget>();
var task = TemporaryTask.AddTempTask("网页结构计算中",
HtmlDoc.SearchPropertiesSmart(CrawlItems, RootXPath, IsAttribute), crawTarget =>
{
crawTargets.Add(crawTarget);
var datas = HtmlDoc.GetDataFromXPath(crawTarget.CrawItems, IsMultiData, crawTarget.RootXPath);
crawTarget.Datas = datas;
}, d =>
{
isBusy = false;
if (crawTargets.Count == 0)
{
CrawTarget = null;
XLogSys.Print.Warn("没有检查到任何可选的列表页面");
return;
}
var luckModel = new FeelLuckyModel(crawTargets, HtmlDoc);
var view = PluginProvider.GetObjectInstance <ICustomView>("手气不错面板") as UserControl;
view.DataContext = luckModel;
var name = "手气不错";
var window = new Window {
Title = name
};
window.WindowState = WindowState.Maximized;
window.Content = view;
luckModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
var crawTarget = luckModel.CurrentTarget;
RootXPath = crawTarget.RootXPath;
CrawlItems.Clear();
CrawlItems.AddRange(crawTarget.CrawItems.Where(r => r.IsEnabled));
}
});
SysProcessManager.CurrentProcessTasks.Add(task);
}
public IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents)
{
foreach (var document in documents)
{
IFreeDocument doc = null;
if (string.IsNullOrEmpty(NewColumn))
{
doc = document.Clone();
}
else
{
doc = new FreeDocument();
doc.MergeQuery(document, NewColumn + " " + Column);
}
if (AddTask)
{
var name = doc[Column];
ControlExtended.UIInvoke(() =>
{
var task = TemporaryTask.AddTempTask("ETL" + name, func(new List <IFreeDocument> {
doc
}),
d => d.ToList());
processManager.CurrentProcessTasks.Add(task);
});
}
else
{
var r = func(new List <IFreeDocument> {
doc
}).ToList();
}
yield return(document);
}
}
public void RefreshSamples(bool canGetDatas = true)
{
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
if (SysProcessManager.CurrentProcessTasks.Any(d => d.Publisher == this))
{
XLogSys.Print.WarnFormat("{0}已经有任务在执行，请在执行完毕后再刷新，或取消该任务", Name);
return;
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
Documents.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var hasInit = false;
var func = Aggregate(d => d, alltools, false);
if (!canGetDatas)
{
return;
}
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
Documents.Add((data));
if (hasInit == false && Documents.Count > 2)
{
InitUI();
hasInit = true;
}
});
}, d =>
{
if (!hasInit)
{
InitUI();
hasInit = true;
}
}
, SampleMount);
temptask.Publisher = this;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}
public void ExecuteDatas()
{
var etls = CurrentETLTools.Take(ETLMount).Where(d => d.Enabled).ToList();
EnumerableFunc func = d => d;
var index = 0;
if (GenerateMode == GenerateMode.串行模式)
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
var task = TemporaryTask.AddTempTask(Name + "串行任务", generator.Generate(),
d => { realfunc3(new List <IFreeDocument> {
d
}).ToList(); }, null, generator.GenerateCount() ?? (-1));
SysProcessManager.CurrentProcessTasks.Add(task);
}
else
{
var timer = new DispatcherTimer();
TemporaryTask paratask = null;
var tolistTransformer = etls.FirstOrDefault(d => d.TypeName == "启动并行") as ToListTF;
if (tolistTransformer != null)
{
index = etls.IndexOf(tolistTransformer);
var beforefunc = Aggregate(func, etls.Take(index), true);
List <IFreeDocument> taskbuff = new List <IFreeDocument>();
paratask = TemporaryTask.AddTempTask("清洗任务并行化", beforefunc(new List <IFreeDocument>())
,
d2 =>
{ //TODO:这种分组方式可能会丢数据！！
if (taskbuff.Count < tolistTransformer.GroupMount)
{
taskbuff.Add(d2);
return;
}
var newtaskbuff = taskbuff.ToList();
taskbuff.Clear();
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var countstr = d2.Query(tolistTransformer.MountColumn);
var name = d2.Query(tolistTransformer.IDColumn);
if (name == null)
{
name = "清洗任务";
}
var rcount = -1;
int.TryParse(countstr, out rcount);
var afterfunc = Aggregate(func, etls.Skip(index + 1), true);
var task = TemporaryTask.AddTempTask(name, afterfunc(newtaskbuff), d => { },
null, rcount, false);
if (tolistTransformer.DisplayProgress)
{
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
}
task.Start();
}, d => timer.Stop(), -1, false);
}
else
{
var generator = etls.FirstOrDefault() as IColumnGenerator;
if (generator == null)
{
return;
}
var realfunc3 = Aggregate(func, etls.Skip(1), true);
paratask = TemporaryTask.AddTempTask("并行清洗任务", generator.Generate(),
d =>
{
if (paratask.IsPause == false &&
SysProcessManager.CurrentProcessTasks.Count > MaxThreadCount)
{
iswait = true;
paratask.IsPause = true;
}
var task = TemporaryTask.AddTempTask("子任务", realfunc3(new List <IFreeDocument> {
d
}),
d2 => { },
null, 1, false);
ControlExtended.UIInvoke(() => SysProcessManager.CurrentProcessTasks.Add(task));
task.Start();
}, d => timer.Stop(), generator.GenerateCount() ?? (-1), false);
}
SysProcessManager.CurrentProcessTasks.Add(paratask);
timer.Interval = TimeSpan.FromSeconds(3);
timer.Tick += (s, e) =>
{
if (paratask.IsCanceled)
{
timer.Stop();
return;
}
if (paratask.IsStart == false)
{
paratask.Start();
return;
}
if (iswait && SysProcessManager.CurrentProcessTasks.Count < MaxThreadCount)
{
paratask.IsPause = false;
iswait = false;
}
};
timer.Start();
}
}
public void FeelLucky()
{
if (string.IsNullOrEmpty(this.URLHTML))
{
this.VisitUrlAsync();
}
isBusy = true;
var crawTargets = new List <XPathAnalyzer.CrawTarget>();
ICollection <CrawlItem> existItems = CrawlItems;
if (IsMultiData == ScriptWorkMode.One)
{
existItems = new List <CrawlItem> {
new CrawlItem {
Name = "temp", XPath = SelectXPath
}
}
}
;
var task = TemporaryTask.AddTempTask("网页结构计算中",
HtmlDoc.DocumentNode.SearchPropertiesSmart(IsMultiData, existItems, RootXPath, RootFormat, IsAttribute),
crawTarget =>
{
crawTargets.Add(crawTarget);
//var datas =
// HtmlDoc.DocumentNode.GetDataFromXPath(crawTarget.CrawItems, IsMultiData, crawTarget.RootXPath,
// RootFormat).ToList();
//crawTarget.Datas = datas;
}, d =>
{
isBusy = false;
if (crawTargets.Count == 0)
{
CrawTarget = null;
XLogSys.Print.Warn("没有检查到任何可选的列表页面");
return;
}
var luckModel = new FeelLuckyModel(crawTargets, HtmlDoc, IsMultiData);
var view = PluginProvider.GetObjectInstance <ICustomView>("手气不错面板") as UserControl;
view.DataContext = luckModel;
var name = "手气不错";
var window = new Window {
Title = name
};
window.WindowState = WindowState.Maximized;
window.Content = view;
luckModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
var crawTarget = luckModel.CurrentTarget;
if (string.IsNullOrEmpty(RootXPath))
{
RootFormat = SelectorFormat.XPath;
}
RootXPath = crawTarget.RootXPath;
CrawlItems.AddRange(crawTarget.CrawItems.Where(r => r.IsEnabled && CrawlItems.FirstOrDefault(d2 => d2.XPath == r.XPath) == null));
}
});
SysProcessManager.CurrentProcessTasks.Add(task);
}
public void RefreshSamples(bool canGetDatas = true)
{
if (shouldUpdate == false)
{
return;
}
if (SysProcessManager == null)
{
return;
}
if (!mudoleHasInit)
{
return;
}
OnPropertyChanged("AllETLMount");
var tasks = SysProcessManager.CurrentProcessTasks.Where(d => d.Publisher == this).ToList();
if (tasks.Any())
{
var str = $"{Name}已经有任务在执行，由于调整参数，是否要取消当前任务重新执行？\n 【取消】:【不再提醒】";
if (isErrorRemind == false)
{
XLogSys.Print.Warn($"{Name}已经有任务在执行，请在任务管理器中取消该任务后再刷新");
return;
}
if (!MainDescription.IsUIForm)
{
return;
}
var result =
MessageBox.Show(str, "提示信息", MessageBoxButton.YesNoCancel);
if (result == MessageBoxResult.Yes)
{
foreach (var item in tasks)
{
item.Remove();
}
XLogSys.Print.Warn(str + " 已经取消");
}
else if (result == MessageBoxResult.Cancel)
{
isErrorRemind = false;
}
else
{
return;
}
}
if (dataView == null && MainDescription.IsUIForm && IsUISupport)
{
var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager;
var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this);
if (control != null)
{
if (control.View is IRemoteInvoke)
{
var invoke = control.View as IRemoteInvoke;
invoke.RemoteFunc = DropAction;
}
dynamic dy = control.View;
dataView = dy.DataList;
scrollViewer = dy.ScrollViewer;
alltoolList = dy.ETLToolList;
currentToolList = dy.CurrentETLToolList;
currentToolList.MouseDoubleClick += (s, e) =>
{
if (e.ChangedButton != MouseButton.Left)
{
return;
}
var process = currentToolList.SelectedItem as IColumnProcess;
if (process == null)
{
return;
}
var oldProp = process.UnsafeDictSerializePlus();
var window = PropertyGridFactory.GetPropertyWindow(process);
window.Closed += (s2, e2) =>
{
if (
(oldProp.IsEqual(process.UnsafeDictSerializePlus()) == false && IsAutoRefresh).SafeCheck
("检查模块参数是否修改", LogType.Debug))
{
RefreshSamples();
}
};
window.ShowDialog();
};
dragMgr = new ListViewDragDropManager <IColumnProcess>(currentToolList);
dragMgr.ShowDragAdorner = true;
alltoolList.MouseMove += (s, e) =>
{
if (e.LeftButton == MouseButtonState.Pressed)
{
var attr = alltoolList.SelectedItem as XFrmWorkAttribute;
if (attr == null)
{
return;
}
var data = new DataObject(typeof(XFrmWorkAttribute), attr);
try
{
DragDrop.DoDragDrop(control.View as UserControl, data, DragDropEffects.Move);
}
catch (Exception ex)
{
}
}
};
}
}
if (dataView == null)
{
return;
}
Analyzer.Items.Clear();
var alltools = CurrentETLTools.Take(ETLMount).ToList();
var func = alltools.Aggregate(isexecute: false, analyzer: Analyzer);
if (!canGetDatas)
{
return;
}
SmartGroupCollection.Clear();
Documents.Clear();
shouldUpdate = false;
var i = 0;
foreach (var currentEtlTool in CurrentETLTools)
{
(currentEtlTool).ETLIndex = i++;
}
shouldUpdate = true;
if (!MainDescription.IsUIForm)
{
return;
}
all_columns.Clear();
dataView.Columns.Clear();
AddColumn("", alltools);
var temptask = TemporaryTask.AddTempTask(Name + "_转换",
func(new List <IFreeDocument>()).Take(SampleMount),
data =>
{
ControlExtended.UIInvoke(() =>
{
foreach (var key in data.GetKeys().Where(d => all_columns.Contains(d) == false).OrderBy(d => d))
{
AddColumn(key, alltools);
DeleteColumn("");
all_columns.Add(key);
}
Documents.Add((data));
InitUI();
});
}, r =>
{
var tool = CurrentTool;
var outputCol = new List <string>();
var inputCol = new List <string>();
if (tool != null)
{
inputCol.Add(tool.Column);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
if (transformer is CrawlerTF)
{
var crawler = transformer as CrawlerTF;
outputCol = crawler?.Crawler?.CrawlItems.Select(d => d.Name).ToList();
}
else if (transformer is ETLBase)
{
var etl = transformer as ETLBase;
var target = etl.GetModule <SmartETLTool>(etl.ETLSelector.SelectItem);
outputCol = target?.Documents.GetKeys().ToList();
inputCol.AddRange(etl.MappingSet.Split(' ').Select(d => d.Split(':')[0]));
}
else
{
outputCol = transformer.NewColumn.Split(' ').ToList();
}
SmartGroupCollection.Where(d => outputCol != null && outputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Output);
SmartGroupCollection.Where(d => inputCol.Contains(d.Name))
.Execute(d => d.GroupType = GroupType.Input);
}
}
var firstOutCol = outputCol?.FirstOrDefault();
if (firstOutCol != null)
{
var index = all_columns.IndexOf(firstOutCol);
if (index != -1 && ETLMount < AllETLMount)
{
scrollViewer.ScrollToHorizontalOffset(index * CellWidth);
}
}
var nullgroup = SmartGroupCollection.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
nullgroup?.OnPropertyChanged("Value");
}
, SampleMount);
temptask.Publisher = this;
temptask.IsSelected = true;
SysProcessManager.CurrentProcessTasks.Add(temptask);
}