public DataCollection ReadFile(string fileName, string format = null)
{
IFileConnector exporter = null;
if (format != null)
{
exporter = PluginProvider.GetObjectInstance <IFileConnector>(format);
}
else
{
exporter = FileConnector.SmartGetExport(fileName);
}
if (exporter == null)
{
return(null);
}
exporter.FileName = fileName;
fileName = exporter.FileName;
ControlExtended.SafeInvoke(
() => AddDataCollection(exporter.ReadFile(), Path.GetFileNameWithoutExtension(fileName)), LogType.Important);
return(GetCollection(fileName));
}
/// <summary>
/// 在集合中获取一个实例，若无该实例，将搜索插件，并自动添加之
/// </summary>
/// <param name="name"></param>
/// <param name="isAddToList">是否要加入列表，被托管 </param>
/// <returns></returns>
public static T Get <T>(this ICollection <T> collection, string name, bool?isAddToList = true)
where T : class, IProcess
{
var process = collection.FirstOrDefault(d => name == d.GetType().Name);
if (process != null)
{
return(process);
}
var newProcess =
PluginProvider.GetPluginCollection(typeof(T)).FirstOrDefault(d => d.Name == name);
if (newProcess == null)
{
// throw new Exception(string.Format("要获取的插件{0}无法在插件集合中找到"));
return(null);
}
if (isAddToList == true)
{
collection.Add(newProcess.MyType);
var newone = collection.FirstOrDefault(d => name == d.GetType().Name);
return(newone);
}
var plugin = PluginProvider.GetObjectInstance(newProcess.MyType) as T;
return(plugin);
}
public static T GetConfig <T>(string name = null) where T : class, IConfigFile
{
if (name == null)
{
name = AttributeHelper.GetCustomAttribute(typeof(T)).Name;
}
IConfigFile instance = null;
if (configFiles.TryGetValue(name, out instance))
{
return(instance as T);
}
instance = PluginProvider.GetObjectInstance <IConfigFile>(name) ??
PluginProvider.GetObjectInstance(typeof(T)) as IConfigFile;
try
{
instance.ReadConfig(instance.SavePath);
}
catch (Exception ex)
{
instance.RebuildConfig();
instance.SaveConfig();
}
configFiles.Add(name, instance);
return(instance as T);
}
public static IConfigFile GetConfig(string name = null)
{
XFrmWorkAttribute first = PluginProvider.GetFirstPlugin(typeof(IConfigFile), name);
if (first == null)
{
return(null);
}
IConfigFile instance = null;
if (configFiles.TryGetValue(first.Name, out instance))
{
return(instance);
}
try
{
instance = PluginProvider.GetObjectInstance <IConfigFile>(first.Name);
instance.ReadConfig(instance.SavePath);
}
catch (Exception ex)
{
if (instance == null)
{
throw new Exception("不包含任何有关配置文件的任何信息，请检查");
}
instance.RebuildConfig();
instance.SaveConfig();
}
configFiles.Add(first.Name, instance);
return(instance);
}
private void SetConfig()
{
dynamic view = PluginProvider.GetObjectInstance <ICustomView>(GlobalHelper.Get("xpath_detector")) as UserControl;
var name = GlobalHelper.Get("xpath_detector_desc");
var window = new Window {
Title = name
};
window.Content = view;
var textBox = view.HtmlTextBox;
var xPathDetectorModel = new XPathDetectorModel(htmlResults, this.IsManyData, window, textBox);
xPathDetectorModel.XPath._SelectItem = this.XPath;
view.DataContext = xPathDetectorModel;
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
XPath = xPathDetectorModel.XPath.SelectItem;
OnPropertyChanged("XPath");
OnPropertyChanged("MappingSet");
}
}
public void EditProperty()
{
var crawTargets = new List <XPathAnalyzer.CrawTarget>();
crawTargets.Add(new XPathAnalyzer.CrawTarget(CrawlItems.Select(d => d.Clone()).ToList(), RootXPath,
RootFormat)
{
RootNode = this.HtmlDoc.DocumentNode, WorkMode = IsMultiData
});
var luckModel = new FeelLuckyModel(crawTargets, HtmlDoc, IsMultiData);
luckModel.CanChange = false;
var view = PluginProvider.GetObjectInstance <ICustomView>(GlobalHelper.Get("key_657")) as UserControl;
view.DataContext = luckModel;
var name = GlobalHelper.Get("key_658");
var window = new Window {
Title = name
};
window.WindowState = WindowState.Maximized;
window.Content = view;
luckModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
CrawlItems.Clear();
RootXPath = luckModel.CurrentTarget.RootXPath;
CrawlItems.AddRange(luckModel.CurrentTarget.CrawItems);
}
}
private void SetConfig()
{
Init(null);
var subTaskModel = new SubTaskModel(Father, etl, this, this.Father.Documents.GetKeys().ToArray(), etl.Documents.GetKeys().ToArray());
var view = PluginProvider.GetObjectInstance <ICustomView>(GlobalHelper.Get("key_417")) as UserControl;
view.DataContext = subTaskModel;
var name = GlobalHelper.Get("key_418");
var window = new Window {
Title = name
};
window.Content = view;
subTaskModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
ETLRange = subTaskModel.ETLRange;
MappingSet = subTaskModel.MappingSet;
OnPropertyChanged("ETLRange");
OnPropertyChanged("MappingSet");
}
}
private IEnumerable <IFreeDocument> ReadText(XmlDocument xdoc, Action <int> alreadyGetSize = null)
{
XmlNode xTable = xdoc.DocumentElement;
if (xTable == null)
{
yield break;
}
alreadyGetSize?.Invoke(xTable.ChildNodes.Count);
foreach (XmlNode xnode in xTable)
{
var data = PluginProvider.GetObjectInstance(DataType) as IFreeDocument;
var dict = new FreeDocument();
dict.Name = xnode.Name;
XMLNode2Dict(xnode, dict);
data.DictDeserialize(dict.DictSerialize());
var doc = data as FreeDocument;
if (doc != null)
{
doc.Children = dict.Children;
}
yield return(data);
}
}
private void AddColumn(string key, IEnumerable <IColumnProcess> alltools)
{
if (dataView == null)
{
return;
}
var col = new DataGridTemplateColumn
{
Header = key,
Width = CellWidth
};
var dt = new DataTemplate();
col.CellTemplate = dt;
var fef = new FrameworkElementFactory(typeof(MultiLineTextEditor));
var binding = new Binding();
binding.Path = new PropertyPath(($"[{key}]"));
fef.SetBinding(ContentControl.ContentProperty, binding);
fef.SetBinding(MultiLineTextEditor.TextProperty, binding);
dt.VisualTree = fef;
col.CellTemplate = dt;
dataView.Columns.Add(col);
var group = new SmartGroup
{
Name = key,
Value = new ObservableCollection <IColumnProcess>(alltools.Where(d => d.Column == key))
};
group.PropertyChanged += (s, e) =>
{
if (e.PropertyName == "Name")
{
var last = alltools.LastOrDefault() as IColumnDataTransformer;
if (last != null && last.TypeName == "列名修改器" && last.NewColumn == key)
{
last.NewColumn = group.Name;
}
else
{
last = PluginProvider.GetObjectInstance("列名修改器") as IColumnDataTransformer;
last.NewColumn = group.Name;
last.Column = key;
InsertModule(last);
ETLMount++;
OnPropertyChanged("ETLMount");
if (IsAutoRefresh)
{
RefreshSamples();
}
}
}
};
SmartGroupCollection.Add(group
);
}
static FileConnector()
{
ConnectorDictionary = new Dictionary <XFrmWorkAttribute, string>();
foreach (XFrmWorkAttribute item in PluginProvider.GetPluginCollection(typeof(IFileConnector)))
{
var ins = PluginProvider.GetObjectInstance(item.MyType) as IFileConnector;
ConnectorDictionary.Add(item, ins.ExtentFileName);
}
}
private void EnterAnalyzer()
{
var view = PluginProvider.GetObjectInstance <ICustomView>("调试分析面板") as UserControl;
view.DataContext = Analyzer;
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, view, "调试分析 ");
}
/// <summary>
/// 添加一个新实例
/// </summary>
/// <param name="value">要添加的类型</param>
public static void Add <T>(this ICollection <T> collection, string name) where T : class
{
var item = PluginProvider.GetObjectInstance <T>(name);
if (item == null)
{
return;
}
collection.Add(item);
}
public FileDataTransformer()
{
ConnectorSelector = new ExtendSelector <XFrmWorkAttribute>(PluginProvider.GetPluginCollection(typeof(IFileConnector)));
ConnectorSelector.SelectChanged += (s, e) =>
{
Connector = PluginProvider.GetObjectInstance <IFileConnector>(ConnectorSelector.SelectItem.Name);
OnPropertyChanged("Connector");
};
Enabled = false;
}
public static T Clone <T>(this T source) where T : class, IDictionarySerializable
{
if (source == null)
{
return(default(T));
}
var type = source.GetType();
var newItem = PluginProvider.GetObjectInstance(type) as IDictionarySerializable;
source.DictCopyTo(newItem);
return(newItem as T);
}
//public void AutoVisit()
//{
// if (Documents.Any())
// {
// var item = new HttpItem();
// Documents[0].DictCopyTo(item);
// var res = helper != null && helper.AutoVisit(item);
// XLogSys.Print.Info("成功模拟登录");
// Http.SetValue("Cookie", item.GetValue("Cookie"));
// if (res)
// {
// URL = item.URL;
// }
// }
//}
private void GreatHand()
{
var crawitems = HtmlDoc.SearchPropertiesSmart(CrawlItems, IsAttribute).FirstOrDefault();
if ((crawitems != null).SafeCheck("网页属性获取", LogType.Info) == false)
{
return;
}
var datas = HtmlDoc.GetDataFromXPath(crawitems, IsMultiData);
var propertyNames = new FreeDocument(datas.GetKeys().ToDictionary(d => d, d => (object)d));
datas.Insert(0, propertyNames);
var view = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
var r = view.SetCurrentView(datas);
var name = "手气不错_可修改第一列的属性名称";
var window = new Window {
Title = name
};
window.Content = r;
window.Closing += (s, e) =>
{
if (ControlExtended.UserCheck("是否确认选择当前的数据表") == false)
{
return;
}
foreach (var propertyName in propertyNames)
{
var item = crawitems.FirstOrDefault(d => d.Name == propertyName.Key);
if (item == null)
{
continue;
}
if (propertyName.Value == null)
{
continue;
}
item.Name = propertyName.Value.ToString();
}
CrawlItems.Clear();
CrawlItems.AddRange(crawitems);
};
window.ShowDialog();
}
public override IEnumerable <IFreeDocument> ReadFile(Action <int> alreadyGetSize = null)
{
XSSFWorkbook hssfworkbook;
using (var file = new FileStream(FileName, FileMode.Open, FileAccess.Read))
{
hssfworkbook = new XSSFWorkbook(file);
}
ISheet sheet = hssfworkbook.GetSheetAt(0);
List <string> titles = null;
try
{
titles = sheet.GetRow(0).Cells.Select(d => d.StringCellValue).ToList();
}
catch (Exception ex)
{
throw new Exception("请填写Excel的表头信息");
}
for (int i = 1; i < sheet.LastRowNum; i++)
{
IRow row = sheet.GetRow(i);
var data = PluginProvider.GetObjectInstance(DataType) as IFreeDocument;
var dict = new Dictionary <string, object>();
for (int index = 0; index < titles.Count; index++)
{
string title = titles[index];
dict.Add(title, row.GetCell(index).ToString());
}
var freeDocument = data as IFreeDocument;
if (freeDocument != null)
{
if (i == 1)
{
PropertyNames = titles.ToDictionary(d => d, d => d);
}
}
data.DictDeserialize(dict);
yield return(data);
if (i % 1000 == 0)
{
XLogSys.Print.Info(string.Format(Common.AlreadyImportedX, i, sheet.LastRowNum));
}
}
}
public static IFileConnector SmartGetExport(string fileName)
{
string item = SmartGetExpotStr(fileName);
if (item == null)
{
return(null);
}
var con = PluginProvider.GetObjectInstance <IFileConnector>(item);
con.FileName = fileName;
return(con);
}
private void timeCycle(object sender, EventArgs e)
{
if (NeedSave())
{
dynamic welcomeWindow = PluginProvider.GetObjectInstance <ICustomView>(GlobalHelper.Get("auto_save_tooltip"));
welcomeWindow.ShowDialogAdvance();
if (welcomeWindow.DialogResult == true)
{
SaveCurrentProject(true);
}
}
}
private bool DropAction(string sender, object attr)
{
if (sender == "Drop")
{
var objs = attr as object[];
if (objs.Count() == 2)
{
var p = objs[0] as SmartGroup;
var t = objs[1] as XFrmWorkAttribute;
var item = PluginProvider.GetObjectInstance(t.MyType) as IColumnProcess;
if (string.IsNullOrEmpty(p.Name) == false)
{
item.Column = p.Name;
}
InsertModule(item);
ETLMount++;
RefreshSamples();
}
}
if (sender == "Click")
{
var smart = attr as SmartGroup;
if (smart != null)
{
attr = smart.ColumnInfo;
}
var window = PropertyGridFactory.GetPropertyWindow(attr);
window.Closed += (s, e) => RefreshSamples();
window.ShowDialog();
}
if (sender != "Delete")
{
return(true);
}
var a = attr as IColumnProcess;
if (MessageBox.Show("确实要删除" + a.TypeName + "吗?", "提示信息", MessageBoxButton.OKCancel) !=
MessageBoxResult.OK)
{
return(true);
}
CurrentETLTools.Remove(a);
RefreshSamples();
return(true);
}
public ReadFileTF()
{
ConnectorSelector = new ExtendSelector <XFrmWorkAttribute>(PluginProvider.GetPluginCollection(typeof(IFileConnector)));
ConnectorSelector.SelectChanged += (s, e) =>
{
if (ConnectorSelector.SelectItem == null)
{
return;
}
Connector = PluginProvider.GetObjectInstance <IFileConnector>(ConnectorSelector.SelectItem.Name);
OnPropertyChanged("Connector");
};
Enabled = false;
}
public ProxyProvider()
{
ProxyStrategy = ProxyStrategy.NoAgent;
MaxVisitCount = 100000;
Proxies = new ObservableCollection <Proxy>();
ParaGeneratorSelector =
new ExtendSelector <XFrmWorkAttribute>(PluginProvider.GetPluginCollection(typeof(IColumnGenerator)));
ParaGeneratorSelector.SelectChanged += (s, e) =>
{
ParaGenerator = PluginProvider.GetObjectInstance <IColumnGenerator>(ParaGeneratorSelector.SelectItem.Name);
ParaGenerator.Column = "ProxyIp";
OnPropertyChanged("ParaGenerator");
};
ParaGeneratorSelector.SelectItem = ParaGeneratorSelector.Collection.FirstOrDefault(d => d.Name == "从文本生成");
}
public void FeelLucky()
{
isBusy = true;
var crawTargets = new List <XPathAnalyzer.CrawTarget>();
var task = TemporaryTask.AddTempTask("网页结构计算中",
HtmlDoc.SearchPropertiesSmart(CrawlItems, RootXPath, IsAttribute), crawTarget =>
{
crawTargets.Add(crawTarget);
var datas = HtmlDoc.GetDataFromXPath(crawTarget.CrawItems, IsMultiData, crawTarget.RootXPath);
crawTarget.Datas = datas;
}, d =>
{
isBusy = false;
if (crawTargets.Count == 0)
{
CrawTarget = null;
XLogSys.Print.Warn("没有检查到任何可选的列表页面");
return;
}
var luckModel = new FeelLuckyModel(crawTargets, HtmlDoc);
var view = PluginProvider.GetObjectInstance <ICustomView>("手气不错面板") as UserControl;
view.DataContext = luckModel;
var name = "手气不错";
var window = new Window {
Title = name
};
window.WindowState = WindowState.Maximized;
window.Content = view;
luckModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
var crawTarget = luckModel.CurrentTarget;
RootXPath = crawTarget.RootXPath;
CrawlItems.Clear();
CrawlItems.AddRange(crawTarget.CrawItems.Where(r => r.IsEnabled));
}
});
SysProcessManager.CurrentProcessTasks.Add(task);
}
public void SaveFile(DataCollection dataCollection, string path = null, string format = null)
{
IFileConnector exporter = null;
if (format != null)
{
exporter = PluginProvider.GetObjectInstance <IFileConnector>(format);
}
else
{
exporter = FileConnector.SmartGetExport(path);
}
if (exporter == null)
{
return;
}
var data = dataCollection.ComputeData;
exporter.FileName = path;
processManager.CurrentProcessTasks.Add(
TemporaryTask <FreeDocument> .AddTempTask(dataCollection + GlobalHelper.Get("key_252"),
exporter.WriteData(data), null, result =>
{
if (MainDescription.IsUIForm && string.IsNullOrEmpty(exporter.FileName) == false)
{
if (
MessageBox.Show(GlobalHelper.Get("key_253"), GlobalHelper.Get("key_99"),
MessageBoxButton.OKCancel) == MessageBoxResult.OK)
{
try
{
System.Diagnostics.Process.Start(exporter.FileName);
}
catch (Exception ex)
{
MessageBox.Show(GlobalHelper.Get("key_254") + ex.Message);
}
}
}
}, data.Count, notifyInterval: 1000));
}
/// <summary>
/// 添加新实例
/// </summary>
/// <param name="index"></param>
public static void Add <T>(this ICollection <T> collection, int index)
{
T instance;
try
{
instance = (T)PluginProvider.GetObjectInstance(typeof(T), index);
}
catch (Exception ex)
{
throw;
}
if (instance != null)
{
// instance.Init();
collection.Add(instance);
}
}
public IEnumerable <IFreeDocument> ReadText(string text, Action <int> alreadyGetSize = null)
{
var totals = FormatJsonData(text);
if (totals == null)
{
throw new Exception("文件不是合法的Json文件");
}
var array = totals as JsonArray;
if (array != null)
{
if (alreadyGetSize != null)
{
alreadyGetSize(array.Count);
}
foreach (object d in array)
{
var data = PluginProvider.GetObjectInstance(DataType) as IFreeDocument;
ItemtoNode(d, data);
yield return(data);
}
}
var obj = totals as JsonObject;
if (obj != null)
{
if (alreadyGetSize != null)
{
alreadyGetSize(1);
}
var data = PluginProvider.GetObjectInstance(DataType) as IFreeDocument;
ItemtoNode(obj, data);
yield return(data);
}
}
private void InitUI()
{
var alltools = CurrentETLTools.Take(ETLMount).ToList();
if (generateFloatGrid)
{
var gridview = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
var r = gridview.SetCurrentView(Documents);
if (ControlExtended.DockableManager == null)
{
return;
}
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, r, "样例数据");
generateFloatGrid = false;
}
else
{
var view = new GridView();
Dict.Clear();
var keys = new List <string> {
""
};
var docKeys = Documents.GetKeys(null, SampleMount);
keys.AddRange(docKeys);
var tool = CurrentTool;
foreach (var key in keys)
{
var col = new GridViewColumn
{
Header = key,
DisplayMemberBinding = new Binding($"[{key}]"),
Width = 155
};
view.Columns.Add(col);
var group = new SmartGroup
{
Name = key,
Value = alltools.Where(d => d.Column == key).ToList()
};
group.PropertyChanged += (s, e) =>
{
if (e.PropertyName == "Name")
{
var last = alltools.LastOrDefault() as IColumnDataTransformer;
if (last != null && last.TypeName == "列名修改器" && last.NewColumn == key)
{
last.NewColumn = group.Name;
}
else
{
last = PluginProvider.GetObjectInstance("列名修改器") as IColumnDataTransformer;
last.NewColumn = group.Name;
last.Column = key;
InsertModule(last);
ETLMount++;
OnPropertyChanged("ETLMount");
RefreshSamples();
}
}
};
Dict.Add(group
);
}
if (tool != null)
{
Dict.Where(d => d.Name == tool.Column).Execute(d => d.GroupType = GroupType.Input);
var transformer = tool as IColumnDataTransformer;
if (transformer != null)
{
var newcol = transformer.NewColumn.Split(' ');
if (transformer.IsMultiYield)
{
Dict.Execute(d => d.GroupType = newcol.Contains(d.Name)? GroupType.Input:GroupType.Output);
}
else
{
Dict.Where(d => d.Name == transformer.NewColumn).Execute(d => d.GroupType = GroupType.Output);;
}
}
}
var nullgroup = Dict.FirstOrDefault(d => string.IsNullOrEmpty(d.Name));
nullgroup?.Value.AddRange(
alltools.Where(
d =>
Documents.GetKeys().Contains(d.Column) == false &&
string.IsNullOrEmpty(d.Column) == false));
if (MainDescription.IsUIForm && IsUISupport)
{
if (dataView != null)
{
dataView.View = view;
}
}
}
}
public override bool Init()
{
if (MainDescription.IsUIForm)
{
DataCollections = new SafeObservable <DataCollection>();
dockableManager = MainFrmUI as IDockableManager;
var view = PluginProvider.GetObjectInstance <ICustomView>("系统状态视图");
var userControl = view as UserControl;
if (userControl != null)
{
userControl.DataContext = MainFrmUI;
dockableManager.AddDockAbleContent(FrmState.Mini, view, "系统状态视图");
}
}
else
{
DataCollections = new ObservableCollection <DataCollection>();
}
processManager = MainFrmUI.PluginDictionary["模块管理"] as IProcessManager;
commands = new List <ICommand>();
var dbaction = new BindingAction();
dbaction.ChildActions.Add(new Command("配置连接",
obj =>
{
Window w = PropertyGridFactory.GetPropertyWindow(obj);
w.ShowDialog();
},
obj => obj != null));
dbaction.ChildActions.Add(
new Command("刷新", obj => RefreshConnect(obj as IDataBaseConnector), obj => obj != null));
dbaction.ChildActions.Add(
new Command("执行查询", obj =>
{
var query = new QueryEntity();
query.Connector = obj as IDataBaseConnector;
query.GetQueryFunc = d =>
{
if (d == null)
{
return;
}
if (d.Any() == false)
{
return;
}
AddDataCollection(d, GetNewName());
};
propertyGridWindow.SetObjectView(query);
}, obj => obj != null));
dbaction.ChildActions.Add(
new Command("删除连接", obj =>
{
if (MessageBox.Show("确定要删除该连接吗？", "提示信息", MessageBoxButton.YesNoCancel) == MessageBoxResult.Yes)
{
var con = obj as DBConnectorBase;
_dbConnections.Remove(con);
}
}, obj => obj != null));
var dataaction = new BindingAction();
var tableAction = new BindingAction();
tableAction.ChildActions.Add(new Command(
"查看",
async obj =>
{
var items = obj as TableInfo;
List <IFreeDocument> dataAll = null;
try
{
dataAll = await
GetDataFromDB(items.Connector, items.Name, true,
items.Connector is FileManager ? -1 : 200);
}
catch (Exception ex)
{
MessageBox.Show("文件打开失败" + ex.Message);
return;
}
if (dataAll == null || dataAll.Count == 0)
{
XLogSys.Print.Warn("没有在表中的发现可用的数据");
return;
}
if (items.Connector is FileManager)
{
string file = (items.Connector as FileManager).LastFileName;
var name = Path.GetFileNameWithoutExtension(file);
AddDataCollection(dataAll, name);
return;
}
var excel = PluginProvider.GetObjectInstance <IDataViewer>("可编辑列表");
if (excel == null)
{
return;
}
object view = excel.SetCurrentView(dataAll.Select(d => d as IFreeDocument).ToList());
if (ControlExtended.DockableManager != null)
{
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, view, items.Name);
}
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"添加到数据集",
async obj =>
{
var items = obj as TableInfo;
List <IFreeDocument> datas = await GetDataFromDB(items.Connector, items.Name, true);
if (datas == null)
{
return;
}
AddDataCollection(datas, items.Name);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"添加虚拟数据集",
obj =>
{
var con = obj as TableInfo;
ReadCollection(con.Connector, con.Name, true);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"删除表",
obj =>
{
var items = obj as TableInfo;
DropTable(items.Connector, items.Name);
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"查看属性",
obj =>
{
Window w = PropertyGridFactory.GetPropertyWindow(obj);
w.ShowDialog();
},
obj => obj != null));
tableAction.ChildActions.Add(new Command(
"执行查询",
obj =>
{
var query = new QueryEntity();
query.TableInfo = obj as TableInfo;
query.Connector = query.TableInfo.Connector;
query.GetQueryFunc = d =>
{
if (d == null)
{
return;
}
if (d.Any() == false)
{
return;
}
AddDataCollection(d, GetNewName());
};
propertyGridWindow.SetObjectView(query);
}, obj => obj != null));
var visitData = new BindingAction("浏览方式");
IEnumerable <Command> visitCommands = PluginProvider.GetPluginCollection(typeof(IDataViewer)).Select(
d =>
{
var comm = new Command(d.Name);
comm.Execute = d2 =>
{
var data = d2 as DataCollection;
if (data.Count == 0)
{
MessageBox.Show("不存在任何数据", "提示信息");
return;
}
ControlExtended.UIInvoke(() =>
{
var view = PluginProvider.GetObjectInstance <IDataViewer>(d.Name);
object r = view.SetCurrentView(data.ComputeData);
if (ControlExtended.DockableManager != null)
{
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Float, r, data.Name + " " + d.Name);
}
});
};
return(comm);
});
visitData.Execute = obj => visitCommands.FirstOrDefault(d => d.Text == "可编辑列表").Execute(obj);
foreach (Command visitCommand in visitCommands)
{
visitData.ChildActions.Add(visitCommand);
}
dataaction.ChildActions.Add(new Command(
"数据清洗", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection == null)
{
return;
}
var plugin = processManager.GetOneInstance("SmartETLTool", true, true) as SmartETLTool;
dynamic generator = PluginProvider.GetObjectByType <IColumnProcess>("TableGE");
generator.TableSelector.SelectItem = collection;
plugin.CurrentETLTools.Add(generator);
plugin.RefreshSamples(true);
ControlExtended.DockableManager.ActiveModelContent(plugin);
}, obj => true, "new"));
dataaction.ChildActions.Add(new Command(
"拷贝", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection == null)
{
return;
}
DataCollection n = collection.Clone(true);
n.Name = GetNewName(collection.Name);
DataCollections.Add(n);
}, obj => true, "page_new"));
var saveData = new Command("另存为", d =>
{
DataCollection collection = GetCollection(d);
if (collection == null)
{
return;
}
var ofd = new SaveFileDialog {
Filter = FileConnector.GetDataFilter(), DefaultExt = "*"
};
ofd.FileName = collection.Name + ".xlsx";
if (ofd.ShowDialog() == true)
{
string filename = ofd.FileName;
SaveFile(collection.Name, filename);
}
}, obj => true, "save");
dataaction.ChildActions.Add(saveData);
dataaction.ChildActions.Add(visitData);
dataaction.ChildActions.Add(new Command(
"新建",
obj =>
DataCollections.Add(new DataCollection(new List <IFreeDocument>())
{
Name = GetNewName("新建数据集")
}), obj => true, "box"));
dataaction.ChildActions.Add(new Command(
"配置", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection != null)
{
propertyGridWindow.SetObjectView(collection);
}
}, obj => true, "settings"));
dataaction.ChildActions.Add(new Command(
"删除", obj =>
{
DataCollection collection = GetCollection(obj);
if (collection != null)
{
DataCollections.Remove(collection);
}
}, obj => true, "delete"));
var convert = new BindingAction("转换表类型");
dataaction.ChildActions.Add(convert);
convert.ChildActions.Add(new Command("转为非虚拟数据集", obj =>
{
DataCollection coll = GetCollection(obj);
if (coll.Count > 500000)
{
if (
MessageBox.Show("本集合数据量较大，转换可能会占用较高的内存和导致程序崩溃，确定继续吗?", "提示信息", MessageBoxButton.YesNoCancel) !=
MessageBoxResult.Yes)
{
return;
}
}
var docuts = new List <IFreeDocument>();
var task = TemporaryTask.AddTempTask("转为非虚拟数据集", coll.ComputeData, d =>
{
if (d != null)
{
docuts.Add(d as IFreeDocument);
}
}, result =>
{
var collection = new DataCollection(docuts)
{
Name = coll.Name + '1'
};
AddDataCollection(collection);
this.DataCollections.Remove(coll);
});
processManager.CurrentProcessTasks.Add(task);
}));
var insertdb = new BindingAction("保存到数据库");
insertdb.SetChildActionSource(() =>
{
return(_dbConnections.Select(dataBaseConnector => new Command(dataBaseConnector.Name, obj =>
{
var data = obj as DataCollection;
processManager.CurrentProcessTasks.Add(TemporaryTask.AddTempTask(data.Name + "插入到数据库", dataBaseConnector.InserDataCollection(data), result => dataBaseConnector.RefreshTableNames(), count: data.Count / 1000));
})).Cast <ICommand>().ToList());
});
dataaction.ChildActions.Add(insertdb);
var otherDataAction = new BindingAction();
otherDataAction.ChildActions.Add(new Command("清空数据", obj => CleanData(), obj => DataCollections.Count > 0));
commands.Add(dbaction);
commands.Add(tableAction);
commands.Add(dataaction);
commands.Add(otherDataAction);
var dblistAction = new BindingAction("数据库管理");
var addnew = new BindingAction("增加新连接");
dblistAction.ChildActions.Add(addnew);
foreach (XFrmWorkAttribute item in PluginProvider.GetPluginCollection(typeof(IDataBaseConnector)))
{
addnew.ChildActions.Add(new Command(item.Name)
{
Execute = obj =>
{
var con = PluginProvider.GetObjectInstance(item.MyType) as DBConnectorBase;
con.Name = item.Name;
_dbConnections.Add(con);
}
});
}
commands.Add(dblistAction);
dockableManager = MainFrmUI as IDockableManager;
if (processManager?.CurrentProject != null)
{
LoadDataConnections();
}
processManager.OnCurrentProjectChanged += (s, e) => LoadDataConnections();
ConfigUI = PropertyGridFactory.GetInstance(_dbConnections.FirstOrDefault());
propertyGridWindow = MainFrmUI.PluginDictionary["属性配置器"] as XFrmWorkPropertyGrid;
var changed = DataCollections as INotifyCollectionChanged;
changed.CollectionChanged += (s, e) => OnDataSourceChanged(new EventArgs());
return(true);
}
public void FeelLucky()
{
if (string.IsNullOrEmpty(this.URLHTML))
{
this.VisitUrlAsync();
}
isBusy = true;
var crawTargets = new List <XPathAnalyzer.CrawTarget>();
ICollection <CrawlItem> existItems = CrawlItems;
if (IsMultiData == ScriptWorkMode.One)
{
existItems = new List <CrawlItem> {
new CrawlItem {
Name = "temp", XPath = SelectXPath
}
}
}
;
var task = TemporaryTask <XPathAnalyzer.CrawTarget> .AddTempTaskSimple(GlobalHelper.Get("key_659"),
HtmlDoc.DocumentNode.SearchPropertiesSmart(IsMultiData, existItems, RootXPath, RootFormat, IsAttribute),
crawTarget =>
{
crawTargets.Add(crawTarget);
//var datas =
// HtmlDoc.DocumentNode.GetDataFromXPath(crawTarget.CrawItems, IsMultiData, crawTarget.RootXPath,
// RootFormat).ToList();
//crawTarget.Datas = datas;
}, d =>
{
isBusy = false;
if (crawTargets.Count == 0)
{
XLogSys.Print.Warn(GlobalHelper.Get("key_660"));
return;
}
var luckModel = new FeelLuckyModel(crawTargets, HtmlDoc, IsMultiData);
var view = PluginProvider.GetObjectInstance <ICustomView>(GlobalHelper.Get("key_657")) as UserControl;
view.DataContext = luckModel;
var name = GlobalHelper.Get("feellucky");
var window = new Window {
Title = name
};
window.WindowState = WindowState.Maximized;
window.Content = view;
luckModel.SetView(view, window);
window.Activate();
window.ShowDialog();
if (window.DialogResult == true)
{
var crawTarget = luckModel.CurrentTarget;
if (string.IsNullOrEmpty(RootXPath))
{
RootFormat = SelectorFormat.XPath;
}
RootXPath = crawTarget.RootXPath;
CrawlItems.AddRange(crawTarget.CrawItems.Where(r => r.IsEnabled && CrawlItems.FirstOrDefault(d2 => d2.XPath == r.XPath) == null));
}
});
SysProcessManager.CurrentProcessTasks.Add(task);
}
public SmartCrawler()
{
Http = new HttpItem();
CrawlItems = new ObservableCollection <CrawlItem>();
helper = new HttpHelper();
URL = "";
HtmlDoc = new HtmlDocument();
SelectText = "";
IsMultiData = ScriptWorkMode.List;
IsAttribute = true;
URL = "www.cnblogs.com";
ShareCookie = new TextEditSelector();
ShareCookie.GetItems = AppHelper.GetAllCrawlerNames(null);
Commands2 = CommandBuilder.GetCommands(
this,
new[]
{
new Command(GlobalHelper.Get("key_302"), obj => AddNewItem(),
obj =>
string.IsNullOrEmpty(SelectName) == false && string.IsNullOrEmpty(SelectXPath) == false,
"add"),
new Command(GlobalHelper.Get("search"), obj => GetXPathAsync(),
obj =>
currentXPaths != null, "magnify"),
new Command(GlobalHelper.Get("feellucky"),
obj => FeelLucky(),
obj => IsMultiData != ScriptWorkMode.NoTransform && isBusy == false, "smiley_happy"
),
new Command(GlobalHelper.Get("key_624"), obj =>
{
if (!(CrawlItems.Count > 0).SafeCheck(GlobalHelper.Get("key_625")))
{
return;
}
if (IsMultiData == ScriptWorkMode.List && CrawlItems.Count < 2)
{
MessageBox.Show(GlobalHelper.Get("key_626"), GlobalHelper.Get("key_99"));
return;
}
if (string.IsNullOrEmpty(this.URLHTML))
{
this.VisitUrlAsync();
}
var datas =
HtmlDoc.DocumentNode.GetDataFromXPath(CrawlItems, IsMultiData, RootXPath, RootFormat).Take(20)
.ToList();
var view = PluginProvider.GetObjectInstance <IDataViewer>(GlobalHelper.Get("key_230"));
var r = view.SetCurrentView(datas);
ControlExtended.DockableManager.AddDockAbleContent(
FrmState.Custom, r, GlobalHelper.Get("key_627"));
var rootPath =
XPath.GetMaxCompareXPath(CrawlItems.Select(d => d.XPath));
if (datas.Count > 0 && MessageBox.Show(GlobalHelper.Get("is_save_to_tables"), GlobalHelper.Get("key_99"), MessageBoxButton.YesNo) == MessageBoxResult.Yes)
{
SysDataManager.AddDataCollection(datas,
GlobalHelper.Get("key_624") + "_" + DateTime.Now.ToShortTimeString());
}
if (datas.Count > 1 && string.IsNullOrEmpty(RootXPath) && rootPath.Length > 0 &&
IsMultiData == ScriptWorkMode.List &&
MessageBox.Show(string.Format(GlobalHelper.Get("key_628"), rootPath), GlobalHelper.Get("key_99"),
MessageBoxButton.YesNo) == MessageBoxResult.Yes)
{
RootXPath = rootPath;
RootFormat = SelectorFormat.XPath;
HtmlDoc.CompileCrawItems(CrawlItems);
OnPropertyChanged("RootXPath");
}
}, icon: "page_search")
});
}
public override IEnumerable<IFreeDocument> ReadFile(Action<int> alreadyGetSize = null)
{
var titles = new List<string>();
var intColCount = 0;
var blnFlag = true;
foreach (var strline in FileEx.LineRead(FileName, AttributeHelper.GetEncoding(EncodingType)))
{
if (string.IsNullOrWhiteSpace(strline))
continue;
var aryline = strline.Split(new[] {SplitChar}, StringSplitOptions.None);
string[] objs = null;
//给datatable加上列名
if (blnFlag)
{
blnFlag = false;
intColCount = aryline.Length;
objs = new string[intColCount];
if (ContainHeader)
{
titles.AddRange(aryline);
continue;
}
for (var i = 0; i < intColCount; i++)
{
titles.Add("属性" + i);
}
for (var i = 0; i < intColCount; i++)
{
objs[i] = aryline[i].Trim();
}
}
else
{
var min = Math.Min(intColCount, aryline.Count());
objs = new string[min];
for (var i = 0; i < min; i++)
{
objs[i] = aryline[i].Trim();
}
}
var data = PluginProvider.GetObjectInstance(DataType) as IFreeDocument;
var dict = new Dictionary<string, object>();
for (var index = 0; index < Math.Min(titles.Count, objs.Length); index++)
{
var freeDocument = data as IFreeDocument;
if (freeDocument != null)
{
if (index == 0 && PropertyNames.Any() == false)
{
PropertyNames = titles.ToDictionary(d => d, d => d);
}
}
var title = titles[index];
var key = PropertyNames.FirstOrDefault(d => d.Value == title).Key;
if (key != null)
{
dict.Add(key, objs[index]);
}
}
data.DictDeserialize(dict);
yield return data;
}
}
