public static T GetModule <T>(this IColumnProcess process, string name) where T : class { var moduleName = (typeof(T) == typeof(SmartETLTool)) ? "数据清洗" : "网页采集器"; if (string.IsNullOrEmpty(name)) { XLogSys.Print.Error($"您没有填写“{process.TypeName}”的对应参数。"); return(default(T)); } var processManager = MainDescription.MainFrm.PluginDictionary[_static_name] as IProcessManager; var module = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T; if (module != null) { return(module); } var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name); if (task == null) { XLogSys.Print.Error($"没有找到名称为'{name}'的{moduleName},请检查“{process.TypeName}”是否填写错误"); throw new NullReferenceException($"can't find a ETL Module named {name}"); } ControlExtended.UIInvoke(() => { task.Load(false); }); module = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as T; return(module); }
public virtual bool Init(IEnumerable <IFreeDocument> datas) { mainstream = processManager.CurrentProcessCollections.OfType <SmartETLTool>() .FirstOrDefault(d => d.CurrentETLTools.Contains(this)); etl = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool; if (etl != null) { return(true); } var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == ETLSelector); if (task == null) { return(false); } ControlExtended.UIInvoke(() => { task.Load(false); }); etl = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool; etl.InitProcess(true); return(etl != null); }
public QueryEntity() { commands = new List <ICommand>(); commands.Add(new Command("执行查询", async obj => { List <FreeDocument> datas = null; try { ControlExtended.SetBusy(true); int count; datas = await Task.Run(() => Connector.QueryEntities(SQL, out count, TableInfo == null ? null : TableInfo.Name)); ControlExtended.SetBusy(true); } catch (Exception ex) { MessageBox.Show("查询出现异常" + ex.Message); XLogSys.Print.Error(ex); return; } finally { ControlExtended.SetBusy(false); } GetQueryFunc(datas); }, obj => string.IsNullOrEmpty(SQL) == false)); }
private void SaveCurrentProject(bool isDefaultPosition = true) { if (currentProject == null) { return; } if (CurrentProject.Tasks.Any() == false && MessageBox.Show("当前工程中没有包含任何任务,请在任务管理页中,将要保存的任务插入到当前工程中", "警告信息", MessageBoxButton.OKCancel) == MessageBoxResult.Cancel) { return; } if (isDefaultPosition) { ControlExtended.SafeInvoke(() => currentProject.Save(), LogType.Important, "保存当前工程"); var pro = ConfigFile.GetConfig <DataMiningConfig>().Projects.FirstOrDefault(); if (pro != null) { pro.SavePath = currentProject.SavePath; } } else { currentProject.SavePath = null; ControlExtended.SafeInvoke(() => currentProject.Save(), LogType.Important, "另存为当前工程"); } ConfigFile.Config.SaveConfig(); }
private void FiddlerApplicationAfterSessionComplete(Session oSession) { if (oSession.oRequest.headers == null) { return; } var httpitem = new HttpItem { Parameters = oSession.oRequest.headers.ToString() }; if ((oSession.BitFlags & SessionFlags.IsHTTPS) != 0) { httpitem.URL = "https://" + oSession.url; } else { httpitem.URL = "http://" + oSession.url; } httpitem.Postdata = Encoding.Default.GetString(oSession.RequestBody); if (string.IsNullOrWhiteSpace(SelectText) == false) { var content = oSession.GetResponseBodyAsString(); if (content.Contains(SelectText) == false) { return; } } IsSuperMode = true; StopVisit(); httpitem.DictCopyTo(Http); var post = ""; if (Http.Method == MethodType.POST) { post = "post请求的内容为:\n" + httpitem.Postdata + "\n"; } var window = MainFrm as Window; ControlExtended.UIInvoke(() => { if (window != null) { window.Topmost = true; } }); var info = $"已经成功获取嗅探字段! 真实请求地址:\n{oSession.url},\n已自动配置了网页采集器,请求类型为{Http.Method}\n {post}已经刷新了网页采集器的内容"; XLogSys.Print.Info(info); ControlExtended.UIInvoke(() => { if (window != null) { window.Topmost = false; } }); URL = oSession.url; }
public void StartVisit() { if (IsRunning) { return; } if (string.IsNullOrWhiteSpace(SelectText)) { MessageBox.Show("请填写包含在页面中的关键字信息:【2.属性提取】->【搜索字符】"); return; } ControlExtended.SafeInvoke(() => { IsJson2xml = true; var url = URL; if (url.StartsWith("http") == false) { url = "http://" + url; } System.Diagnostics.Process.Start(url); FiddlerApplication.BeforeResponse += FiddlerApplicationAfterSessionComplete; FiddlerApplication.Startup(8888, FiddlerCoreStartupFlags.Default); IsRunning = FiddlerApplication.IsStarted(); OnPropertyChanged("IsRunning"); }, LogType.Important, "请在关闭软件前关闭嗅探服务。【否则可能无法正常上网】,尝试启动服务"); }
public void AddErrorLog(IFreeDocument item, Exception ex, IColumnProcess process) { if (string.IsNullOrEmpty(errorLogName)) { return; } var param = item.Clone() as FreeDocument; param["__SysObjectID"] = process.ObjectID; param["__SysETL"] = (process as ToolBase)?.Father.Name; param["__SysERROR"] = ex.Message; param["__SysTime"] = DateTime.Now.ToString(); ControlExtended.UIInvoke(() => { if (ConfigFile.GetConfig <DataMiningConfig>().IsAddErrorCollection) { if (errorCollection == null) { errorCollection = new DataCollection() { Name = errorLogName }; DataManager.AddDataCollection(errorCollection); } errorCollection?.ComputeData.Add(param); errorCollection?.OnPropertyChanged("Count"); } else { XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), process.Column, process.TypeName, ex)); } }); }
public IDataProcess GetOneInstance(string name, bool isAddToList = true, bool newOne = false, bool isAddUI = false) { if (newOne) { var process = PluginProvider.GetObjectByType <IDataProcess>(name); if (process != null) { if (isAddToList) { ProcessCollection.Add(process); process.SysDataManager = dataManager; process.SysProcessManager = this; var rc4 = process as AbstractProcessMethod; if (rc4 != null) { rc4.MainPluginLocation = MainFrmUI.MainPluginLocation; rc4.MainFrm = MainFrmUI; } XLogSys.Print.Info("已经成功添加" + process.TypeName + "到当前列表"); } if (isAddUI) { ControlExtended.UIInvoke(() => LoadProcessView(process)); ControlExtended.UIInvoke(() => ShowConfigUI(process)); } return(process); } } return(ProcessCollection.Get(name, isAddToList)); }
protected SmartCrawler GetCrawler(string name) { var crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler; if (crawler != null) { IsMultiYield = crawler?.IsMultiData == ListType.List; } else { var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name); if (task != null) { ControlExtended.UIInvoke(() => { task.Load(false); }); crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler; } if (crawler == null) { XLogSys.Print.Error($"没有找到名称为'{name}'的网页采集器,是否没有填写或填写错误?"); } } return(crawler); }
public override bool Init(IEnumerable <IFreeDocument> datas) { OneOutput = false; crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as SmartCrawler; if (crawler != null) { IsMultiYield = crawler?.IsMultiData == ListType.List; } else { var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == CrawlerSelector); if (task == null) { return(false); } ControlExtended.UIInvoke(() => { task.Load(false); }); crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == CrawlerSelector) as SmartCrawler; } return(crawler != null && base.Init(datas)); }
public void StartVisit() { if (IsRunning) { return; } if (string.IsNullOrWhiteSpace(SelectText)) { MessageBox.Show("请填写包含在页面中的关键字信息:【2.属性提取】->【搜索字符】"); return; } ControlExtended.SafeInvoke(() => { var url = URL; if (url.StartsWith("http") == false) { url = "http://" + url; } CONFIG.bMITM_HTTPS = true; FiddlerApplication.AfterSessionComplete += FiddlerApplicationAfterSessionComplete; FiddlerApplication.Startup(8888, true, true); System.Diagnostics.Process.Start(url); OnPropertyChanged("IsRunning"); }, LogType.Important, "启动嗅探服务"); }
public QueryEntity() { commands = new List <ICommand>(); commands.Add(new Command(GlobalHelper.Get("key_213"), async obj => { List <FreeDocument> datas = null; try { ControlExtended.SetBusy(ProgressBarState.Indeterminate); int count; datas = await Task.Run(() => Connector.QueryEntities(SQL, out count, TableInfo == null ? null : TableInfo.Name)); ControlExtended.SetBusy(ProgressBarState.NoProgress); XLogSys.Print.Info(GlobalHelper.Get("key_325")); } catch (Exception ex) { MessageBox.Show(GlobalHelper.Get("key_214") + ex.Message); XLogSys.Print.Error(ex); return; } finally { ControlExtended.SetBusy(ProgressBarState.NoProgress); } GetQueryFunc(datas); }, obj => string.IsNullOrEmpty(SQL) == false, "page_search")); }
public void StartVisit() { if (IsRunning) { return; } if (string.IsNullOrWhiteSpace(SelectText) && ConfigFile.Config.Get <bool>("AutoStartStopFiddler") == true) { MessageBox.Show(GlobalHelper.Get("remind_10")); return; } ControlExtended.SafeInvoke(() => { var url = URL; if (url.StartsWith("http") == false) { url = "http://" + url; } CONFIG.IgnoreServerCertErrors = true; CONFIG.bMITM_HTTPS = true; FiddlerApplication.AfterSessionComplete += FiddlerApplicationAfterSessionComplete; var port = ConfigFile.Config.Get <int>("FiddlerPort"); FiddlerApplication.Startup(port, true, true, true); System.Diagnostics.Process.Start(url); XLogSys.Print.Info(GlobalHelper.FormatArgs("fiddler_start", "localhost", port)); OnPropertyChanged("IsRunning"); }, LogType.Important, GlobalHelper.Get("key_661")); }
public void StartVisit() { if (IsRunning) { return; } if (string.IsNullOrEmpty(URLFilter) && string.IsNullOrEmpty(this.ContentFilter)) { MessageBox.Show("请填写至少填写URL前缀或关键字中一项过滤规则"); return; } ControlExtended.SafeInvoke(() => { if (CanSave) { Documents.Clear(); } var url = URL; if (url.StartsWith("http") == false) { url = "http://" + url; } System.Diagnostics.Process.Start(url); FiddlerApplication.BeforeResponse += FiddlerApplicationAfterSessionComplete; FiddlerApplication.Startup(8888, FiddlerCoreStartupFlags.Default); IsRunning = FiddlerApplication.IsStarted(); OnPropertyChanged("IsRunning"); }, LogType.Important, "尝试启动服务"); }
private void AddNewItem(bool isAlert = true) { var path = SelectXPath; if (!string.IsNullOrEmpty(RootXPath)) { //TODO: 当XPath路径错误时,需要捕获异常 HtmlNode root = null; try { root = HtmlDoc.DocumentNode.SelectSingleNode(RootXPath); } catch (Exception ex) { XLogSys.Print.Error($"{RootXPath} 不能被识别为正确的XPath表达式,请检查"); } if (!(root != null).SafeCheck("使用当前父节点XPath,在文档中找不到任何父节点")) { return; } root = HtmlDoc.DocumentNode.SelectSingleNode(RootXPath)?.ParentNode; HtmlNode node = null; if ( !ControlExtended.SafeInvoke(() => HtmlDoc.DocumentNode.SelectSingleNode(path), ref node, LogType.Info, "检查子节点XPath正确性", true)) { return; } if (!(node != null).SafeCheck("使用当前子节点XPath,在文档中找不到任何子节点")) { return; } if (!node.IsAncestor(root) && isAlert) { if ( MessageBox.Show("当前XPath所在节点不是父节点的后代,请检查对应的XPath,是否依然要添加?", "提示信息", MessageBoxButton.YesNo) == MessageBoxResult.No) { return; } } path = XPath.TakeOff(node.XPath, root.XPath); } if (CrawlItems.FirstOrDefault(d => d.Name == SelectName) == null || MessageBox.Show("已经存在同名的属性,是否依然添加?", "提示信息", MessageBoxButton.OKCancel) == MessageBoxResult.OK) { var item = new CrawlItem { XPath = path, Name = SelectName, SampleData1 = SelectText }; CrawlItems.Add(item); SelectXPath = ""; SelectName = ""; XLogSys.Print.Info("成功添加属性"); } }
private async void VisitUrlAsync() { if (!enableRefresh) { return; } URLHTML = await MainFrm.RunBusyWork(() => { HttpStatusCode code; return(GetHtml(URL, out code)); }); if (URLHTML.Contains("尝试自动重定向") && MessageBox.Show("网站提示: " + URLHTML + "\n 通常原因是网站对请求合法性做了检查, 建议填写关键字对网页内容进行自动嗅探", "提示信息", MessageBoxButton.OK) == MessageBoxResult.OK) { return; } ControlExtended.SafeInvoke(() => HtmlDoc.LoadHtml(URLHTML), name: "解析html文档"); if (string.IsNullOrWhiteSpace(selectText) == false) { currentXPaths = HtmlDoc.SearchXPath(SelectText, () => IsAttribute).GetEnumerator(); GetXPathAsync(); } OnPropertyChanged("URLHTML"); }
public DataCollection ReadFile(string fileName, string format = null) { IFileConnector exporter = null; if (format != null) { exporter = PluginProvider.GetObjectInstance <IFileConnector>(format); } else { exporter = FileConnector.SmartGetExport(fileName); } if (exporter == null) { return(null); } exporter.FileName = fileName; fileName = exporter.FileName; ControlExtended.SafeInvoke( () => AddDataCollection(exporter.ReadFile(), Path.GetFileNameWithoutExtension(fileName)), LogType.Important); return(GetCollection(fileName)); }
public async Task <IEnumerable <ProjectItem> > GetProjects(string url = null) { try { if (url == null) { url = this.MarketUrl; } string username = null; string project = null; string target = null; if (!this.GetRepoInfo(url, out username, out project, out target)) { XLogSys.Print.Error(GlobalHelper.Get("market_url_check")); return(null); } IReadOnlyList <RepositoryContent> result = null; result = await client.Repository.Content.GetAllContents(username, project, target); var items = result?.Where(d => d.Type == ContentType.File && (d.Name.EndsWith(".xml", true, null) || d.Name.EndsWith(".hproj", true, null))).Select( d => { var projectItem = new ProjectItem() { IsRemote = true, SavePath = d.DownloadUrl }; var suffix = d.Name.Split('.').Last(); var name = d.Name.Replace("." + suffix, ""); projectItem.Name = name; var meta = name + ".meta"; var metafile = result.FirstOrDefault(d2 => d2.Name == meta); if (metafile != null) { Task.Factory.StartNew(() => { var response = WebRequest.Create(metafile.DownloadUrl).GetResponse().GetResponseStream(); using (StreamReader reader = new StreamReader(response, Encoding.UTF8)) { var item = reader.ReadToEnd(); var metainfo = ParameterItem.GetParameters(item); ControlExtended.UIInvoke(() => { projectItem.DictDeserialize(metainfo.ToDictionary(d2 => d2.Key, d2 => (object)d2.Value)); }); } }); } return(projectItem); }); return(items); } catch (Exception ex) { XLogSys.Print.Error(ex.Message); return(new List <ProjectItem>()); } }
private void SaveCurrentProject(bool isDefaultPosition = true) { if (CurrentProject == null) { return; } SaveCurrentTasks(); if (CurrentProject.Tasks.Any() == false && MessageBox.Show(GlobalHelper.Get("key_316"), GlobalHelper.Get("key_151"), MessageBoxButton.OKCancel) == MessageBoxResult.Cancel) { return; } if (isDefaultPosition) { ControlExtended.SafeInvoke(() => CurrentProject.Save(dataManager.DataCollections), LogType.Important, GlobalHelper.Get("key_317")); var pro = ConfigFile.GetConfig <DataMiningConfig>().Projects.FirstOrDefault(); if (pro != null) { pro.SavePath = CurrentProject.SavePath; } } else { CurrentProject.SavePath = null; ControlExtended.SafeInvoke(() => CurrentProject.Save(dataManager.DataCollections), LogType.Important, GlobalHelper.Get("key_318")); } ConfigFile.Config.SaveConfig(); }
protected SmartCrawler GetCrawler(string name) { var crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler; if (crawler != null) { IsMultiYield = crawler?.IsMultiData == ListType.List; } else { var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == name); if (task != null) { ControlExtended.UIInvoke(() => { task.Load(false); }); crawler = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == name) as SmartCrawler; } if (crawler == null) { if (string.IsNullOrEmpty(name)) { XLogSys.Print.Error($"您没有填写“从爬虫转换”的“爬虫选择”。需要填写要调用的网页采集器的名称"); } else { XLogSys.Print.Error($"没有找到名称为'{name}'的网页采集器,请检查“从爬虫转换”的“爬虫选择”是否填写错误"); } } } return(crawler); }
public virtual bool Init(IEnumerable <IFreeDocument> datas) { if (string.IsNullOrEmpty(ETLSelector)) { return(false); } etl = processManager.CurrentProcessCollections.FirstOrDefault(d => d.Name == ETLSelector) as SmartETLTool; if (etl != null) { return(true); } var task = processManager.CurrentProject.Tasks.FirstOrDefault(d => d.Name == ETLSelector); if (task == null) { throw new NullReferenceException($"can't find a ETL Module named {ETLSelector}"); } ControlExtended.UIInvoke(() => { task.Load(false); }); etl.InitProcess(true); return(etl != null); }
public IEnumerable <IFreeDocument> Execute(IEnumerable <IFreeDocument> documents) { foreach (var document in documents) { var doc = MappingDocument(document); if (AddTask) { var name = doc[Column]; ControlExtended.UIInvoke(() => { var task = TemporaryTask <FreeDocument> .AddTempTaskSimple("ETL" + name, func(new List <IFreeDocument> { doc }), d => d.LastOrDefault()); processManager.CurrentProcessTasks.Add(task); }); } else { var r = func(new List <IFreeDocument> { doc }).ToList(); } yield return(document); } }
private bool Refresh() { //motherkeys = this.Mother.Generate(Mother.CurrentETLTools.Take(Mother.CurrentETLTools.IndexOf(this.ETLModule)), false).Take(3).GetKeys().ToList(); // subkeys = // SubTask.Generate( // SubTask.CurrentETLTools.Take(RangeStart),false) // .Take(3).GetKeys().ToList(); var dict = MappingPairs.Where( d => string.IsNullOrEmpty(d.Target.SelectItem) == false && string.IsNullOrEmpty(d.Source.SelectItem) == false).Distinct().GroupBy(d => d.Target.SelectItem).Select(group => group.First()) .ToDictionary(d => d.Target.SelectItem, d => d.Source.SelectItem); ControlExtended.UIInvoke(() => { MappingPairs.Clear(); foreach (var key in subkeys) { var pair = new MappingPair(motherkeys, subkeys); pair.Target.SelectItem = key; string value = key; dict.TryGetValue(key, out value); pair.Source.SelectItem = value; MappingPairs.Add(pair); } }); return(true); //var index = Mother.CurrentETLTools.IndexOf(ETLModule); //if (index == -1) // return; //Mother.Generate(Mother.CurrentETLTools.Take(index), false); }
private async void VisitUrlAsync() { if (!enableRefresh) { return; } if (hasInit == false) { return; } URLHTML = await MainFrm.RunBusyWork(() => { HttpStatusCode code; ConfigFile.GetConfig <DataMiningConfig>().RequestCount++; return(GetHtml(URL, out code)); }, title : GlobalHelper.Get("long_visit_web")); if (URLHTML.Contains(GlobalHelper.Get("key_671")) && MessageBox.Show(GlobalHelper.Get("key_672") + URLHTML + GlobalHelper.Get("key_673"), GlobalHelper.Get("key_99"), MessageBoxButton.OK) == MessageBoxResult.OK) { return; } ControlExtended.SafeInvoke(() => { HtmlDoc.LoadHtml(URLHTML); if (MainDescription.IsUIForm) { var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager; var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this); if (control != null) { dynamic invoke = control.View; if (IsSuperMode == false) { invoke.UpdateHtml(URLHTML); OnPropertyChanged("HtmlDoc"); } else { invoke.UpdateHtml(GlobalHelper.Get("key_674")); } } } }, name: GlobalHelper.Get("key_675")); if (string.IsNullOrWhiteSpace(selectText) == false) { currentXPaths = HtmlDoc.SearchXPath(SelectText, () => IsAttribute).GetEnumerator(); GetXPathAsync(); } OnPropertyChanged("URLHTML"); }
private async void VisitUrlAsync() { if (!enableRefresh) { return; } if (hasInit == false) { return; } URLHTML = await MainFrm.RunBusyWork(() => { HttpStatusCode code; RequestManager.Instance.RequestCount++; return(GetHtml(URL, out code)); }); if (URLHTML.Contains("尝试自动重定向") && MessageBox.Show("网站提示: " + URLHTML + "\n 通常原因是网站对请求合法性做了检查, 建议填写关键字对网页内容进行自动嗅探", "提示信息", MessageBoxButton.OK) == MessageBoxResult.OK) { return; } ControlExtended.SafeInvoke(() => { HtmlDoc.LoadHtml(URLHTML); if (MainDescription.IsUIForm) { var dock = MainFrm as IDockableManager ?? ControlExtended.DockableManager; var control = dock?.ViewDictionary.FirstOrDefault(d => d.Model == this); if (control != null) { dynamic invoke = control.View; if (IsSuperMode == false) { invoke.UpdateHtml(URLHTML); OnPropertyChanged("HtmlDoc"); } else { invoke.UpdateHtml("超级模式下内置浏览器不展示内容,请查看左侧的文本内容"); } } } }, name: "解析html文档"); if (string.IsNullOrWhiteSpace(selectText) == false) { currentXPaths = HtmlDoc.SearchXPath(SelectText, () => IsAttribute).GetEnumerator(); GetXPathAsync(); } OnPropertyChanged("URLHTML"); }
public virtual void Remove() { ControlExtended.UIInvoke(() => ProcessManager.CurrentProcessTasks.Remove(this)); CancellationToken?.Cancel(); autoReset.Close(); IsStart = false; IsCanceled = true; }
public virtual void OnPropertyChanged(string propName) { //this.VerifyPropertyName(propName); if (null != PropertyChanged) { ControlExtended.UIInvoke(() => PropertyChanged(this, new PropertyChangedEventArgs(propName))); } }
private void SafeConnectDB() { ControlExtended.SafeInvoke(() => ConnectDB(), LogType.Important, GlobalHelper.Get("connect_db")); if (IsUseable) { RefreshTableNames(); } }
private void SafeConnectDB() { ControlExtended.SafeInvoke(() => ConnectDB(), LogType.Important, "连接数据库"); if (IsUseable) { RefreshTableNames(); } }
private void ExecuteAllExecutors() { if (MainDescription.IsUIForm && ControlExtended.UserCheck("确定启动执行器?", "警告信息")) { ExecuteDatas(); } }