public async Task <IActionResult> GetResult(SpiderViewModel spiderViewModel) { var runner = new PhantomJsRunner(null, "", ""); runner.WebDriver.Navigate().GoToUrl(spiderViewModel.Url); var result = runner.WebDriver.PageSource; var dataType = ""; if (result.First() == '<') { if (result.Contains("<html>")) { dataType = "html"; } else { dataType = "xml"; } } else { dataType = "json"; } return(View()); }
public void ProcessSpiderTask(Guid Id) { var spiderTaskModel = dbClient.Queryable <SpiderTaskModel>().First(x => x.Id == Id); var spiderTaskItemList = dbClient.Queryable <SpiderTaskItem>().Where(x => x.TaskId == Id).ToList(); var runner = new PhantomJsRunner(null, "", ""); runner.WebDriver.Navigate().GoToUrl(spiderTaskModel.Url); switch (spiderTaskModel.DataType) { case "josn": break; case "html": var htmlElement = runner.WebDriver.FindElement(By.TagName("html")); var dbDict = new Dictionary <string, object>(); foreach (var item in spiderTaskItemList) { if (item.ValueType == "single") { var targetValue = htmlElement.FindValueByCss(item.Expression); dbDict.Add(item.MappedDbColumn, targetValue); } else { var targetValues = htmlElement.FindMultiValueByCss(item.Expression); dbDict.Add(item.MappedDbColumn, string.Join(',', targetValues)); } } dbClient.Insertable(dbDict).AS(spiderTaskItemList.First().MappedDb).ExecuteCommandAsync(); break; case "xml": break; } }