public async Task <IActionResult> GetResult(SpiderViewModel spiderViewModel)
{
var runner = new PhantomJsRunner(null, "", "");
runner.WebDriver.Navigate().GoToUrl(spiderViewModel.Url);
var result = runner.WebDriver.PageSource;
var dataType = "";
if (result.First() == '<')
{
if (result.Contains("<html>"))
{
dataType = "html";
}
else
{
dataType = "xml";
}
}
else
{
dataType = "json";
}
return(View());
}
public void ProcessSpiderTask(Guid Id)
{
var spiderTaskModel = dbClient.Queryable <SpiderTaskModel>().First(x => x.Id == Id);
var spiderTaskItemList = dbClient.Queryable <SpiderTaskItem>().Where(x => x.TaskId == Id).ToList();
var runner = new PhantomJsRunner(null, "", "");
runner.WebDriver.Navigate().GoToUrl(spiderTaskModel.Url);
switch (spiderTaskModel.DataType)
{
case "josn":
break;
case "html":
var htmlElement = runner.WebDriver.FindElement(By.TagName("html"));
var dbDict = new Dictionary <string, object>();
foreach (var item in spiderTaskItemList)
{
if (item.ValueType == "single")
{
var targetValue = htmlElement.FindValueByCss(item.Expression);
dbDict.Add(item.MappedDbColumn, targetValue);
}
else
{
var targetValues = htmlElement.FindMultiValueByCss(item.Expression);
dbDict.Add(item.MappedDbColumn, string.Join(',', targetValues));
}
}
dbClient.Insertable(dbDict).AS(spiderTaskItemList.First().MappedDb).ExecuteCommandAsync();
break;
case "xml":
break;
}
}
