public ScrapingConfigBase GetConfig()
{
List <ScrapingConfigBase> stepsTemp = new List <ScrapingConfigBase>();
var GetDocument = new PagingStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = Guid.Empty,
StepName = "Get Document",
StepType = StepType.Paging,
ExecuteConfig = new PagingExecuteConfig
{
Pattern = "https://batdongsan.com.vn/nha-dat-cho-thue/p@page",
UrlBuildType = UrlBuildType.ReplaceQuantity,
PagingPattern = "@page",
ScrapeType = ScrapingType.Paging,
ToPage = 4,
FromPage = 1
}
};
stepsTemp.Add(GetDocument);
var navigateToPageLink = new ProcessStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = GetDocument.StepId,
StepName = "Navigate To Page Link",
StepType = StepType.ProcessStep,
ExecuteConfig = new NavigateExecuteConfig
{
CanNavigateByDoc = true,
ScrapeType = ScrapingType.WebUrl,
Interval = 1000,
Description = "navigateToPageLink",
HeaderConfigs = new Dictionary <string, string>
{
{ "Cookie", "SERVERID=H; __cfduid=d3de39fbce46c6e4f0207c7ff79bc99ad1544108336; _ga=GA1.3.1314493246.1544108340; _gid=GA1.3.1671812483.1544108340; scs=%7B%22t%22%3A1%7D; sidtb=IAY8ZHzn8vHgNlf242x2X1kELl5edvy4; usidtb=iXDzGkacE9GZchJo09f0H5A3coyzQcXY; ASP.NET_SessionId=admsrwfwy4hk2d14yh4jovgw; __asc=28ff267516784077712e8aca640; __auc=28ff267516784077712e8aca640; ins-mig-done=1; _fbp=fb.2.1544108341321.1813261443; spUID=15441083412080d057358b4.6898a6cc; USER_SEARCH_PRODUCT_CONTEXT=49%7C326%7CSG%7C66%7C8898%7C2656%7C1883%2C18752252; insdrSV=4; current-currency=VND" },
{ "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" }
}
}
};
stepsTemp.Add(navigateToPageLink);
var GetContainer = new ProcessStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = navigateToPageLink.StepId,
StepName = "Get Container",
StepType = StepType.ProcessStep,
ExecuteConfig = new ExecuteConfig()
{
Pattern = "//div[@class='Main']/div[contains(@class, 'search-productItem')]",
ScrapeType = ScrapingType.HTMLDoc,
}
};
stepsTemp.Add(GetContainer);
var extractProductLink = new ProcessStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = GetContainer.StepId,
StepName = "extract Product Link",
StepType = StepType.ProcessStep,
ExecuteConfig = new HTMLAttributeExecuteConfig()
{
Pattern = "//div[@class='p-title']/h3[1]/a",
HtmlAttribute = "href",
ScrapeType = ScrapingType.HTMLAttribute,
EnclosePrevious = "https://batdongsan.com.vn"
}
};
stepsTemp.Add(extractProductLink);
var navigateToProductLink = new ProcessStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = extractProductLink.StepId,
StepName = "navigate To Product Link",
StepType = StepType.ProcessStep,
ExecuteConfig = new NavigateExecuteConfig
{
CanNavigateByDoc = true,
ScrapeType = ScrapingType.WebUrl,
Interval = 1000,
Description = "navigateToProductLink",
HeaderConfigs = new Dictionary <string, string>
{
{ "Cookie", "SERVERID=H; __cfduid=d3de39fbce46c6e4f0207c7ff79bc99ad1544108336; _ga=GA1.3.1314493246.1544108340; _gid=GA1.3.1671812483.1544108340; scs=%7B%22t%22%3A1%7D; sidtb=IAY8ZHzn8vHgNlf242x2X1kELl5edvy4; usidtb=iXDzGkacE9GZchJo09f0H5A3coyzQcXY; ASP.NET_SessionId=admsrwfwy4hk2d14yh4jovgw; __asc=28ff267516784077712e8aca640; __auc=28ff267516784077712e8aca640; ins-mig-done=1; _fbp=fb.2.1544108341321.1813261443; spUID=15441083412080d057358b4.6898a6cc; USER_SEARCH_PRODUCT_CONTEXT=49%7C326%7CSG%7C66%7C8898%7C2656%7C1883%2C18752252; insdrSV=4; current-currency=VND" },
{ "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" }
}
}
};
stepsTemp.Add(navigateToProductLink);
var buildData = new StorageStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = navigateToProductLink.StepId,
StepName = "build Customer Data",
ExecuteConfig = null,
StepType = StepType.Storage,
DataName = "DrawingDataModel"
};
stepsTemp.Add(buildData);
var GetDivInfo = new DataExtractionStepConfig()
{
StepId = Guid.NewGuid(),
ParentId = buildData.StepId,
StepName = "Get Div Info",
StepType = StepType.DataExtraction,
AttributeName = "Data",
ExecuteConfig = new ExecuteConfig()
{
Pattern = "//div[@class='div-table-cell table2']/div[@class='table-detail']/div[@id='divCustomerInfo']",
ScrapeType = ScrapingType.HTMLDoc,
}
};
stepsTemp.Add(GetDivInfo);
var rootStep = stepsTemp.First(x => x.ParentId == Guid.Empty);
var json1 = JsonConvert.SerializeObject(stepsTemp);
AddNextStep(stepsTemp, rootStep);
return(rootStep);
}
