public ScrapingConfigBase GetConfig() { List <ScrapingConfigBase> stepsTemp = new List <ScrapingConfigBase>(); var GetDocument = new PagingStepConfig() { StepId = Guid.NewGuid(), ParentId = Guid.Empty, StepName = "Get Document", StepType = StepType.Paging, ExecuteConfig = new PagingExecuteConfig { Pattern = "https://batdongsan.com.vn/nha-dat-cho-thue/p@page", UrlBuildType = UrlBuildType.ReplaceQuantity, PagingPattern = "@page", ScrapeType = ScrapingType.Paging, ToPage = 4, FromPage = 1 } }; stepsTemp.Add(GetDocument); var navigateToPageLink = new ProcessStepConfig() { StepId = Guid.NewGuid(), ParentId = GetDocument.StepId, StepName = "Navigate To Page Link", StepType = StepType.ProcessStep, ExecuteConfig = new NavigateExecuteConfig { CanNavigateByDoc = true, ScrapeType = ScrapingType.WebUrl, Interval = 1000, Description = "navigateToPageLink", HeaderConfigs = new Dictionary <string, string> { { "Cookie", "SERVERID=H; __cfduid=d3de39fbce46c6e4f0207c7ff79bc99ad1544108336; _ga=GA1.3.1314493246.1544108340; _gid=GA1.3.1671812483.1544108340; scs=%7B%22t%22%3A1%7D; sidtb=IAY8ZHzn8vHgNlf242x2X1kELl5edvy4; usidtb=iXDzGkacE9GZchJo09f0H5A3coyzQcXY; ASP.NET_SessionId=admsrwfwy4hk2d14yh4jovgw; __asc=28ff267516784077712e8aca640; __auc=28ff267516784077712e8aca640; ins-mig-done=1; _fbp=fb.2.1544108341321.1813261443; spUID=15441083412080d057358b4.6898a6cc; USER_SEARCH_PRODUCT_CONTEXT=49%7C326%7CSG%7C66%7C8898%7C2656%7C1883%2C18752252; insdrSV=4; current-currency=VND" }, { "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" } } } }; stepsTemp.Add(navigateToPageLink); var GetContainer = new ProcessStepConfig() { StepId = Guid.NewGuid(), ParentId = navigateToPageLink.StepId, StepName = "Get Container", StepType = StepType.ProcessStep, ExecuteConfig = new ExecuteConfig() { Pattern = "//div[@class='Main']/div[contains(@class, 'search-productItem')]", ScrapeType = ScrapingType.HTMLDoc, } }; stepsTemp.Add(GetContainer); var extractProductLink = new ProcessStepConfig() { StepId = Guid.NewGuid(), ParentId = GetContainer.StepId, StepName = "extract Product Link", StepType = StepType.ProcessStep, ExecuteConfig = new HTMLAttributeExecuteConfig() { Pattern = "//div[@class='p-title']/h3[1]/a", HtmlAttribute = "href", ScrapeType = ScrapingType.HTMLAttribute, EnclosePrevious = "https://batdongsan.com.vn" } }; stepsTemp.Add(extractProductLink); var navigateToProductLink = new ProcessStepConfig() { StepId = Guid.NewGuid(), ParentId = extractProductLink.StepId, StepName = "navigate To Product Link", StepType = StepType.ProcessStep, ExecuteConfig = new NavigateExecuteConfig { CanNavigateByDoc = true, ScrapeType = ScrapingType.WebUrl, Interval = 1000, Description = "navigateToProductLink", HeaderConfigs = new Dictionary <string, string> { { "Cookie", "SERVERID=H; __cfduid=d3de39fbce46c6e4f0207c7ff79bc99ad1544108336; _ga=GA1.3.1314493246.1544108340; _gid=GA1.3.1671812483.1544108340; scs=%7B%22t%22%3A1%7D; sidtb=IAY8ZHzn8vHgNlf242x2X1kELl5edvy4; usidtb=iXDzGkacE9GZchJo09f0H5A3coyzQcXY; ASP.NET_SessionId=admsrwfwy4hk2d14yh4jovgw; __asc=28ff267516784077712e8aca640; __auc=28ff267516784077712e8aca640; ins-mig-done=1; _fbp=fb.2.1544108341321.1813261443; spUID=15441083412080d057358b4.6898a6cc; USER_SEARCH_PRODUCT_CONTEXT=49%7C326%7CSG%7C66%7C8898%7C2656%7C1883%2C18752252; insdrSV=4; current-currency=VND" }, { "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" } } } }; stepsTemp.Add(navigateToProductLink); var buildData = new StorageStepConfig() { StepId = Guid.NewGuid(), ParentId = navigateToProductLink.StepId, StepName = "build Customer Data", ExecuteConfig = null, StepType = StepType.Storage, DataName = "DrawingDataModel" }; stepsTemp.Add(buildData); var GetDivInfo = new DataExtractionStepConfig() { StepId = Guid.NewGuid(), ParentId = buildData.StepId, StepName = "Get Div Info", StepType = StepType.DataExtraction, AttributeName = "Data", ExecuteConfig = new ExecuteConfig() { Pattern = "//div[@class='div-table-cell table2']/div[@class='table-detail']/div[@id='divCustomerInfo']", ScrapeType = ScrapingType.HTMLDoc, } }; stepsTemp.Add(GetDivInfo); var rootStep = stepsTemp.First(x => x.ParentId == Guid.Empty); var json1 = JsonConvert.SerializeObject(stepsTemp); AddNextStep(stepsTemp, rootStep); return(rootStep); }