public async Task <List <T> > ScrapPage(string url) { List <T> data; try { IPageScraper siteReader = new PageScraper(); Task <string> rawPageText = siteReader.GetHtmlAsync(url); Console.WriteLine("13"); string rawPageTextResult = await rawPageText; Console.WriteLine("14"); data = ParseDataFromRawPageTxt(rawPageTextResult); } catch (Exception ex) { Console.WriteLine(ex.ToString()); throw; } return(data); }
public static async Task Main(string[] args) { var url = string.Empty; var pagesNumber = 8; for (int i = 1; i < pagesNumber; i++) { if (i != 1) { url = Constants.Url + string.Format("/{0}-pg", i); } PageScraper scrape = new PageScraper(url); scrape.Start(); foreach (var item in scrape.resultItems) { var propertyScrape = new PropertyScraper(item.Value.ToString()); propertyScrape.Start(); var prediction = new PredictionBase(propertyScrape.Result.Pictures); var propertyPredictionVector = await prediction.GetPredictionsForProperty(); var property = new Property { Area = propertyScrape.Result.Area, Description = propertyScrape.Result.Description, PropertyDescriptionTitle = propertyScrape.Result.PropertyDescriptionTitle, Title = propertyScrape.Result.Title, Pictures = JsonConvert.SerializeObject(propertyScrape.Result.Pictures), Location = propertyScrape.Result.Location, PropertyVector = JsonConvert.SerializeObject(propertyPredictionVector) }; var propertyObject = new PropertyDbLogic(); propertyObject.SaveProperty(property); } } }