public void Scrape()
{
_scrapedShoes.Clear();
ScrapeResponse serviceResponse = new ScrapeResponse();
try
{
string[,] websites = WebScraperFactory.Websites;
WebScraper scraper;
ScrapePageResponse websiteResponse = new ScrapePageResponse();
string[] website;
for (int i = 0; i < WebScraperFactory.WebsiteCount; ++i)
{
scraper = WebScraperFactory.CreateWebScraper(websites[i, 0]);
website = new string[] { websites[i, 0], websites[i, 1] };
ScrapePageRequest request = new ScrapePageRequest
{
Website = website,
ShoeList = _scrapedShoes
};
websiteResponse = scraper.ScrapeWebpage(request);
if (!websiteResponse.Success)
{
break;
}
}
serviceResponse.Success = websiteResponse.Success;
serviceResponse.Message = websiteResponse.Message;
serviceResponse.ScrapeCount = _scrapedShoes.Count;
}
catch (Exception ex)
{
serviceResponse.Success = false;
serviceResponse.Message = ex.Message;
serviceResponse.ScrapeCount = 0;
_scrapedShoes.Clear();
}
finally
{
FilterShoes("");
if (ScrapeCompletedEvent != null)
{
ScrapeCompletedEvent(this, new ScrapeResponseEventArgs(serviceResponse));
}
}
}
public override ScrapePageResponse ScrapeWebpage(ScrapePageRequest request)
{
ShoeModel tmpShoe;
ShoeModel[] tmpArray = null;
ScrapePageResponse response = new ScrapePageResponse {
Success = true, Message = ""
};
try
{
HtmlDocument webpage = _webscraper.Load(request.Website[1]);
HtmlNodeCollection nodes = webpage.DocumentNode.SelectNodes("//div[contains(@class, 'rt-holder')]");
_shoeCount = nodes.Count;
response.ScrapeCount = _shoeCount;
if (_shoeCount != 0)
{
tmpArray = new ShoeModel[_shoeCount];
for (int i = 0; i < _shoeCount; ++i)
{
tmpShoe = new ShoeModel();
tmpShoe.ID = 0;
tmpShoe.WebsiteName = request.Website[0];
tmpShoe.WebsiteURL = request.Website[1];
tmpShoe.ModelName = WebScraper.CleanHTMLString(nodes[i].SelectNodes("//h3 [contains(@class, 'entry-title')]")[i].InnerText);
tmpShoe.Description = WebScraper.CleanHTMLString(nodes[i].SelectNodes("//h3 [contains(@class, 'entry-title')]")[i].InnerText);
//tmpShoe.Description = WebScraper.CleanHTMLString(nodes[i].SelectNodes("//div[contains(@class, 'post-meta-user')]//i")[i].InnerText);
tmpShoe.ImageURI = nodes[i].SelectNodes("//div[contains(@class, 'rt-holder')]//img")[i].GetAttributeValue("src", "");
tmpShoe.ImageBinary = WebScraper.LoadImageBinary(tmpShoe.ImageURI);
tmpArray[i] = tmpShoe;
}
request.ShoeList.AddRange(tmpArray);
}
}
catch (Exception ex)
{
_shoeCount = 0;
response.Success = false;
response.Message = ex.Message;
}
return(response);
}
public abstract ScrapePageResponse ScrapeWebpage(ScrapePageRequest request);
