public static List <string> GetProductsUri(string pagination_uri) { //go over the product page and extract product url's List <string> productsLinks = new List <string>(); HtmlDocument dataPage = DataGetter.GetHtmlpage(new Uri(pagination_uri)); //now get all product url's from page List <string> elements = new List <string>(); elements.Add("//li[@class='productNumberValue']"); List <KeyValuePair <string, HtmlNodeCollection> > data = DataGetter.GetDataByXPATH(dataPage, elements); HtmlNodeCollection productsNodes = data[0].Value; for (int j = 0; /*j < numProductsToGet && */ j < productsNodes.Count; j++) { string link = productsNodes[j].InnerHtml; string extractedLink = ExtractLinkFromHtml(productsNodes[j], "href=", "\">"); productsLinks.Add(SigmaAldrichConstants.SigmaAldrichMain + "/" + extractedLink); } System.Threading.Thread.Sleep((int)DataGetter.GetRandomNumber(5.0, 15.0) * 1000); return(productsLinks); }
public static List <string> GetAllProductsUri(List <string> pagination_uri, int num_pages_to_get) { //go over all product pages and extract product url's List <string> productsLinks = new List <string>(); //int numProductsToGet = num_pages_to_get; //if (numProductsToGet == 0) //{ // numProductsToGet = pagination_uri.Count; //} for (int i = 0; i < pagination_uri.Count; i++) { HtmlDocument dataPage = DataGetter.GetHtmlpage(new Uri(pagination_uri[i])); //now get all product url's from page List <string> elements = new List <string>(); elements.Add("//li[@class='productNumberValue']"); List <KeyValuePair <string, HtmlNodeCollection> > data = DataGetter.GetDataByXPATH(dataPage, elements); HtmlNodeCollection productsNodes = data[0].Value; for (int j = 0; /*j < numProductsToGet && */ j < productsNodes.Count; j++) { string link = productsNodes[j].InnerHtml; string extractedLink = ExtractLinkFromHtml(productsNodes[j], "href=", "\">"); productsLinks.Add(SigmaAldrichConstants.SigmaAldrichMain + "/" + extractedLink); } //foreach (HtmlNode node in productsNodes) //{ // string link = node.InnerHtml; // string extractedLink = ExtractLinkFromHtml(node, "href=", "\">"); // productsLinks.Add(SigmaAldrichConstants.SigmaAldrichMain + "/" + extractedLink); //} //if (productsLinks.Count < numProductsToGet) //{ System.Threading.Thread.Sleep((int)DataGetter.GetRandomNumber(5.0, 15.0) * 1000); //} //else //{ // break; //} } return(productsLinks); }
public static List <Description> GetAllProductsForCategory(string category, int num_products_to_get, int pages_to_get) { string categoryUri = GetCategoryUri(category); int numProductsToGet = 0; List <string> paginationUri = GetCategoryPaginationUrls(categoryUri, pages_to_get); List <string> productsUris = GetAllProductsUri(paginationUri, num_products_to_get); List <Description> products = new List <Description>(); List <string> elements = new List <string>(); List <string> headers = new List <string>(); headers.Add("Components"); headers.Add("Application"); headers.Add("Features and Benefits"); headers.Add("General description"); headers.Add("Packaging"); headers.Add("Reconstitution"); headers.Add("Other Notes"); headers.Add("Legal Information"); headers.Add("Caution"); headers.Add("Biochem/physiol Actions"); headers.Add("Preparation Note"); elements.Add("//div[@class='descriptionContent']"); if (num_products_to_get == 0) { //get all products numProductsToGet = productsUris.Count; } for (int i = 0; i < num_products_to_get; i++) { List <KeyValuePair <string, HtmlNodeCollection> > dataDescription = DataGetter.GetDataByXPATH(new Uri(productsUris[i]), elements); Description description = SigmaAldrichParser.ParseDescription(dataDescription, headers); products.Add(description); System.Threading.Thread.Sleep((int)DataGetter.GetRandomNumber(5.0, 15.0) * 1000); } return(products); }