public static Product GetProduct(string product_uri) { Product p = new Product(); List <string> elements = new List <string>(); List <string> headers = new List <string>(); headers.Add("Components"); headers.Add("Application"); headers.Add("Features and Benefits"); headers.Add("General description"); headers.Add("Packaging"); headers.Add("Reconstitution"); headers.Add("Other Notes"); headers.Add("Legal Information"); headers.Add("Caution"); headers.Add("Biochem/physiol Actions"); headers.Add("Preparation Note"); elements.Add("//div[@class='descriptionContent']"); HtmlDocument dataPage = DataGetter.GetHtmlpage(new Uri(product_uri)); List <KeyValuePair <string, HtmlNodeCollection> > dataDescription = DataGetter.GetDataByXPATH(dataPage, elements); p.Description = SigmaAldrichParser.ParseDescription(dataDescription, headers); p.Properties = SigmaAldrichParser.ParseDetailProperties(dataPage.DocumentNode); List <string> elements1 = new List <string>(); elements1.Add("//p[@class='product-name']"); p.Name = GetProductName(dataPage, elements1); return(p); }
public static List <Description> GetAllProductsForCategory(string category, int num_products_to_get, int pages_to_get) { string categoryUri = GetCategoryUri(category); int numProductsToGet = 0; List <string> paginationUri = GetCategoryPaginationUrls(categoryUri, pages_to_get); List <string> productsUris = GetAllProductsUri(paginationUri, num_products_to_get); List <Description> products = new List <Description>(); List <string> elements = new List <string>(); List <string> headers = new List <string>(); headers.Add("Components"); headers.Add("Application"); headers.Add("Features and Benefits"); headers.Add("General description"); headers.Add("Packaging"); headers.Add("Reconstitution"); headers.Add("Other Notes"); headers.Add("Legal Information"); headers.Add("Caution"); headers.Add("Biochem/physiol Actions"); headers.Add("Preparation Note"); elements.Add("//div[@class='descriptionContent']"); if (num_products_to_get == 0) { //get all products numProductsToGet = productsUris.Count; } for (int i = 0; i < num_products_to_get; i++) { List <KeyValuePair <string, HtmlNodeCollection> > dataDescription = DataGetter.GetDataByXPATH(new Uri(productsUris[i]), elements); Description description = SigmaAldrichParser.ParseDescription(dataDescription, headers); products.Add(description); System.Threading.Thread.Sleep((int)DataGetter.GetRandomNumber(5.0, 15.0) * 1000); } return(products); }