public void PopulateMobileSpecifications(bool evenId) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); using (var dbContext = new MobilesDbContext()) { try { List<Product> dbProducts = dbContext.Products.Where(p => p.IsRead == false && ((p.Id%2)==0)==evenId).ToList(); foreach (var dbProduct in dbProducts) { // KillIeProcesses(); string url = dbProduct.Url; using (var browser = new IE(url, true)) { //browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized); //browser.WaitForComplete(); Div specificationsDiv = browser.Div(Find.ById("specs-list")); var productDetails = new ProductDetails(); foreach (Table table in specificationsDiv.Tables) { var specification = new Specification(); Element specificationTypeCell = table.TableRows.First().Children().Where(c => c.TagName.ToUpper() == "TH").First(); if (specificationTypeCell != null && specificationTypeCell.Exists) { specification.Type = specificationTypeCell.Text; } foreach (TableRow tableRow in table.TableRows) { if (tableRow.TableCells.Count > 1) { TableCell keyCell = tableRow.TableCells.FirstOrDefault(); TableCell valueCell = tableRow.TableCells.LastOrDefault(); if (keyCell != null && keyCell.Exists && valueCell != null && valueCell.Exists) { if (keyCell.Links.Count > 0) { var specDetail = new SpecificationDetail(); specDetail.Key = keyCell.Links.First().Text; specDetail.Value = valueCell.Text; specification.Details.Add(specDetail); } else if (specification.Details.Count > 0) { specification.Details.Last().Value = string.Format("{0}{1}", specification.Details .Last(). Value, valueCell.Text); } } } } productDetails.Specifications.Add(specification); dbProduct.IsRead = true; } GenerateProductDetails(dbProduct,productDetails, dbContext); dbContext.SaveChanges(); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateMobileImages() { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { var products = dbContext.Products.Include("Brand").ToList(); var productImageUrls = dbContext.ProductImages.Select(p=>p.ImageUrl).ToList() ; foreach(var product in products) { var productUrl = product.Url; var productName = product.ProductName; var brandName = product.Brand.BrandName; var replaceString = brandName.Replace(" ", "_")+"_"+ productName.Replace(" ", "_"); var picutreUrl = productUrl.Replace(replaceString.ToLower(), replaceString.ToLower() + "-pictures"); KillIeProcesses(); using (var browser = new IE(picutreUrl, true)) { Div pictureDiv = browser.Div(Find.ById("pictures")); var imageUrls = pictureDiv.Images.Select(i => i.Src); foreach(var imageUrl in imageUrls ) { if(productImageUrls.Contains(imageUrl)==false) { dbContext.ProductImages.Add( new ProductImage() { ImageUrl = imageUrl, Product = product } ); } } } dbContext.SaveChanges(); } } }
public void PopulateBrandPages(IEnumerable<int> brandIdList) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { try { List<string> brandPagesList = dbContext.BrandPages.Select(b => b.Url).Distinct().ToList(); List<DataModels.Brand> brands = dbContext.Brands.Where(b => brandIdList.Contains(b.Id)).ToList(); // List<DataModels.Brand> brands = dbContext.Brands.ToList(); foreach (DataModels.Brand brand in brands) { // KillIeProcesses(); if (brandPagesList.Contains(brand.Url).Equals(false)) { brand.BrandPages.Add(new BrandPage { Url = brand.Url, IsInitialPage = true }); } string url = brand.Url; using (var browser = new IE(url,true)) { browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized); browser.WaitForComplete(); Div mainDiv = browser.Div(Find.ById("main")); if (mainDiv.Exists) { Div navigationDiv = mainDiv.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("nav-pages")). FirstOrDefault(); if (navigationDiv != null && navigationDiv.Exists) { List<string> pageLinks = navigationDiv.Links.Select(l=>l.Url).Distinct().ToList(); foreach (var pageLink in pageLinks) { if (brandPagesList.Contains(pageLink).Equals(false)) { brand.BrandPages.Add(new BrandPage { Url = pageLink }); } } } } } dbContext.SaveChanges(); } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateBrands() { Url = "http://www.gsmarena.com/makers.php3"; using (var browser = new IE(Url,true)) { try { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; Thread.Sleep(1000); var shop = new OnlineShoppingModel(); Div allBrandsDiv = browser.Div(Find.ById("mid-col")); Table allBrandTable = allBrandsDiv.Tables.First(); int index = 1; foreach (TableRow brandRow in allBrandTable.TableRows) { IEnumerable<Link> brandLinks = brandRow.Links.Where(l => l.Images.Count > 0); if (brandLinks.Count() > 0) { foreach (Link brandLink in brandLinks) { Image brandImage = brandLink.Images.First(); Link textBrandlink = brandRow.Links.Where(b => b.Url.Contains(brandLink.Url) && b.Images.Count <= 0). First(); string brandText = textBrandlink.Text; string numberOfProducts = brandText.Substring(brandText.IndexOf("(")).Replace("(", "").Replace(")", ""); var brand = new Brand { Id = index, Name = brandImage.Alt, Url = brandLink.Url, NoOfProducts = int.Parse(numberOfProducts), ImageUrl = brandImage.Src }; shop.Brands.Add(brand); index++; } } Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); using (var dbContext = new MobilesDbContext()) { List<string> brandUrlList = dbContext.Brands.Select(c => c.Url).ToList(); foreach (Brand brand in shop.Brands.Where(c => brandUrlList.Contains(c.Url) == false)) { dbContext.Brands.Add(new DataModels.Brand { BrandName = brand.Name, IsActive = true, AddedDate = DateTime.Now, LastModifiedDate = DateTime.Now, Url = brand.Url, NumberOfProducts = brand.NoOfProducts, ImageUrl = brand.ImageUrl }); } dbContext.SaveChanges(); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateBrandMobilePhones(IEnumerable<int> brandIdList) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { try { List<BrandPage> brandPages = dbContext.BrandPages.Include("Brand").Where(p => brandIdList.Contains(p.Brand.Id) && p.IsRead == false).ToList(); foreach (BrandPage brandPage in brandPages) { string url = brandPage.Url; try { using (var browser = new IE(url, true)) { browser.ShowWindow(NativeMethods.WindowShowStyle.Hide); browser.WaitForComplete(); IEnumerable<Div> productsDivs = browser.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("makers")); var listOfProducts = new List<OnlineShopModel.Product>(); foreach (Div productDiv in productsDivs) { List productsList = productDiv.Lists.First(); if (productsList != null && productsList.Exists) { foreach (ListItem productListItem in productsList.ListItems) { Link productLink = productListItem.Links.First(); Element productName = productLink.Children().Where(c => c.TagName.ToUpper() == "STRONG").First(); Image productImage = productLink.Images.First(); listOfProducts.Add(new OnlineShopModel.Product { ProductName = productName.Text, Url = productLink.Url, ImageUrl = productImage.Src, Description = productImage.Title }); } } } Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); List<Product> dbProductsList = dbContext.Products.ToList(); foreach (OnlineShopModel.Product product in listOfProducts) { Product filterProduct = dbProductsList.Where(p => p.Url == product.Url).FirstOrDefault(); if (filterProduct == null) { Product dbProduct = GetDbProduct(product); dbProduct.Brand = brandPage.Brand; dbContext.Products.Add(dbProduct); } } brandPage.IsRead = true; dbContext.SaveChanges(); } } catch (Exception) { } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }