public void DownloadMobileImages() { Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); var dbContext = new MobilesDbContext(); var brandImages = dbContext.ProductImages.Select(b => b.ImageUrl).ToList(); var folderPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + @"\MobileImages\"; foreach (string imageUrl in brandImages) { var fileName = Path.GetFileName(imageUrl); var request = (HttpWebRequest)WebRequest.Create(imageUrl); WebResponse response = request.GetResponse(); using (Stream stream = response.GetResponseStream()) using (FileStream fs = File.OpenWrite(folderPath + fileName.Replace("lg_", ""))) { byte[] bytes = new byte[1024]; int count; while ((count = stream.Read(bytes, 0, bytes.Length)) != 0) { fs.Write(bytes, 0, count); } } } }
public void PopulateMobileImages() { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { var products = dbContext.Products.Include("Brand").ToList(); var productImageUrls = dbContext.ProductImages.Select(p=>p.ImageUrl).ToList() ; foreach(var product in products) { var productUrl = product.Url; var productName = product.ProductName; var brandName = product.Brand.BrandName; var replaceString = brandName.Replace(" ", "_")+"_"+ productName.Replace(" ", "_"); var picutreUrl = productUrl.Replace(replaceString.ToLower(), replaceString.ToLower() + "-pictures"); KillIeProcesses(); using (var browser = new IE(picutreUrl, true)) { Div pictureDiv = browser.Div(Find.ById("pictures")); var imageUrls = pictureDiv.Images.Select(i => i.Src); foreach(var imageUrl in imageUrls ) { if(productImageUrls.Contains(imageUrl)==false) { dbContext.ProductImages.Add( new ProductImage() { ImageUrl = imageUrl, Product = product } ); } } } dbContext.SaveChanges(); } } }
public void PopulateMobileSpecifications(bool evenId) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); using (var dbContext = new MobilesDbContext()) { try { List<Product> dbProducts = dbContext.Products.Where(p => p.IsRead == false && ((p.Id%2)==0)==evenId).ToList(); foreach (var dbProduct in dbProducts) { // KillIeProcesses(); string url = dbProduct.Url; using (var browser = new IE(url, true)) { //browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized); //browser.WaitForComplete(); Div specificationsDiv = browser.Div(Find.ById("specs-list")); var productDetails = new ProductDetails(); foreach (Table table in specificationsDiv.Tables) { var specification = new Specification(); Element specificationTypeCell = table.TableRows.First().Children().Where(c => c.TagName.ToUpper() == "TH").First(); if (specificationTypeCell != null && specificationTypeCell.Exists) { specification.Type = specificationTypeCell.Text; } foreach (TableRow tableRow in table.TableRows) { if (tableRow.TableCells.Count > 1) { TableCell keyCell = tableRow.TableCells.FirstOrDefault(); TableCell valueCell = tableRow.TableCells.LastOrDefault(); if (keyCell != null && keyCell.Exists && valueCell != null && valueCell.Exists) { if (keyCell.Links.Count > 0) { var specDetail = new SpecificationDetail(); specDetail.Key = keyCell.Links.First().Text; specDetail.Value = valueCell.Text; specification.Details.Add(specDetail); } else if (specification.Details.Count > 0) { specification.Details.Last().Value = string.Format("{0}{1}", specification.Details .Last(). Value, valueCell.Text); } } } } productDetails.Specifications.Add(specification); dbProduct.IsRead = true; } GenerateProductDetails(dbProduct,productDetails, dbContext); dbContext.SaveChanges(); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateBrands() { Url = "http://www.gsmarena.com/makers.php3"; using (var browser = new IE(Url,true)) { try { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; Thread.Sleep(1000); var shop = new OnlineShoppingModel(); Div allBrandsDiv = browser.Div(Find.ById("mid-col")); Table allBrandTable = allBrandsDiv.Tables.First(); int index = 1; foreach (TableRow brandRow in allBrandTable.TableRows) { IEnumerable<Link> brandLinks = brandRow.Links.Where(l => l.Images.Count > 0); if (brandLinks.Count() > 0) { foreach (Link brandLink in brandLinks) { Image brandImage = brandLink.Images.First(); Link textBrandlink = brandRow.Links.Where(b => b.Url.Contains(brandLink.Url) && b.Images.Count <= 0). First(); string brandText = textBrandlink.Text; string numberOfProducts = brandText.Substring(brandText.IndexOf("(")).Replace("(", "").Replace(")", ""); var brand = new Brand { Id = index, Name = brandImage.Alt, Url = brandLink.Url, NoOfProducts = int.Parse(numberOfProducts), ImageUrl = brandImage.Src }; shop.Brands.Add(brand); index++; } } Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); using (var dbContext = new MobilesDbContext()) { List<string> brandUrlList = dbContext.Brands.Select(c => c.Url).ToList(); foreach (Brand brand in shop.Brands.Where(c => brandUrlList.Contains(c.Url) == false)) { dbContext.Brands.Add(new DataModels.Brand { BrandName = brand.Name, IsActive = true, AddedDate = DateTime.Now, LastModifiedDate = DateTime.Now, Url = brand.Url, NumberOfProducts = brand.NoOfProducts, ImageUrl = brand.ImageUrl }); } dbContext.SaveChanges(); } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateBrandPages(IEnumerable<int> brandIdList) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { try { List<string> brandPagesList = dbContext.BrandPages.Select(b => b.Url).Distinct().ToList(); List<DataModels.Brand> brands = dbContext.Brands.Where(b => brandIdList.Contains(b.Id)).ToList(); // List<DataModels.Brand> brands = dbContext.Brands.ToList(); foreach (DataModels.Brand brand in brands) { // KillIeProcesses(); if (brandPagesList.Contains(brand.Url).Equals(false)) { brand.BrandPages.Add(new BrandPage { Url = brand.Url, IsInitialPage = true }); } string url = brand.Url; using (var browser = new IE(url,true)) { browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized); browser.WaitForComplete(); Div mainDiv = browser.Div(Find.ById("main")); if (mainDiv.Exists) { Div navigationDiv = mainDiv.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("nav-pages")). FirstOrDefault(); if (navigationDiv != null && navigationDiv.Exists) { List<string> pageLinks = navigationDiv.Links.Select(l=>l.Url).Distinct().ToList(); foreach (var pageLink in pageLinks) { if (brandPagesList.Contains(pageLink).Equals(false)) { brand.BrandPages.Add(new BrandPage { Url = pageLink }); } } } } } dbContext.SaveChanges(); } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public void PopulateBrandMobilePhones(IEnumerable<int> brandIdList) { Settings.AttachToBrowserTimeOut = 240; Settings.WaitUntilExistsTimeOut = 240; Settings.WaitForCompleteTimeOut = 240; using (var dbContext = new MobilesDbContext()) { try { List<BrandPage> brandPages = dbContext.BrandPages.Include("Brand").Where(p => brandIdList.Contains(p.Brand.Id) && p.IsRead == false).ToList(); foreach (BrandPage brandPage in brandPages) { string url = brandPage.Url; try { using (var browser = new IE(url, true)) { browser.ShowWindow(NativeMethods.WindowShowStyle.Hide); browser.WaitForComplete(); IEnumerable<Div> productsDivs = browser.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("makers")); var listOfProducts = new List<OnlineShopModel.Product>(); foreach (Div productDiv in productsDivs) { List productsList = productDiv.Lists.First(); if (productsList != null && productsList.Exists) { foreach (ListItem productListItem in productsList.ListItems) { Link productLink = productListItem.Links.First(); Element productName = productLink.Children().Where(c => c.TagName.ToUpper() == "STRONG").First(); Image productImage = productLink.Images.First(); listOfProducts.Add(new OnlineShopModel.Product { ProductName = productName.Text, Url = productLink.Url, ImageUrl = productImage.Src, Description = productImage.Title }); } } } Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>()); List<Product> dbProductsList = dbContext.Products.ToList(); foreach (OnlineShopModel.Product product in listOfProducts) { Product filterProduct = dbProductsList.Where(p => p.Url == product.Url).FirstOrDefault(); if (filterProduct == null) { Product dbProduct = GetDbProduct(product); dbProduct.Brand = brandPage.Brand; dbContext.Products.Add(dbProduct); } } brandPage.IsRead = true; dbContext.SaveChanges(); } } catch (Exception) { } } } catch (Exception ex) { Console.WriteLine(ex.ToString()); Console.ReadLine(); //throw ex; } } }
public Product GenerateProductDetails(Product product, ProductDetails specdetails, MobilesDbContext dbContext) { int index = 1; List<SpecifictionType> specTypeList = dbContext.SpecifictionTypes.ToList(); List<SpecifictionHeading> specHeadingList = dbContext.SpecifictionHeadings.ToList(); foreach (Specification specification in specdetails.Specifications) { SpecifictionType specType = specTypeList.Where(s => s.Name == specification.Type).FirstOrDefault(); if (specType == null) { specType = new SpecifictionType {Name = specification.Type}; dbContext.SpecifictionTypes.Add(specType); } var dbSpecifiction = new Specifiction { Type = specType, Product = product }; foreach (SpecificationDetail specDetail in specification.Details) { SpecifictionHeading specHeading = specHeadingList.Where(s => s.Name == specDetail.Key).FirstOrDefault(); if (specHeading == null) { specHeading = new SpecifictionHeading {Name = specDetail.Key}; dbContext.SpecifictionHeadings.Add(specHeading); } var dbSpecDetail = new SpecificatonDetail { Heading = specHeading, Specifiction = dbSpecifiction, Value = specDetail.Value }; dbSpecifiction.SpecificatonDetails.Add(dbSpecDetail); } product.Specifictions.Add(dbSpecifiction); } return product; }
private static void Main(string[] args) { var gsmarena = new GsmarenaOnline(); //var gsmarenaBrands = new Thread(gsmarena.PopulateBrands); //gsmarenaBrands.SetApartmentState(ApartmentState.STA); //gsmarenaBrands.Start(); List<Mobiles.DataModels.Brand> brands; using (var dbContext = new MobilesDbContext()) { brands = dbContext.Brands.ToList(); } var threads = new List<Thread>(); List<List<int>> brandIdLists = new List<List<int>>(); var maxId = brands.Max(b => b.Id); int idToRange = 0; while (idToRange <= maxId) { var idFromRange = idToRange; idToRange = idToRange + ((idToRange + 20) < maxId ? 20 : maxId - idToRange); var brandIds = (from b in brands where b.Id > idFromRange && b.Id <= idToRange select b.Id).ToList(); brandIdLists.Add(brandIds); if (idToRange == maxId) { break; } } //foreach (var ids in brandIdLists) //{ // var thread = new Thread(() => gsmarena.PopulateBrandPages(ids)) // { // Name = string.Format("BrandIds-{0}-to-{1}", ids.First(), ids.Last()) // }; // thread.SetApartmentState(ApartmentState.STA); // threads.Add(thread); //} //threads.ForEach(t => t.Start()); //////var gsmarenaBrandPages = new Thread(gsmarena.PopulateBrandPages); //////gsmarenaBrandPages.SetApartmentState(ApartmentState.STA); //////gsmarenaBrandPages.Start(); foreach (var ids in brandIdLists) { var thread = new Thread(() => gsmarena.PopulateBrandMobilePhones(ids)) { Name = string.Format("MobilesForBrandIds-{0}-to-{1}", ids.First(), ids.Last()) }; thread.SetApartmentState(ApartmentState.STA); threads.Add(thread); } threads.ForEach(t => t.Start()); //var gsmarenaProductSpecificationsOdd = new Thread(() => gsmarena.PopulateMobileSpecifications(false)); //gsmarenaProductSpecificationsOdd.SetApartmentState(ApartmentState.STA); //gsmarenaProductSpecificationsOdd.Start(); //var gsmarenaProductSpecificationsEven = new Thread(() => gsmarena.PopulateMobileSpecifications(true)); //gsmarenaProductSpecificationsEven.SetApartmentState(ApartmentState.STA); //gsmarenaProductSpecificationsEven.Start(); //var brandImages = new Thread(() => gsmarena.DownloadMobileThumbImages()); //brandImages.SetApartmentState(ApartmentState.STA); //brandImages.Start(); var mobileImages = new Thread(gsmarena.PopulateMobileImages); mobileImages.SetApartmentState(ApartmentState.STA); mobileImages.Start(); //var gsmarenaProductSpecifications = new Thread(gsmarena.PopulateUrls); //gsmarenaProductSpecifications.SetApartmentState(ApartmentState.STA); //gsmarenaProductSpecifications.Start(); //while (gsmarenaBrands.IsAlive) //{ // if (gsmarenaBrands.IsAlive==false) // { // gsmarenaBrandPages.Start(); // break; // } //} //while (gsmarenaBrandPages.IsAlive) //{ // if (gsmarenaBrandPages.IsAlive == false) // { // gsmarenaBrandPagePhones.Start(); // break; // } //} }