public void PopulateMobileSpecifications(bool evenId)
{
Settings.AttachToBrowserTimeOut = 240;
Settings.WaitUntilExistsTimeOut = 240;
Settings.WaitForCompleteTimeOut = 240;
Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>());
using (var dbContext = new MobilesDbContext())
{
try
{
List<Product> dbProducts = dbContext.Products.Where(p => p.IsRead == false && ((p.Id%2)==0)==evenId).ToList();
foreach (var dbProduct in dbProducts)
{
// KillIeProcesses();
string url = dbProduct.Url;
using (var browser = new IE(url, true))
{
//browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized);
//browser.WaitForComplete();
Div specificationsDiv = browser.Div(Find.ById("specs-list"));
var productDetails = new ProductDetails();
foreach (Table table in specificationsDiv.Tables)
{
var specification = new Specification();
Element specificationTypeCell =
table.TableRows.First().Children().Where(c => c.TagName.ToUpper() == "TH").First();
if (specificationTypeCell != null && specificationTypeCell.Exists)
{
specification.Type = specificationTypeCell.Text;
}
foreach (TableRow tableRow in table.TableRows)
{
if (tableRow.TableCells.Count > 1)
{
TableCell keyCell = tableRow.TableCells.FirstOrDefault();
TableCell valueCell = tableRow.TableCells.LastOrDefault();
if (keyCell != null && keyCell.Exists && valueCell != null && valueCell.Exists)
{
if (keyCell.Links.Count > 0)
{
var specDetail = new SpecificationDetail();
specDetail.Key = keyCell.Links.First().Text;
specDetail.Value = valueCell.Text;
specification.Details.Add(specDetail);
}
else if (specification.Details.Count > 0)
{
specification.Details.Last().Value = string.Format("{0}{1}",
specification.Details
.Last().
Value,
valueCell.Text);
}
}
}
}
productDetails.Specifications.Add(specification);
dbProduct.IsRead = true;
}
GenerateProductDetails(dbProduct,productDetails, dbContext);
dbContext.SaveChanges();
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadLine();
//throw ex;
}
}
}
public void PopulateMobileImages()
{
Settings.AttachToBrowserTimeOut = 240;
Settings.WaitUntilExistsTimeOut = 240;
Settings.WaitForCompleteTimeOut = 240;
using (var dbContext = new MobilesDbContext())
{
var products = dbContext.Products.Include("Brand").ToList();
var productImageUrls = dbContext.ProductImages.Select(p=>p.ImageUrl).ToList() ;
foreach(var product in products)
{
var productUrl = product.Url;
var productName = product.ProductName;
var brandName = product.Brand.BrandName;
var replaceString = brandName.Replace(" ", "_")+"_"+ productName.Replace(" ", "_");
var picutreUrl = productUrl.Replace(replaceString.ToLower(), replaceString.ToLower() + "-pictures");
KillIeProcesses();
using (var browser = new IE(picutreUrl, true))
{
Div pictureDiv = browser.Div(Find.ById("pictures"));
var imageUrls = pictureDiv.Images.Select(i => i.Src);
foreach(var imageUrl in imageUrls )
{
if(productImageUrls.Contains(imageUrl)==false)
{
dbContext.ProductImages.Add(
new ProductImage()
{
ImageUrl = imageUrl,
Product = product
}
);
}
}
}
dbContext.SaveChanges();
}
}
}
public void PopulateBrandPages(IEnumerable<int> brandIdList)
{
Settings.AttachToBrowserTimeOut = 240;
Settings.WaitUntilExistsTimeOut = 240;
Settings.WaitForCompleteTimeOut = 240;
using (var dbContext = new MobilesDbContext())
{
try
{
List<string> brandPagesList = dbContext.BrandPages.Select(b => b.Url).Distinct().ToList();
List<DataModels.Brand> brands = dbContext.Brands.Where(b => brandIdList.Contains(b.Id)).ToList();
// List<DataModels.Brand> brands = dbContext.Brands.ToList();
foreach (DataModels.Brand brand in brands)
{
// KillIeProcesses();
if (brandPagesList.Contains(brand.Url).Equals(false))
{
brand.BrandPages.Add(new BrandPage
{
Url = brand.Url,
IsInitialPage = true
});
}
string url = brand.Url;
using (var browser = new IE(url,true))
{
browser.ShowWindow(NativeMethods.WindowShowStyle.ForceMinimized);
browser.WaitForComplete();
Div mainDiv = browser.Div(Find.ById("main"));
if (mainDiv.Exists)
{
Div navigationDiv =
mainDiv.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("nav-pages")).
FirstOrDefault();
if (navigationDiv != null && navigationDiv.Exists)
{
List<string> pageLinks =
navigationDiv.Links.Select(l=>l.Url).Distinct().ToList();
foreach (var pageLink in pageLinks)
{
if (brandPagesList.Contains(pageLink).Equals(false))
{
brand.BrandPages.Add(new BrandPage
{
Url = pageLink
});
}
}
}
}
}
dbContext.SaveChanges();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadLine();
//throw ex;
}
}
}
public void PopulateBrands()
{
Url = "http://www.gsmarena.com/makers.php3";
using (var browser = new IE(Url,true))
{
try
{
Settings.AttachToBrowserTimeOut = 240;
Settings.WaitUntilExistsTimeOut = 240;
Settings.WaitForCompleteTimeOut = 240;
Thread.Sleep(1000);
var shop = new OnlineShoppingModel();
Div allBrandsDiv = browser.Div(Find.ById("mid-col"));
Table allBrandTable = allBrandsDiv.Tables.First();
int index = 1;
foreach (TableRow brandRow in allBrandTable.TableRows)
{
IEnumerable<Link> brandLinks = brandRow.Links.Where(l => l.Images.Count > 0);
if (brandLinks.Count() > 0)
{
foreach (Link brandLink in brandLinks)
{
Image brandImage = brandLink.Images.First();
Link textBrandlink =
brandRow.Links.Where(b => b.Url.Contains(brandLink.Url) && b.Images.Count <= 0).
First();
string brandText = textBrandlink.Text;
string numberOfProducts =
brandText.Substring(brandText.IndexOf("(")).Replace("(", "").Replace(")", "");
var brand = new Brand
{
Id = index,
Name = brandImage.Alt,
Url = brandLink.Url,
NoOfProducts = int.Parse(numberOfProducts),
ImageUrl = brandImage.Src
};
shop.Brands.Add(brand);
index++;
}
}
Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>());
using (var dbContext = new MobilesDbContext())
{
List<string> brandUrlList = dbContext.Brands.Select(c => c.Url).ToList();
foreach (Brand brand in
shop.Brands.Where(c => brandUrlList.Contains(c.Url) == false))
{
dbContext.Brands.Add(new DataModels.Brand
{
BrandName = brand.Name,
IsActive = true,
AddedDate = DateTime.Now,
LastModifiedDate = DateTime.Now,
Url = brand.Url,
NumberOfProducts = brand.NoOfProducts,
ImageUrl = brand.ImageUrl
});
}
dbContext.SaveChanges();
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadLine();
//throw ex;
}
}
}
public void PopulateBrandMobilePhones(IEnumerable<int> brandIdList)
{
Settings.AttachToBrowserTimeOut = 240;
Settings.WaitUntilExistsTimeOut = 240;
Settings.WaitForCompleteTimeOut = 240;
using (var dbContext = new MobilesDbContext())
{
try
{
List<BrandPage> brandPages =
dbContext.BrandPages.Include("Brand").Where(p => brandIdList.Contains(p.Brand.Id) && p.IsRead == false).ToList();
foreach (BrandPage brandPage in brandPages)
{
string url = brandPage.Url;
try
{
using (var browser = new IE(url, true))
{
browser.ShowWindow(NativeMethods.WindowShowStyle.Hide);
browser.WaitForComplete();
IEnumerable<Div> productsDivs =
browser.Divs.Where(d => d.ClassName != null && d.ClassName.Contains("makers"));
var listOfProducts = new List<OnlineShopModel.Product>();
foreach (Div productDiv in productsDivs)
{
List productsList = productDiv.Lists.First();
if (productsList != null && productsList.Exists)
{
foreach (ListItem productListItem in productsList.ListItems)
{
Link productLink = productListItem.Links.First();
Element productName =
productLink.Children().Where(c => c.TagName.ToUpper() == "STRONG").First();
Image productImage = productLink.Images.First();
listOfProducts.Add(new OnlineShopModel.Product
{
ProductName = productName.Text,
Url = productLink.Url,
ImageUrl = productImage.Src,
Description = productImage.Title
});
}
}
}
Database.SetInitializer(new CreateDatabaseIfNotExists<MobilesDbContext>());
List<Product> dbProductsList = dbContext.Products.ToList();
foreach (OnlineShopModel.Product product in listOfProducts)
{
Product filterProduct = dbProductsList.Where(p => p.Url == product.Url).FirstOrDefault();
if (filterProduct == null)
{
Product dbProduct = GetDbProduct(product);
dbProduct.Brand = brandPage.Brand;
dbContext.Products.Add(dbProduct);
}
}
brandPage.IsRead = true;
dbContext.SaveChanges();
}
}
catch (Exception)
{
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadLine();
//throw ex;
}
}
}
