private static void SeedCategoryMenuItems() { FoxEbookDbService foxEbookDbService = new FoxEbookDbService(); FoxEbookCrawlerService foxEbookCrawlerService = new FoxEbookCrawlerService(); var categories = foxEbookCrawlerService.GetCategoryItems(); foreach (var category in categories) { var productsCount = foxEbookCrawlerService.GetNetProductsCountUnderCategory(category); int i; if (category.PageCount > 1) { for (i = 1; i < category.PageCount; i++) { category.Pages.Add(new FoxePage() { PageNo = i, Category = category, CreatedOn = DateTime.Now, DownloadableProductsCount = category.FirstPageProductCount }); } category.Pages.Add(new FoxePage() { PageNo = i, Category = category, CreatedOn = DateTime.Now, DownloadableProductsCount = category.LastPageProductCount }); } else { category.Pages.Add(new FoxePage() { PageNo = 1, Category = category, CreatedOn = DateTime.Now, DownloadableProductsCount = category.FirstPageProductCount }); } LogManager.WriteToLog(new LogMessage() { Message = string.Format("{0} has #{1} Products under #{2} Pages", category.Name, category.ProductCount, category.PageCount) }); } var rootCategory = new FoxeCategory() { Name = "FoxeWeb", SEOFriendlyName = "foxeweb", ParentCategoryId = null, CreatedOn = DateTime.Now, Rank = 0, Weight = 0 }; rootCategory.ChildCategories = categories; foxEbookDbService.AddCategoryMenuItem(rootCategory); }
public int GetNetProductsCountUnderCategory(FoxeCategory category) { string productsCountUnderCategory_PageUrl = CrawlerHelper.GetNetProductCountUnderCategory_PageUrl() .Replace("{{CATEGORY_NAME}}", category.SEOFriendlyName) .Replace("{{PAGE_NO}}", "1"); HtmlDocument productsCountUnderCategoryPageDocument = webClientManager.DownloadHtmlDocument(productsCountUnderCategory_PageUrl); return(foxEbookParser.ParseForNetProductCountUnderCategory(productsCountUnderCategoryPageDocument, category)); }
private FoxeCategoryTreeViewItem CreateRootFoxeTreeViewNode() { //FoxeCategory rootCategory = foxeContext.CategoryItems.Where(cat => cat.ParentCategory == null).First(); var query = foxeContext.CategoryItems.Where(cat => cat.ParentCategory == null); FoxeCategory rootCategory = query.First(); foxeRootTreeViewNode = CreateExapndableCategoryTreeItem(rootCategory); foxeTreeView.Items.Add(foxeRootTreeViewNode); return(foxeRootTreeViewNode); }
private FoxeCategoryTreeViewItem CreateExapndableCategoryTreeItem(FoxeCategory foxeCategory) { FoxeCategoryTreeViewItem item = new FoxeCategoryTreeViewItem(foxeCategory); //item.Items.Add(new FoxeProcessingTreeViewItem()); if (foxeCategory.ChildCategories.Count > 1 || foxeCategory.Pages.Count >= 1) { item.Items.Add(new FoxeProcessingTreeViewItem()); } return(item); }
public void AddCategoryMenuItem(FoxeCategory categoryItem) { try { dbcontext.CategoryItems.Add(categoryItem); dbcontext.SaveChanges(); } catch (Exception e) { // LOG ERROR Console.WriteLine(e.ToString()); } }
public IList <FoxeProduct> GetProductsUnderCategory(FoxeCategory category) { IList <FoxeProduct> products = new List <FoxeProduct>(); for (int currentPageNo = 1; currentPageNo <= category.PageCount; currentPageNo++) { string productsUnderCategory_PageUrl = CrawlerHelper.GetProductsUnderCategory_PageUrl() .Replace("{{CATEGORY_NAME}}", category.SEOFriendlyName) .Replace("{{PAGE_NO}}", currentPageNo.ToString()); HtmlDocument productsCategoryPageDocument = webClientManager.DownloadHtmlDocument(productsUnderCategory_PageUrl); var pageProducts = foxEbookParser.ParseForProductsUnderCategory(productsCategoryPageDocument, category); products.AddRange(pageProducts); } return(products); }
public int ParseForNetProductCountUnderCategory(HtmlDocument document, FoxeCategory category) { var pager = document.DocumentNode.SelectNodes("//ul[@class='pagination']"); int firstPageProductCount = ParseForPageProductCountForCategory(document); int lastPageProductCount = 0; int productCount = 0; int pageCount = 0; if (pager == null) { productCount = firstPageProductCount; pageCount = 1; } else { var pagerLastAnchor = document.DocumentNode.SelectNodes("//ul[@class='pagination']//li//a[@title='Last']"); string pagerLastHref = ""; if (pagerLastAnchor != null) { pagerLastHref = pagerLastAnchor[0].Attributes["href"].Value; } else { var totalAnchors = document.DocumentNode.SelectNodes("//ul[@class='pagination']//li//a").Count; var pagerNextAnchor = document.DocumentNode.SelectNodes("//ul[@class='pagination']//li//a").Last(); pagerLastHref = pagerNextAnchor.Attributes["href"].Value; } string pattern = @"/page/(?<pageNo>\d+)/"; Match m = Regex.Match(pagerLastHref, pattern); if (m.Success) { pageCount = Int32.Parse(m.Groups["pageNo"].Value); } string lastPageUrl = CrawlerHelper.GetSchemeDomainUrl() + pagerLastHref; HtmlDocument lastPageDocument = new WebClientManager().DownloadHtmlDocument(lastPageUrl); lastPageProductCount = ParseForPageProductCountForCategory(lastPageDocument); productCount = (firstPageProductCount * (pageCount - 1)) + lastPageProductCount; } category.PageCount = pageCount; category.ProductCount = productCount; category.FirstPageProductCount = firstPageProductCount; category.LastPageProductCount = lastPageProductCount; return(productCount); }
public IList <FoxeProduct> GetProductsUnderCategory(FoxeCategory category) { return(crawler.GetProductsUnderCategory(category)); }
public int GetNetProductsCountUnderCategory(FoxeCategory category) { return(crawler.GetNetProductsCountUnderCategory(category)); }
public static bool IsLeaf(this FoxeCategory foxeCategory) { return(foxeCategory.ChildCategories.Count == 0); }
public static bool IsRoot(this FoxeCategory foxeCategory) { return(foxeCategory.ParentCategory == null); }
public IList <FoxeProduct> ParseForProductsUnderCategory(HtmlDocument document, FoxeCategory category) { IList <FoxeProduct> products = new List <FoxeProduct>(); HtmlNodeCollection productNodes = document.DocumentNode.SelectNodes("//div[@class='row book-top']"); foreach (var productnode in productNodes) { FoxeProduct product = new FoxeProduct(); product.CoverPageUrl = productnode.ChildNodes["div"].ChildNodes["img"].Attributes["src"].Value; var productDetailsPageUrl = CrawlerHelper.GetSchemeDomainUrl() + productnode.Descendants("a").Last().Attributes["href"].Value; HtmlDocument productDetailPageDocument = new WebClientManager().DownloadHtmlDocument(productDetailsPageUrl); ParseProductDetailPageUpdateProduct(productDetailPageDocument, product); product.Category = category; products.Add(product); } return(products); }
public FoxeCategoryTreeViewItem(FoxeCategory foxeCategory) { this.FoxeCategory = foxeCategory; this.Tag = foxeCategory; CreateCategoryTreeViewItem(); }
private async Task AddPagesUnderSubCategory(FoxeCategoryTreeViewItem subCategoryTreeViewItem, FoxeCategory foxeSubCategory) { var synchronizationContext = TaskScheduler.FromCurrentSynchronizationContext(); var cancellationToken = new CancellationToken(); await Task.Factory.StartNew(() => { subCategoryTreeViewItem.Items.Clear(); foreach (FoxePage foxePage in foxeSubCategory.Pages) { subCategoryTreeViewItem.Items.Add(CreateExapndablePageTreeItem(foxePage)); //var pageTreeViewItem = new FoxePageTreeViewItem(foxePage); //subCategoryTreeViewItem.Tag = foxePage; //subCategoryTreeViewItem.Items.Add(pageTreeViewItem); } }, cancellationToken, TaskCreationOptions.None, synchronizationContext); }
private async Task AddSubCategoriesUnderRootCategory(FoxeCategoryTreeViewItem rootCategoryTreeViewItem, FoxeCategory rootFoxeCategory) { var taskGetSubCategoriesForRootcategory = new Task <IList <FoxeCategory> >(() => { return(rootFoxeCategory.ChildCategories.ToList()); }); taskGetSubCategoriesForRootcategory.Start(); var foxeSubCategories = await taskGetSubCategoriesForRootcategory; rootCategoryTreeViewItem.Items.Clear(); foreach (var foxeCategory in foxeSubCategories) { rootCategoryTreeViewItem.Items.Add(CreateExapndableCategoryTreeItem(foxeCategory)); } }