public static void RemoveWrongImage(RepositoryManager _repositoryManager) { var images = _repositoryManager.ProductImageRepository.Table.ToList(); var imageFileFolder = ConfigurationManager.AppSettings["imageFolder"]; Console.WriteLine("all images:" + images.Count()); foreach (var productImage in images) { Console.WriteLine("strat image:" + productImage.ProductImageId); var imagePath = $@"{imageFileFolder}/{productImage.FileName}"; bool imageValidated = true; using (Stream stream = new FileStream(imagePath, FileMode.Open)) { imageValidated = ImageHelper.ValidateImageQualityByPixel(stream); } if (!imageValidated) { try { File.Delete(imagePath); _repositoryManager.ProductImageRepository.Delete(productImage); _repositoryManager.ProductImageRepository.Save(); Console.WriteLine("Remove Image:" + imagePath); } catch (Exception ex) { Console.WriteLine("Remove Image Error:" + ex); } } } }
public void RandomMapping() { var _repoManager = new RepositoryManager(); var tags = _repoManager.TagRepository.Table.ToList(); var products = _repoManager.ProductRepository.Table.ToList(); foreach (var product in products.AsParallel()) { if (!_repoManager.ProductTagRepository.Table.Any(m => m.ProductId == product.ProductId)) { var productTags = tags.OrderBy(m => Guid.NewGuid()).Take(5).ToList(); foreach (var productTag in productTags) { _repoManager.ProductTagRepository.Create(new ProductTag() { ProductId = product.ProductId, TagId = productTag.TagId }); } _repoManager.ProductTagRepository.Save(); Console.Write("+"); } } }
public ProductTagBuilder() { _repositoryManager = new RepositoryManager(); }
public void Mapping() { var keywordTypes = new List<KeywordType>(); var shopIds = new List<int>(); using (var db = new NCrawlerEntitiesDbServices()) { shopIds = db.ProductInfoes.Where(m => m.WebsiteId == 4).Select(m => m.ProductInfoId).ToList(); keywordTypes = db.KeywordTypes.ToList(); } var shopNameId = keywordTypes.First(m => m.Name == keywordList[0]).KeywordTypeId; var shopAddressId = keywordTypes.First(m => m.Name == keywordList[1]).KeywordTypeId; var shopTelId = keywordTypes.First(m => m.Name == keywordList[2]).KeywordTypeId; var shopOpenTimeId = keywordTypes.First(m => m.Name == keywordList[3]).KeywordTypeId; var shopCloseDayId = keywordTypes.First(m => m.Name == keywordList[4]).KeywordTypeId; var shopServiceId = keywordTypes.First(m => m.Name == keywordList[5]).KeywordTypeId; //var shopPcId = keywordTypes.First(m => m.Name == keywordList[6]).KeywordTypeId; var shopPayId = keywordTypes.First(m => m.Name == keywordList[7]).KeywordTypeId; var shopStateId = keywordTypes.First(m => m.Name == keywordList[8]).KeywordTypeId; var shopCityId = keywordTypes.First(m => m.Name == keywordList[9]).KeywordTypeId; var shopLatId = keywordTypes.First(m => m.Name == keywordList[10]).KeywordTypeId; var shopLongId = keywordTypes.First(m => m.Name == keywordList[11]).KeywordTypeId; if (shopIds.Any()) { foreach (var shopId in shopIds) { try { var _repoManager = new RepositoryManager(); using (var db = new NCrawlerEntitiesDbServices()) { var retailShop = new RetailerShop(); var keywords = db.ProductKeywords.Where(m => m.ProductId == shopId); var name = keywords.FirstOrDefault(m => m.KeywordTypeId == shopNameId); if (name != null) { retailShop.Name = name.Value; } var address = keywords.FirstOrDefault(m => m.KeywordTypeId == shopAddressId); if (address != null) { retailShop.Address = address.Value; } var tel = keywords.FirstOrDefault(m => m.KeywordTypeId == shopTelId); if (tel != null) { retailShop.Phone = tel.Value; } var openTime = keywords.FirstOrDefault(m => m.KeywordTypeId == shopOpenTimeId); if (openTime != null) { retailShop.OpenTime = openTime.Value; } var closeDay = keywords.Where(m => m.KeywordTypeId == shopCloseDayId).ToList(); if (closeDay.Any()) { retailShop.CloseDay = closeDay.Select(m => m.Value).Aggregate((m1, m2) => m1 + "," + m2); } var service = keywords.Where(m => m.KeywordTypeId == shopServiceId).ToList(); var taxFree = false; if (service.Any()) { //retailShop.Service = service.Select(m => m.Value).Aggregate((m1, m2) => m1 + "," + m2); var serviceString = service.Select(m => m.Value).Aggregate((m1, m2) => m1 + "," + m2); if (serviceString.Contains("免税")) { taxFree = true; } } var unionPay = false; var pay = keywords.Where(m => m.KeywordTypeId == shopPayId).ToList(); if (pay.Any()) { //retailShop.PayWay = pay.Select(m => m.Value).Aggregate((m1, m2) => m1 + "," + m2); var payString = pay.Select(m => m.Value).Aggregate((m1, m2) => m1 + "," + m2); if (payString.Contains("銀聯")) { unionPay = true; } } var lat = keywords.FirstOrDefault(m => m.KeywordTypeId == shopLatId); if (lat != null) { retailShop.Latitude = float.Parse(lat.Value); } var longt = keywords.FirstOrDefault(m => m.KeywordTypeId == shopLongId); if (longt != null) { retailShop.Longitude = float.Parse(longt.Value); } var state = keywords.FirstOrDefault(m => m.KeywordTypeId == shopStateId); if (state != null) { var stateI = _repoManager.StateRepository.Table.FirstOrDefault(m => m.Name == state.Value); if (stateI == null) { stateI = new State() { Name = state.Value }; _repoManager.StateRepository.Create(stateI); _repoManager.StateRepository.Save(); } //retailShop.StateId = stateI.StateId; var city = keywords.FirstOrDefault(m => m.KeywordTypeId == shopCityId); if (city != null) { var cityI = _repoManager.CityRepository.Table.FirstOrDefault(m => m.Name == city.Value); if (cityI == null) { cityI = new City() { Name = city.Value, StateId = stateI.StateId }; _repoManager.CityRepository.Create(cityI); _repoManager.CityRepository.Save(); } retailShop.CityId = cityI.CityId; } } retailShop.Enabled = true; retailShop.TaxFree = taxFree; retailShop.Unionpay = unionPay; retailShop.RetailerId = 1; _repoManager.RetailerShopRepository.Create(retailShop); _repoManager.RetailerShopRepository.Save(); //var saleCategory = keywords.Where(m => m.KeywordTypeId == shopPcId); //if (saleCategory.Any()) //{ // foreach (var sc in saleCategory) // { // var scI = _repoManager.SaleProductCategoryRepository.Table.FirstOrDefault(m => m.Name == sc.Value); // if (scI == null) // { // scI = new SaleProductCategory() { Name = sc.Value }; // _repoManager.SaleProductCategoryRepository.Create(scI); // _repoManager.SaleProductCategoryRepository.Save(); // } // if (!_repoManager.RetailShopSaleProductCategoryRepository.Table.Any(m => m.RetailShopId == retailShop.RetailShopId && m.RetailShopSaleProductCategoryId == scI.SaleProductCategoryId)) // { // _repoManager.RetailShopSaleProductCategoryRepository.Create(new RetailShopSaleProductCategory() { RetailShopId = retailShop.RetailShopId, SaleProductCategoryId = scI.SaleProductCategoryId }); // _repoManager.RetailShopSaleProductCategoryRepository.Save(); // } // } //} } } catch (Exception ex) { Console.WriteLine(ex); } Console.Write("+"); } } }
private void ProcessSingleFile(string filePath) { try { var repo = new RepositoryManager(); if (File.Exists(filePath)) { string title, corver, body, bodyHtml; var info = new HtmlDocument(); info.LoadHtml(File.ReadAllText(filePath)); var fileId = int.Parse(Path.GetFileNameWithoutExtension(filePath)); using (var db = new NCrawlerEntitiesDbServices()) { var url = db.CrawlHistory.Single(m => m.Id == fileId).Key; //if(repo.NArticleRepository.Table.Any(m=>m.SourceUrl==url)) // return; if (db.ProductInfoes.Any(m => m.Url == url)) { Console.WriteLine("Duplicate Url:" + url); return; } //Todo add article //var product = AddArticle(url); var titleNode = info.DocumentNode.SelectNodes( "//head//title"); if (titleNode != null) { title = titleNode.First().InnerText; } else { return; } var imageNode = info.DocumentNode.SelectSingleNode( "//meta[@property='og:image']"); if (imageNode != null) { corver = imageNode.Attributes["content"].Value; } else { return; } var bodyNode = info.DocumentNode.SelectSingleNode( "//article[@class='news-detail']//div[@class='body']//section//div[@class='inner']"); if (bodyNode != null) { body = bodyNode.InnerText; bodyHtml = bodyNode.InnerHtml; } else { return; } var article = new NArticle() { Body = body, BodyHtml = bodyHtml, CorverUrl = corver, CreateDate = DateTime.Now, PublicshDate = DateTime.Now, SourceUrl = url, Title = title }; //repo.NArticleRepository.Create(article); //repo.NArticleRepository.Save(); var imagesNode = info.DocumentNode.SelectNodes("//article[@class='news-detail']//div[@class='body']//img"); if (imagesNode != null && imagesNode.Any()) { foreach (var image in imagesNode) { var imageUrl = image.Attributes["src"].Value; //repo.NArticleImageRepository.Create(new NArticleImage() { NArticleId = article.NArticleId, Url = imageUrl }); } //repo.NArticleRepository.Save(); } Console.Write("+"); } } } catch (Exception ex) { Console.Write("-"); Console.WriteLine("Single product Error:" + ex); } }
public StatisticsController() { _repositoryManager = new RepositoryManager(); }
public CountriesController() { _repositoryManager = new RepositoryManager(); }
public ArticlesController() { _repositoryManager = new RepositoryManager(); }
// private readonly RetailShopHelper _retailShopHelper; #region ctor public RetailerShopsController() { _repositoryManager = new RepositoryManager(); // _retailShopHelper = new RetailShopHelper(); }
public BannersController() { _repositoryManager = new RepositoryManager(); }
public LocationsController() { _repositoryManager = new RepositoryManager(); }
// ctor public UserFavoritesController() { _repositoryManager = new RepositoryManager(); }
public AccountsController() { _repoManager = new RepositoryManager(); }
public RetailerShopsBuilder() { _repositoryManager = new RepositoryManager(); }
public RetailersController() { _repositoryManager = new RepositoryManager(); }
public BrandsController() { _repositoryManager = new RepositoryManager(); }
private void ProcessSingleFile(string filePath) { try { var repo = new RepositoryManager(); if (File.Exists(filePath)) { string title, bodyHtml; var info = new HtmlDocument(); info.LoadHtml(File.ReadAllText(filePath)); var fileId = int.Parse(Path.GetFileNameWithoutExtension(filePath)); using (var db = new NCrawlerEntitiesDbServices()) { var url = db.CrawlHistory.Single(m => m.Id == fileId).Key; //if(repo.NArticleRepository.Table.Any(m=>m.SourceUrl==url)) // return; if (db.ProductInfoes.Any(m => m.Url == url)) { Console.WriteLine("Duplicate Url:" + url); return; } //Todo add article //var product = AddArticle(url); var titleNode = info.DocumentNode.SelectNodes( "//h1[@class='title']"); if (titleNode != null) { title = titleNode.First().InnerText; } else { return; } var bodyNode = info.DocumentNode.SelectSingleNode( "//article[@class='entry']"); if (bodyNode != null) { bodyHtml = bodyNode.InnerHtml; } else { return; } var article = new Article() { Body = bodyHtml, Title = title, ArticleTypeId = 4, CreatedDate = DateTime.Now, ModifiedBy = "Admin", CreatedBy = "Admin", ModifiedDate = DateTime.Now, IsPublished = true }; repo.ArticleRepository.Create(article); repo.ArticleRepository.Save(); var articleFolder = "article"; var articleFolderS = articleFolder + "/" + article.ArticleId; var articleFolderST = articleFolderS + "/thumbnails"; Directory.CreateDirectory(articleFolderST); var imagesNode = info.DocumentNode.SelectNodes("//article[@class='entry']//img[contains(@src,'.jpg')]"); if (imagesNode != null && imagesNode.Any()) { foreach (var image in imagesNode) { try { var imageUrl = image.Attributes["src"].Value; var imageName = Guid.NewGuid(); if (imageUrl != null) { var storePath = article.ArticleId + "/" + imageName + Path.GetExtension(imageUrl); var storeTPath = article.ArticleId + "/thumbnails/" + imageName + Path.GetExtension(imageUrl); var c = new WebClient(); c.DownloadFile(imageUrl, articleFolder + "/" + storePath); c.DownloadFile(imageUrl, articleFolder + "/" + storeTPath); repo.ArticleImageRepository.Create(new ArticleImage() { ArticleId = article.ArticleId, Url = storePath, ThumbnailUrl = storeTPath, IsCoverImage = true }); } } catch (Exception ex) { Console.WriteLine(ex); throw; } } repo.ArticleImageRepository.Save(); } Console.Write("+"); } } } catch (Exception ex) { Console.Write("-"); Console.WriteLine("Single product Error:" + ex); } }
public ProductTagsController() { _repositoryManager = new RepositoryManager(); }
public BrandBuilder() { _repositoryManager = new RepositoryManager(); }
public CategoriesController() { _repositoryManager = new RepositoryManager(); }
public void Mapping() { var _repoManager = new RepositoryManager(); var punchedBarcode = new List<string>(); using (var db = new NCrawlerEntitiesDbServices()) { var productBarcode = _repoManager.ProductRepository.Table.Select(m => m.BarCode.ToString()).ToList(); var crawlBarcode = db.ProductKeywords.Where(m => m.KeywordTypeId == 1).Select(m => m.Value.Substring(0, 13)).ToList(); punchedBarcode = crawlBarcode.Intersect(productBarcode).ToList(); Console.WriteLine(punchedBarcode.Count); } foreach (var barcode in punchedBarcode) { using (var db = new NCrawlerEntitiesDbServices()) { var productId = db.ProductKeywords.First(m => m.KeywordTypeId == 1 && m.Value.Substring(0, 13) == barcode) .ProductId; var functions = db.ProductKeywords.Where(m => m.KeywordTypeId == 2 && m.ProductId == productId).ToList(); var pId = _repoManager.ProductRepository.Table.First(m => m.BarCode == barcode).ProductId; if (functions.Any()) { foreach (var f in functions) { if (f.Value.Length<20) { var tag = _repoManager.TagRepository.Table.FirstOrDefault(m => m.Name == f.Value); if (tag == null) { tag = new Tag() { Name = f.Value }; _repoManager.TagRepository.Create(tag); _repoManager.TagRepository.Save(); Console.Write("+"); } else { Console.Write("-"); } if (!_repoManager.ProductTagRepository.Table.Any(m=>m.ProductId==pId&&m.TagId==tag.TagId)) { _repoManager.ProductTagRepository.Create(new ProductTag() { ProductId = pId, TagId = tag.TagId }); _repoManager.ProductRepository.Save(); } } } } Console.WriteLine("!"); } } }