private DigikalaProduct ConvertProductDTOToEntity(DigikalaProductDTO dto) { DigikalaProduct m = new DigikalaProduct(); m.AvrageRate = dto.AvrageRate; m.Brand = dto.Brand; m.Categories = dto.Categories; m.Category = dto.Category; m.Colors = dto.Colors; m.DKP = dto.DKP; m.Features = dto.Features == null ? null : dto.Features.Select(x => new ProductFeatures { Title = x.Title, Features = x.Features }).ToList(); m.MaxRate = dto.MaxRate; m.Price = dto.Price; m.RatingItems = dto.RatingItems; m.Title = dto.Title; m.TitleEN = dto.TitleEN; m.TotalParticipantsCount = dto.TotalParticipantsCount; m.Url = dto.Url; m.Guaranteed = dto.Guaranteed; if (dto.Comments != null && dto.Comments.Count() > 0) { m.Comments = dto.Comments.Select(x => ConvertCommentDTOToEntity(x)).ToList(); } return(m); }
public DigikalaProductDTO GetFirstProductByCategory(GetFirstProductByCategoryParam param) { var query = Query <DigikalaProduct> .Where(x => x.Category == param.category && (string.IsNullOrEmpty(param.title) || x.Title.Contains(param.title)) && (string.IsNullOrEmpty(param.Brand) || x.Brand != null && x.Brand.Contains(param.Brand)) && x.Comments.Any() && x.Reserved == false && !x.isTagged); var update = Update <DigikalaProduct> .Set(p => p.Reserved, true).Set(p => p.Tagger, param.tagger); digikalaProducts.Update(query, update); System.Threading.Thread.Sleep(50); var product = digikalaProducts.FindOne(query); DigikalaProductDTO digikalaProduct = new DigikalaProductDTO(); digikalaProduct._id = product._id.ToString(); digikalaProduct.AvrageRate = product.AvrageRate; digikalaProduct.Brand = product.Brand; digikalaProduct.Categories = product.Categories; digikalaProduct.Category = product.Category; digikalaProduct.Colors = product.Colors; digikalaProduct.Comments = product.Comments.Select(x => new Models.DTO.Comment.CommentDTO { Author = x.Author, BoughtPrice = x.BoughtPrice, Color = x.Color, CommentDate = x.CommentDate, CommentDisLike = x.CommentDisLike, CommentId = x.CommentId, CommentLike = x.CommentLike, CreateDate = x.CreateDate, Id = x.Id, NegativeAspect = x.NegativeAspect, OpinionType = x.OpinionType, PageId = x.PageId, PositiveAspect = x.PositiveAspect, Purchased = x.Purchased, Review = x.Review, Seller = x.Seller, SellerLink = x.SellerLink, Title = x.Title, Size = x.Size }).ToList(); digikalaProduct.DKP = product.DKP; digikalaProduct.Features = product.Features == null ? null : product.Features.Select(x => new ProductFeaturesDTO { Title = x.Title, Features = x.Features }).ToList(); digikalaProduct.Guaranteed = product.Guaranteed; digikalaProduct.MaxRate = product.MaxRate; digikalaProduct.Price = product.Price; digikalaProduct.RatingItems = product.RatingItems; digikalaProduct.Title = product.Title; digikalaProduct.TitleEN = product.TitleEN; digikalaProduct.TotalParticipantsCount = product.TotalParticipantsCount; digikalaProduct.Url = product.Url; return(digikalaProduct); }
public void AddProduct <T>(T dto) { DigikalaProductDTO digikalaProduct = (DigikalaProductDTO)Convert.ChangeType(dto, typeof(DigikalaProductDTO)); using (DigikalaMongoDBRepository db = new DigikalaMongoDBRepository()) { db.AddDigikalaProduct(ConvertProductDTOToEntity(digikalaProduct)); } }
private void btnNext_Click(object sender, RoutedEventArgs e) { if (opinions != null && opinions.Count() > 0 && !string.IsNullOrEmpty(txtSelectReview.Text)) { sentences.Add(new sentence { id = sentenceId, Text = txtSelectReview.Text.Trim(), Opinions = opinions }); opinions.Clear(); } using (IWebsiteCrawler digikala = new DigikalaHelper()) { if (sentences != null && sentences.Count() > 0) { if (review.sentences == null) { review.sentences = new List <sentence>(); } review.sentences.AddRange(sentences); sentences.Clear(); } if (review != null && review.sentences != null && review.sentences.Count() > 0) { review.CreateDate = DateTime.Now; //review._id = ObjectId.GenerateNewId(DateTime.Now); review.rid = digikalaProduct.DKP; AddReviewToDBParam param = new AddReviewToDBParam { review = review, id = digikalaProduct._id, tagger = user.Username }; AddReviewToDBResponse resultAddReview = new AddReviewToDBResponse() { Success = false }; //bool resultAddReview = digikala.AddReviewToDB(param); // ☺ Api using (var Api = new WebAppApiCall()) { resultAddReview = Api.GetFromApi <AddReviewToDBResponse>("AddReviewtodb", param); } if (resultAddReview.Success) { sentences.Clear(); sentenceIdReset(); review = new ReviewDTO(); } else { MessageBox.Show("ثبت با مشکل روبرو شده است دوباره سعی کنید.", "Warning"); return; } } GetFirstProductByCategoryParam getProductParam = new GetFirstProductByCategoryParam { category = user.Category, title = user.Title, Brand = user.Brand, tagger = user.Username }; // ☺ Api using (var Api = new WebAppApiCall()) { digikalaProduct = Api.GetFromApi <DigikalaProductDTO>("GetFirstProductByCategory", getProductParam); } //digikalaProduct = digikala.GetFirstProductByCategory<DigikalaProductDTO>(getProductParam).Result; commentCurrentIndex = -1; commentCount = digikalaProduct.Comments.Count(); nextReview(); txtProductId.Text = digikalaProduct.DKP.ToString(); txtTitle.Text = digikalaProduct.Title; txtEnTitle.Text = digikalaProduct.TitleEN; review.ProductID = digikalaProduct.DKP; } }
private DigikalaProductDTO commentConcat(DigikalaProductDTO dto, string url) { int? cmPageCount = (int?)null; int DKP = getDKPWithUrl(url); string firstCmUrl = GetReviewUrl(DKP); var firstCmPage = GetPage(firstCmUrl).Result; var doc = new HtmlDocument(); doc.LoadHtml(firstCmPage); var rate = doc.DocumentNode.SelectNodes("//h2[@class='c-comments__headline']//span//span"); if (rate != null) { using (HtmlHelper html = new HtmlHelper()) { dto.AvrageRate = int.Parse(html.NumberEN(rate[2].InnerText.Trim())); dto.MaxRate = int.Parse(html.NumberEN(rate[3].InnerText.Replace("/", "").Trim())); dto.TotalParticipantsCount = int.Parse(html.NumberEN(rate[4].InnerText.Replace("(", "").Replace(")", "").Replace("\n", "").Replace("نفر", "").Trim())); } } if (doc.DocumentNode.SelectSingleNode("//div[@class='c-comments__summary']//div[@class='c-comments__summary-box']") != null) { var _ratingItems = doc.DocumentNode .SelectSingleNode("//div[@class='c-comments__summary']" + "//div[@class='c-comments__summary-box']" + "//ul[@class='c-comments__item-rating']").ChildNodes.ToArray().Where(x => x.Name.Contains("li")); List <string[]> ratingItems = new List <string[]>(); foreach (var item in _ratingItems) { var hasCell = item.SelectSingleNode("//div[@class='cell']") != null; var cell = item.SelectSingleNode("//div[@class='cell']").InnerHtml.Replace(" ", " ").Trim(); if (hasCell) { var docCell = new HtmlDocument(); docCell.LoadHtml(item.InnerHtml.Replace("\n", "").Replace(" ", " ").Trim()); string[] _rate = new string[3]; _rate[0] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']").InnerText.Replace(":", "").Replace("\n", "").Trim(); _rate[1] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']//div[@class='c-rating c-rating--general js-rating']").Attributes["data-rate-digit"].Value.Replace(":", "").Replace("\n", "").Trim(); _rate[2] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']//div[@class='c-rating c-rating--general js-rating']//div[@class='c-rating__rate js-rating-value']").Attributes["data-rate-value"].Value.Replace("%", "").Replace("\n", "").Trim(); ratingItems.Add(_rate); } } dto.RatingItems = ratingItems; } if (doc.GetElementbyId("comment-pagination") != null && doc.GetElementbyId("comment-pagination").SelectNodes("//ul[@class='c-pager__items']//li[@class='js-pagination-item']") != null) { cmPageCount = Int16.Parse(doc.GetElementbyId("comment-pagination").SelectNodes("//ul[@class='c-pager__items']//li[@class='js-pagination-item']").LastOrDefault().SelectNodes("//a[@class='c-pager__next']").Select(x => x.Attributes["data-page"].Value.ToString()).FirstOrDefault()); } else if (doc.GetElementbyId("product-comment-list") != null && doc.GetElementbyId("product-comment-list").SelectNodes("//ul[@class='c-comments__list']") != null) { cmPageCount = 1; } if (cmPageCount.HasValue) { dto.Comments = GetComments(url, cmPageCount.Value); } return(dto); }
public async Task <T> GetProduct <T>(string url) { try { string content = string.Empty; var res = client.GetHttp(url, true, user_agent); if (res.Success) { content = res.Content; } if (res.HttpStatusCode == (int)HttpStatusCode.NotFound) { using (DigikalaMongoDBRepository db = new DigikalaMongoDBRepository()) { db.RemoveBasePage(url); } } DigikalaProductDTO dto = new DigikalaProductDTO(); if (string.IsNullOrEmpty(content)) { System.Threading.Thread.Sleep(200); content = await GetPage1(url); } if (string.IsNullOrEmpty(content)) { System.Threading.Thread.Sleep(400); content = await GetPage(url); } if (string.IsNullOrEmpty(content)) { return((T)Convert.ChangeType(null, typeof(DigikalaProductDTO))); } dto.Url = url; dto.DKP = getDKPWithUrl(url); var doc = new HtmlDocument(); doc.LoadHtml(content); var body = doc.DocumentNode.SelectSingleNode("//body"); var main = body.SelectSingleNode("//main"); #region Container var divContainer = main.SelectSingleNode("//div[@id='content']//div[@class='o-page c-product-page']//div[@class='container']"); var _cats = divContainer.SelectNodes("//div[@class='c-product__nav-container']//nav//ul//li[@property='itemListElement']"); dto.Categories = _cats.Any() && _cats.Count() > 0 ? _cats.Select(x => x.InnerText).ToList() : new List <string>(); //// dto.Title = divContainer.SelectNodes("//div[@class='c-product__nav-container']//nav//ul//li//span[@property='name']").Last().InnerText; var article_info = divContainer.SelectSingleNode("//article//section[@class='c-product__info']"); var title_fa_1 = article_info.SelectSingleNode("//div[@class='c-product__headline']//h1[@class='c-product__title']").ChildNodes["#Text"].InnerText.Replace("\n", "").Trim(); var enTitle = article_info.SelectSingleNode("//div[@class='c-product__headline']//h1[@class='c-product__title']//span[@class='c-product__title-en']"); if (enTitle != null) { dto.TitleEN = enTitle.InnerHtml.Replace("\n", "").Trim(); } var product__guaranteed = article_info.SelectSingleNode("//div[@class='c-product__headline']//div[@class='c-product__guaranteed']//span"); if (product__guaranteed != null) { using (HtmlHelper html = new HtmlHelper()) { dto.Guaranteed = short.Parse(html.NumberEN(product__guaranteed.InnerText.Replace("\n", "").Replace(" ", "").Trim().Replace("بیش از", "").Replace("نفر از خریداران این محصول را پیشنهاد دادهاند", "").Trim())); } } var productWrapper = article_info.SelectSingleNode("//div[@class='c-product__attributes js-product-attributes']//div[@class='c-product__config']//div[@class='c-product__config-wrapper']"); string brandElementQuery1 = "//div[@class='c-product__directory']//ul//li//a[@class='btn-link-spoiler product-brand-title']"; string brandElementQuery2 = "//div[@class='c-product__directory']//ul//li//span[@class='product-brand-title']"; dto.Brand = productWrapper.SelectSingleNode(brandElementQuery1) != null?productWrapper.SelectSingleNode(brandElementQuery1).InnerText : (productWrapper.SelectSingleNode(brandElementQuery2) != null ? productWrapper.SelectSingleNode(brandElementQuery2).InnerText : ""); dto.Category = productWrapper.SelectSingleNode("//div[@class='c-product__directory']//ul//li//a[@class='btn-link-spoiler']").InnerText; List <string> colors = new List <string>(); bool isColors = productWrapper.SelectNodes("//div[@class='c-product__variants']") != null; if (isColors) { colors.AddRange(productWrapper.SelectNodes("//div[@class='c-product__variants']//ul//li").Select(x => x.InnerText).ToList()); dto.Colors = colors; } if (productWrapper.SelectNodes("//div[@class='c-product__params js-is-expandable']//ul//li") != null) { var feature_list = productWrapper.SelectNodes("//div[@class='c-product__params js-is-expandable']//ul//li").Select(x => new { name = x.FirstChild.InnerText.Replace(":", "").Trim(), val = x.LastChild.InnerText.Replace("\n", "").Trim() }).ToList(); } var c_box = article_info.SelectSingleNode("//div[@class='c-product__attributes js-product-attributes']//div[@class='c-product__summary js-product-summary']//div[@class='c-box']"); string priceOffQuery = "//div[@class='c-product__seller-info js-seller-info']" + "//div[@class='js-seller-info-changable c-product__seller-box']" + "//div[@class='c-product__seller-row c-product__seller-row--price']" + "//div[@class='c-product__seller-price-real']"; string priceQuery = "//div[@class='c-product__seller-info js-seller-info']" + "//div[@class='js-seller-info-changable c-product__seller-box']" + "//div[@class='c-product__seller-row c-product__seller-row--price']" + "//div[@class='c-product__seller-price-prev js-rrp-price u-hidden']"; var priceElement = article_info.SelectSingleNode(priceQuery); var isExistPrice = priceElement != null; if (isExistPrice && priceElement.InnerText.Trim() != "") { using (HtmlHelper html = new HtmlHelper()) { dto.Price = Int64.Parse(html.NumberEN(article_info.SelectSingleNode(priceQuery).InnerText.Replace("\n", "").Replace(",", "").Trim())); } } else { var priceOff = article_info.SelectSingleNode(priceOffQuery); if (priceOff != null) { using (HtmlHelper html = new HtmlHelper()) { dto.Price = Int64.Parse(html.NumberEN(priceOff.InnerText .Replace("\n", "").Replace("\r", "").Replace(",", "").Replace("تومان", "").Replace(" ", " ").Trim())); } } } #endregion #region Tabs string tabsQuery = "//div[@id='tabs']" + "//div[@class='c-box c-box--tabs p-tabs__content']" + "//div[@class='c-params']" + "//article" + "//section"; if (main.SelectNodes(tabsQuery) != null) { var tabSections = main.SelectNodes(tabsQuery).ToArray(); List <ProductFeaturesDTO> features = new List <ProductFeaturesDTO>(); foreach (var feat in tabSections) { ProductFeaturesDTO p = new ProductFeaturesDTO(); p.Title = feat.ChildNodes[0].InnerText; var _features = feat.ChildNodes[1].ChildNodes .Select(x => new string[] { GetFeatureKey(x.ChildNodes[0].InnerHtml), x.ChildNodes[1].InnerText.Replace("\n", "").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Trim(), }).ToArray(); int _lastFillNumber = 0; int _TotalCount = _features.Count(x => x[0] != ""); string[][] _tempFeature = new string[_features.Count()][]; for (int i = 0; i < _features.Length; i++) { string currentKey = _features[i][0]; if (!string.IsNullOrEmpty(currentKey) || i == 0) { _lastFillNumber = i; _tempFeature[i] = new string[] { currentKey, _features[i][1] }; } else { List <string> _featuePlus = _tempFeature[_lastFillNumber].ToList(); _featuePlus.Add(_features[i][1]); _tempFeature[_lastFillNumber] = _featuePlus.ToArray(); } } p.Features = _tempFeature.Where(x => x != null).ToList(); features.Add(p); } dto.Features = features; } #endregion dto = commentConcat(dto, url); //var jjj = JsonConvert.SerializeObject(dto); return((T)Convert.ChangeType(dto, typeof(DigikalaProductDTO))); } catch (Exception ex) { LogDTO log = new LogDTO() { _id = ObjectId.GenerateNewId().ToString(), DateTime = DateTime.Now, Description = ex.Message.ToString(), ProjectId = (int)ProjectNames.Services, Url = url, MethodName = "Digikala - Digikala Helper - GetProduct", Title = "GetProduct" }; Logger.AddLog(log); return((T)Convert.ChangeType(null, typeof(DigikalaProductDTO))); } }
private async static void digikala_7_AddProductToMongo() { for (short i = 0; i < 120; i++) { List <DigikalaPageBaseDTO> getAll = new List <DigikalaPageBaseDTO>(); List <BasePage> pages = new List <BasePage>(); using (IWebsiteCrawler digikala = new DigikalaHelper()) { getAll = (await digikala.GetAllBasePage <GetAllBasePageDigikalaResult>()).BasePages; Console.WriteLine($"list total {getAll.Count()}"); pages = getAll.Select(x => new BasePage { Id = x._id, Loc = x.Loc }).ToList(); getAll.Clear(); } long x = 0; short errorCount = 0; foreach (var item in pages) { try { var _s = DateTime.Now; DigikalaProductDTO product = null; using (IWebsiteCrawler digikala = new DigikalaHelper()) { product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc); } if (product == null) { int dkp = getDKPWithUrl(item.Loc); Console.WriteLine($"Try Again , DKP - {dkp} Wait: {1000} Secs"); System.Threading.Thread.Sleep(1000 * errorCount); using (IWebsiteCrawler digikala = new DigikalaHelper()) { product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc); } } var _t = Math.Round((DateTime.Now - _s).TotalSeconds, 2); if (product != null) { ProductTemp prd = new ProductTemp(); prd.BasePage = item; prd.DigikalaProduct = product; digikala_SaveProductBatch(prd); Console.WriteLine($"S{i}, {++x} = DKP-{product.DKP} , Comment={(product.Comments != null ? product.Comments.Count() + "+ " : "0 ")} , in {_t} Secs "); if (x % 5 == 0) { Console.WriteLine("--------------"); System.Threading.Thread.Sleep(100); } if (x % 100 == 0) { System.Threading.Thread.Sleep(500); Console.Clear(); } errorCount = 0; } else { if (errorCount < 3) { errorCount += 1; } int dkp = getDKPWithUrl(item.Loc); Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** ,"); System.Threading.Thread.Sleep(1000 * errorCount); } } catch (Exception ex) { if (errorCount < 3) { errorCount += 1; } int dkp = getDKPWithUrl(item.Loc); Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** Problem"); using (ILoger Logger = new MongoDBLoggerHelper()) { LogDTO log = new LogDTO() { _id = ObjectId.GenerateNewId().ToString(), DateTime = DateTime.Now, Description = ex.Message.ToString(), ProjectId = (int)ProjectNames.Console, Url = item.Loc, MethodName = "Digikala - Console App", Title = "Get Product Error - " + dkp }; Logger.AddLog(log); } System.Threading.Thread.Sleep(1000 * errorCount); } } } digikala_SaveProductBatch(null, true); }