private DigikalaProduct ConvertProductDTOToEntity(DigikalaProductDTO dto)
{
DigikalaProduct m = new DigikalaProduct();
m.AvrageRate = dto.AvrageRate;
m.Brand = dto.Brand;
m.Categories = dto.Categories;
m.Category = dto.Category;
m.Colors = dto.Colors;
m.DKP = dto.DKP;
m.Features = dto.Features == null ? null : dto.Features.Select(x => new ProductFeatures {
Title = x.Title, Features = x.Features
}).ToList();
m.MaxRate = dto.MaxRate;
m.Price = dto.Price;
m.RatingItems = dto.RatingItems;
m.Title = dto.Title;
m.TitleEN = dto.TitleEN;
m.TotalParticipantsCount = dto.TotalParticipantsCount;
m.Url = dto.Url;
m.Guaranteed = dto.Guaranteed;
if (dto.Comments != null && dto.Comments.Count() > 0)
{
m.Comments = dto.Comments.Select(x => ConvertCommentDTOToEntity(x)).ToList();
}
return(m);
}
public DigikalaProductDTO GetFirstProductByCategory(GetFirstProductByCategoryParam param)
{
var query = Query <DigikalaProduct> .Where(x => x.Category == param.category &&
(string.IsNullOrEmpty(param.title) || x.Title.Contains(param.title)) &&
(string.IsNullOrEmpty(param.Brand) || x.Brand != null && x.Brand.Contains(param.Brand)) &&
x.Comments.Any() &&
x.Reserved == false &&
!x.isTagged);
var update = Update <DigikalaProduct> .Set(p => p.Reserved, true).Set(p => p.Tagger, param.tagger);
digikalaProducts.Update(query, update);
System.Threading.Thread.Sleep(50);
var product = digikalaProducts.FindOne(query);
DigikalaProductDTO digikalaProduct = new DigikalaProductDTO();
digikalaProduct._id = product._id.ToString();
digikalaProduct.AvrageRate = product.AvrageRate;
digikalaProduct.Brand = product.Brand;
digikalaProduct.Categories = product.Categories;
digikalaProduct.Category = product.Category;
digikalaProduct.Colors = product.Colors;
digikalaProduct.Comments = product.Comments.Select(x => new Models.DTO.Comment.CommentDTO
{
Author = x.Author,
BoughtPrice = x.BoughtPrice,
Color = x.Color,
CommentDate = x.CommentDate,
CommentDisLike = x.CommentDisLike,
CommentId = x.CommentId,
CommentLike = x.CommentLike,
CreateDate = x.CreateDate,
Id = x.Id,
NegativeAspect = x.NegativeAspect,
OpinionType = x.OpinionType,
PageId = x.PageId,
PositiveAspect = x.PositiveAspect,
Purchased = x.Purchased,
Review = x.Review,
Seller = x.Seller,
SellerLink = x.SellerLink,
Title = x.Title,
Size = x.Size
}).ToList();
digikalaProduct.DKP = product.DKP;
digikalaProduct.Features = product.Features == null ? null : product.Features.Select(x => new ProductFeaturesDTO {
Title = x.Title, Features = x.Features
}).ToList();
digikalaProduct.Guaranteed = product.Guaranteed;
digikalaProduct.MaxRate = product.MaxRate;
digikalaProduct.Price = product.Price;
digikalaProduct.RatingItems = product.RatingItems;
digikalaProduct.Title = product.Title;
digikalaProduct.TitleEN = product.TitleEN;
digikalaProduct.TotalParticipantsCount = product.TotalParticipantsCount;
digikalaProduct.Url = product.Url;
return(digikalaProduct);
}
public void AddProduct <T>(T dto)
{
DigikalaProductDTO digikalaProduct = (DigikalaProductDTO)Convert.ChangeType(dto, typeof(DigikalaProductDTO));
using (DigikalaMongoDBRepository db = new DigikalaMongoDBRepository())
{
db.AddDigikalaProduct(ConvertProductDTOToEntity(digikalaProduct));
}
}
private void btnNext_Click(object sender, RoutedEventArgs e)
{
if (opinions != null && opinions.Count() > 0 && !string.IsNullOrEmpty(txtSelectReview.Text))
{
sentences.Add(new sentence
{
id = sentenceId,
Text = txtSelectReview.Text.Trim(),
Opinions = opinions
});
opinions.Clear();
}
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
if (sentences != null && sentences.Count() > 0)
{
if (review.sentences == null)
{
review.sentences = new List <sentence>();
}
review.sentences.AddRange(sentences);
sentences.Clear();
}
if (review != null && review.sentences != null && review.sentences.Count() > 0)
{
review.CreateDate = DateTime.Now;
//review._id = ObjectId.GenerateNewId(DateTime.Now);
review.rid = digikalaProduct.DKP;
AddReviewToDBParam param = new AddReviewToDBParam
{
review = review,
id = digikalaProduct._id,
tagger = user.Username
};
AddReviewToDBResponse resultAddReview = new AddReviewToDBResponse()
{
Success = false
};
//bool resultAddReview = digikala.AddReviewToDB(param); // ☺ Api
using (var Api = new WebAppApiCall())
{
resultAddReview = Api.GetFromApi <AddReviewToDBResponse>("AddReviewtodb", param);
}
if (resultAddReview.Success)
{
sentences.Clear();
sentenceIdReset();
review = new ReviewDTO();
}
else
{
MessageBox.Show("ثبت با مشکل روبرو شده است دوباره سعی کنید.", "Warning");
return;
}
}
GetFirstProductByCategoryParam getProductParam = new GetFirstProductByCategoryParam
{
category = user.Category,
title = user.Title,
Brand = user.Brand,
tagger = user.Username
};
// ☺ Api
using (var Api = new WebAppApiCall())
{
digikalaProduct = Api.GetFromApi <DigikalaProductDTO>("GetFirstProductByCategory", getProductParam);
}
//digikalaProduct = digikala.GetFirstProductByCategory<DigikalaProductDTO>(getProductParam).Result;
commentCurrentIndex = -1;
commentCount = digikalaProduct.Comments.Count();
nextReview();
txtProductId.Text = digikalaProduct.DKP.ToString();
txtTitle.Text = digikalaProduct.Title;
txtEnTitle.Text = digikalaProduct.TitleEN;
review.ProductID = digikalaProduct.DKP;
}
}
private DigikalaProductDTO commentConcat(DigikalaProductDTO dto, string url)
{
int? cmPageCount = (int?)null;
int DKP = getDKPWithUrl(url);
string firstCmUrl = GetReviewUrl(DKP);
var firstCmPage = GetPage(firstCmUrl).Result;
var doc = new HtmlDocument();
doc.LoadHtml(firstCmPage);
var rate = doc.DocumentNode.SelectNodes("//h2[@class='c-comments__headline']//span//span");
if (rate != null)
{
using (HtmlHelper html = new HtmlHelper())
{
dto.AvrageRate = int.Parse(html.NumberEN(rate[2].InnerText.Trim()));
dto.MaxRate = int.Parse(html.NumberEN(rate[3].InnerText.Replace("/", "").Trim()));
dto.TotalParticipantsCount = int.Parse(html.NumberEN(rate[4].InnerText.Replace("(", "").Replace(")", "").Replace("\n", "").Replace("نفر", "").Trim()));
}
}
if (doc.DocumentNode.SelectSingleNode("//div[@class='c-comments__summary']//div[@class='c-comments__summary-box']") != null)
{
var _ratingItems = doc.DocumentNode
.SelectSingleNode("//div[@class='c-comments__summary']" +
"//div[@class='c-comments__summary-box']" +
"//ul[@class='c-comments__item-rating']").ChildNodes.ToArray().Where(x => x.Name.Contains("li"));
List <string[]> ratingItems = new List <string[]>();
foreach (var item in _ratingItems)
{
var hasCell = item.SelectSingleNode("//div[@class='cell']") != null;
var cell = item.SelectSingleNode("//div[@class='cell']").InnerHtml.Replace(" ", " ").Trim();
if (hasCell)
{
var docCell = new HtmlDocument();
docCell.LoadHtml(item.InnerHtml.Replace("\n", "").Replace(" ", " ").Trim());
string[] _rate = new string[3];
_rate[0] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']").InnerText.Replace(":", "").Replace("\n", "").Trim();
_rate[1] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']//div[@class='c-rating c-rating--general js-rating']").Attributes["data-rate-digit"].Value.Replace(":", "").Replace("\n", "").Trim();
_rate[2] = docCell.DocumentNode.SelectSingleNode("//div[@class='cell']//div[@class='c-rating c-rating--general js-rating']//div[@class='c-rating__rate js-rating-value']").Attributes["data-rate-value"].Value.Replace("%", "").Replace("\n", "").Trim();
ratingItems.Add(_rate);
}
}
dto.RatingItems = ratingItems;
}
if (doc.GetElementbyId("comment-pagination") != null && doc.GetElementbyId("comment-pagination").SelectNodes("//ul[@class='c-pager__items']//li[@class='js-pagination-item']") != null)
{
cmPageCount = Int16.Parse(doc.GetElementbyId("comment-pagination").SelectNodes("//ul[@class='c-pager__items']//li[@class='js-pagination-item']").LastOrDefault().SelectNodes("//a[@class='c-pager__next']").Select(x => x.Attributes["data-page"].Value.ToString()).FirstOrDefault());
}
else if (doc.GetElementbyId("product-comment-list") != null && doc.GetElementbyId("product-comment-list").SelectNodes("//ul[@class='c-comments__list']") != null)
{
cmPageCount = 1;
}
if (cmPageCount.HasValue)
{
dto.Comments = GetComments(url, cmPageCount.Value);
}
return(dto);
}
public async Task <T> GetProduct <T>(string url)
{
try
{
string content = string.Empty;
var res = client.GetHttp(url, true, user_agent);
if (res.Success)
{
content = res.Content;
}
if (res.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
using (DigikalaMongoDBRepository db = new DigikalaMongoDBRepository())
{
db.RemoveBasePage(url);
}
}
DigikalaProductDTO dto = new DigikalaProductDTO();
if (string.IsNullOrEmpty(content))
{
System.Threading.Thread.Sleep(200);
content = await GetPage1(url);
}
if (string.IsNullOrEmpty(content))
{
System.Threading.Thread.Sleep(400);
content = await GetPage(url);
}
if (string.IsNullOrEmpty(content))
{
return((T)Convert.ChangeType(null, typeof(DigikalaProductDTO)));
}
dto.Url = url;
dto.DKP = getDKPWithUrl(url);
var doc = new HtmlDocument();
doc.LoadHtml(content);
var body = doc.DocumentNode.SelectSingleNode("//body");
var main = body.SelectSingleNode("//main");
#region Container
var divContainer = main.SelectSingleNode("//div[@id='content']//div[@class='o-page c-product-page']//div[@class='container']");
var _cats = divContainer.SelectNodes("//div[@class='c-product__nav-container']//nav//ul//li[@property='itemListElement']");
dto.Categories = _cats.Any() && _cats.Count() > 0 ? _cats.Select(x => x.InnerText).ToList() : new List <string>();
////
dto.Title = divContainer.SelectNodes("//div[@class='c-product__nav-container']//nav//ul//li//span[@property='name']").Last().InnerText;
var article_info = divContainer.SelectSingleNode("//article//section[@class='c-product__info']");
var title_fa_1 = article_info.SelectSingleNode("//div[@class='c-product__headline']//h1[@class='c-product__title']").ChildNodes["#Text"].InnerText.Replace("\n", "").Trim();
var enTitle = article_info.SelectSingleNode("//div[@class='c-product__headline']//h1[@class='c-product__title']//span[@class='c-product__title-en']");
if (enTitle != null)
{
dto.TitleEN = enTitle.InnerHtml.Replace("\n", "").Trim();
}
var product__guaranteed = article_info.SelectSingleNode("//div[@class='c-product__headline']//div[@class='c-product__guaranteed']//span");
if (product__guaranteed != null)
{
using (HtmlHelper html = new HtmlHelper())
{
dto.Guaranteed = short.Parse(html.NumberEN(product__guaranteed.InnerText.Replace("\n", "").Replace(" ", "").Trim().Replace("بیش از", "").Replace("نفر از خریداران این محصول را پیشنهاد دادهاند", "").Trim()));
}
}
var productWrapper = article_info.SelectSingleNode("//div[@class='c-product__attributes js-product-attributes']//div[@class='c-product__config']//div[@class='c-product__config-wrapper']");
string brandElementQuery1 = "//div[@class='c-product__directory']//ul//li//a[@class='btn-link-spoiler product-brand-title']";
string brandElementQuery2 = "//div[@class='c-product__directory']//ul//li//span[@class='product-brand-title']";
dto.Brand = productWrapper.SelectSingleNode(brandElementQuery1) != null?productWrapper.SelectSingleNode(brandElementQuery1).InnerText : (productWrapper.SelectSingleNode(brandElementQuery2) != null ? productWrapper.SelectSingleNode(brandElementQuery2).InnerText : "");
dto.Category = productWrapper.SelectSingleNode("//div[@class='c-product__directory']//ul//li//a[@class='btn-link-spoiler']").InnerText;
List <string> colors = new List <string>();
bool isColors = productWrapper.SelectNodes("//div[@class='c-product__variants']") != null;
if (isColors)
{
colors.AddRange(productWrapper.SelectNodes("//div[@class='c-product__variants']//ul//li").Select(x => x.InnerText).ToList());
dto.Colors = colors;
}
if (productWrapper.SelectNodes("//div[@class='c-product__params js-is-expandable']//ul//li") != null)
{
var feature_list = productWrapper.SelectNodes("//div[@class='c-product__params js-is-expandable']//ul//li").Select(x => new { name = x.FirstChild.InnerText.Replace(":", "").Trim(), val = x.LastChild.InnerText.Replace("\n", "").Trim() }).ToList();
}
var c_box = article_info.SelectSingleNode("//div[@class='c-product__attributes js-product-attributes']//div[@class='c-product__summary js-product-summary']//div[@class='c-box']");
string priceOffQuery = "//div[@class='c-product__seller-info js-seller-info']" +
"//div[@class='js-seller-info-changable c-product__seller-box']" +
"//div[@class='c-product__seller-row c-product__seller-row--price']" +
"//div[@class='c-product__seller-price-real']";
string priceQuery = "//div[@class='c-product__seller-info js-seller-info']" +
"//div[@class='js-seller-info-changable c-product__seller-box']" +
"//div[@class='c-product__seller-row c-product__seller-row--price']" +
"//div[@class='c-product__seller-price-prev js-rrp-price u-hidden']";
var priceElement = article_info.SelectSingleNode(priceQuery);
var isExistPrice = priceElement != null;
if (isExistPrice && priceElement.InnerText.Trim() != "")
{
using (HtmlHelper html = new HtmlHelper())
{
dto.Price = Int64.Parse(html.NumberEN(article_info.SelectSingleNode(priceQuery).InnerText.Replace("\n", "").Replace(",", "").Trim()));
}
}
else
{
var priceOff = article_info.SelectSingleNode(priceOffQuery);
if (priceOff != null)
{
using (HtmlHelper html = new HtmlHelper())
{
dto.Price = Int64.Parse(html.NumberEN(priceOff.InnerText
.Replace("\n", "").Replace("\r", "").Replace(",", "").Replace("تومان", "").Replace(" ", " ").Trim()));
}
}
}
#endregion
#region Tabs
string tabsQuery =
"//div[@id='tabs']" +
"//div[@class='c-box c-box--tabs p-tabs__content']" +
"//div[@class='c-params']" +
"//article" +
"//section";
if (main.SelectNodes(tabsQuery) != null)
{
var tabSections = main.SelectNodes(tabsQuery).ToArray();
List <ProductFeaturesDTO> features = new List <ProductFeaturesDTO>();
foreach (var feat in tabSections)
{
ProductFeaturesDTO p = new ProductFeaturesDTO();
p.Title = feat.ChildNodes[0].InnerText;
var _features = feat.ChildNodes[1].ChildNodes
.Select(x => new string[]
{
GetFeatureKey(x.ChildNodes[0].InnerHtml),
x.ChildNodes[1].InnerText.Replace("\n", "").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Replace(" ", " ").Trim(),
}).ToArray();
int _lastFillNumber = 0;
int _TotalCount = _features.Count(x => x[0] != "");
string[][] _tempFeature = new string[_features.Count()][];
for (int i = 0; i < _features.Length; i++)
{
string currentKey = _features[i][0];
if (!string.IsNullOrEmpty(currentKey) || i == 0)
{
_lastFillNumber = i;
_tempFeature[i] = new string[] { currentKey, _features[i][1] };
}
else
{
List <string> _featuePlus = _tempFeature[_lastFillNumber].ToList();
_featuePlus.Add(_features[i][1]);
_tempFeature[_lastFillNumber] = _featuePlus.ToArray();
}
}
p.Features = _tempFeature.Where(x => x != null).ToList();
features.Add(p);
}
dto.Features = features;
}
#endregion
dto = commentConcat(dto, url);
//var jjj = JsonConvert.SerializeObject(dto);
return((T)Convert.ChangeType(dto, typeof(DigikalaProductDTO)));
}
catch (Exception ex)
{
LogDTO log = new LogDTO()
{
_id = ObjectId.GenerateNewId().ToString(),
DateTime = DateTime.Now,
Description = ex.Message.ToString(),
ProjectId = (int)ProjectNames.Services,
Url = url,
MethodName = "Digikala - Digikala Helper - GetProduct",
Title = "GetProduct"
};
Logger.AddLog(log);
return((T)Convert.ChangeType(null, typeof(DigikalaProductDTO)));
}
}
private async static void digikala_7_AddProductToMongo()
{
for (short i = 0; i < 120; i++)
{
List <DigikalaPageBaseDTO> getAll = new List <DigikalaPageBaseDTO>();
List <BasePage> pages = new List <BasePage>();
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
getAll = (await digikala.GetAllBasePage <GetAllBasePageDigikalaResult>()).BasePages;
Console.WriteLine($"list total {getAll.Count()}");
pages = getAll.Select(x => new BasePage
{
Id = x._id,
Loc = x.Loc
}).ToList();
getAll.Clear();
}
long x = 0;
short errorCount = 0;
foreach (var item in pages)
{
try
{
var _s = DateTime.Now;
DigikalaProductDTO product = null;
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc);
}
if (product == null)
{
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"Try Again , DKP - {dkp} Wait: {1000} Secs");
System.Threading.Thread.Sleep(1000 * errorCount);
using (IWebsiteCrawler digikala = new DigikalaHelper())
{
product = await digikala.GetProduct <DigikalaProductDTO>(item.Loc);
}
}
var _t = Math.Round((DateTime.Now - _s).TotalSeconds, 2);
if (product != null)
{
ProductTemp prd = new ProductTemp();
prd.BasePage = item;
prd.DigikalaProduct = product;
digikala_SaveProductBatch(prd);
Console.WriteLine($"S{i}, {++x} = DKP-{product.DKP} , Comment={(product.Comments != null ? product.Comments.Count() + "+ " : "0 ")} , in {_t} Secs ");
if (x % 5 == 0)
{
Console.WriteLine("--------------");
System.Threading.Thread.Sleep(100);
}
if (x % 100 == 0)
{
System.Threading.Thread.Sleep(500);
Console.Clear();
}
errorCount = 0;
}
else
{
if (errorCount < 3)
{
errorCount += 1;
}
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** ,");
System.Threading.Thread.Sleep(1000 * errorCount);
}
}
catch (Exception ex)
{
if (errorCount < 3)
{
errorCount += 1;
}
int dkp = getDKPWithUrl(item.Loc);
Console.WriteLine($"{++x} = DKP-{dkp} , Wait: {1000 * errorCount} Secs , *** Error *** Problem");
using (ILoger Logger = new MongoDBLoggerHelper())
{
LogDTO log = new LogDTO()
{
_id = ObjectId.GenerateNewId().ToString(),
DateTime = DateTime.Now,
Description = ex.Message.ToString(),
ProjectId = (int)ProjectNames.Console,
Url = item.Loc,
MethodName = "Digikala - Console App",
Title = "Get Product Error - " + dkp
};
Logger.AddLog(log);
}
System.Threading.Thread.Sleep(1000 * errorCount);
}
}
}
digikala_SaveProductBatch(null, true);
}
