private void BindArticle()
{
string ktm = gridExProductsAndArticles.CurrentRow.Cells["KTM"].Value.ToString();
string descriptor = gridExProductsAndArticles.CurrentRow.Cells["Deskryptor"].Value.ToString();
int agreementId = gridExAgreementsListGrouped.CurrentRow.GroupCaption == null ? (int)gridExAgreementsListGrouped.CurrentRow.Cells["IdUmowy"].Value : Convert.ToInt16(gridExAgreementsListGrouped.CurrentRow.GroupCaption);
if (MessageBox.Show(string.Format("Do umowy {0} zostanie dopisany towar powiązany:\r\n\r\nKTM:\t\t{1}\r\nDeskryptor:\t{2}", agreementId.ToString(), ktm, descriptor), "Przypisanie", MessageBoxButtons.OKCancel, MessageBoxIcon.Information) == DialogResult.OK)
{
try
{
RelatedArticle relatedArticle = new RelatedArticle()
{
AgreementId = agreementId,
KTM = ktm,
Descriptor = descriptor,
Comments = ""
};
AgreementsRepo.InsertRelatedArticle(relatedArticle);
MessageBox.Show("Towar został poprawnie powiązany.", "Przypisanie", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
catch
{
MessageBox.Show("Nie udało się wykonać powiązania.\r\nOdśwież listę i spróbuj ponownie.", "Przypisanie", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
this.LoadData();
}
}
public void RelatedArticlePropertyChangedShouldBeFired()
{
//arrange
var realedArticle = new RelatedArticle();
//Assert
PropertyChangedAsserter.AssertPropertyChanged(realedArticle, (x) => x.url = "test", "url");
}
void QuoteArticles(string id)
{
if (id != OwnerID)
{
RelatedArticle a = new RelatedArticle();
a.ArticleID = OwnerID;
a.RelatedID = id;
ArticleHelper.AddRelatedArticle(a);
}
}
public void AddRow(RelatedArticle newArticle)
{
DataRow newRow = dataTable.NewRow();
newRow.SetField("umowaId", newArticle.AgreementId);
newRow.SetField("KTM", newArticle.KTM);
newRow.SetField("Deskryptor", newArticle.Descriptor);
newRow.SetField("Uwagi", "");
this.dataTable.Rows.Add(newRow);
}
/// <summary>
/// Updates a related article
/// </summary>
/// <param name="relatedArticle">Related article</param>
public virtual void UpdateRelatedArticle(RelatedArticle relatedArticle)
{
if (relatedArticle == null)
{
throw new ArgumentNullException("relatedArticle");
}
_relatedArticleRepository.Update(relatedArticle);
//event notification
_eventPublisher.EntityUpdated(relatedArticle);
}
public JsonResult Update(int articleID, string articles)
{
var jsonSuccessResult = new JsonSuccessResult();
try
{
string[] arrArticles = articles.Split(',');
// حذف
#region Delete All
RelatedArticles.DeleteRelatedArticles(articleID);
#endregion Delete All
// ثبت مجدد
#region Add
List <RelatedArticle> listItems = new List <RelatedArticle>();
foreach (var item in arrArticles)
{
if (!String.IsNullOrWhiteSpace(item))
{
RelatedArticle article = new RelatedArticle
{
ArticleID = articleID,
RelationID = Int32.Parse(item),
LastUpdate = DateTime.Now,
};
listItems.Add(article);
}
}
RelatedArticles.Insert(listItems);
#endregion Add
jsonSuccessResult.Success = true;
}
catch (Exception ex)
{
jsonSuccessResult.Errors = new string[] { ex.Message };
jsonSuccessResult.Success = false;
}
return(new JsonResult()
{
Data = jsonSuccessResult
});
}
public static void InsertRelatedArticle(RelatedArticle article)
{
using (SqlConnection dbConnection = new DBConnection().Connection)
{
string sqlQuery = string.Format("INSERT INTO [Towary] (UmowaId, Deskryptor, KTM, Uwagi, WprowadzenieUzytkownik) " +
"VALUES (@umowaId, @deskryptor, @ktm, @uwagi, @wprowadzenieUzytkownik) ");
using (SqlCommand command = new SqlCommand(sqlQuery, dbConnection))
{
command.Parameters.Add("@umowaId", SqlDbType.Int, 10).Value = article.AgreementId;
command.Parameters.Add("@deskryptor", SqlDbType.VarChar, 50).Value = article.Descriptor;
command.Parameters.Add("@ktm", SqlDbType.VarChar, 50).Value = article.KTM;
command.Parameters.Add("@uwagi", SqlDbType.VarChar, 1000).Value = article.Comments;
command.Parameters.Add("@wprowadzenieUzytkownik", SqlDbType.Int, 6).Value = AppUser.Instance.UserId;
command.ExecuteNonQuery();
}
}
}
protected virtual int EnsureMutuallyRelatedArticles(List <int> articleIds)
{
int count = 0;
foreach (int id1 in articleIds)
{
var mutualAssociations = (
from rp in _relatedArticleRepository.Table
join p in _articleRepository.Table on rp.ArticleId2 equals p.Id
where !p.Deleted && rp.ArticleId2 == id1
select rp).ToList();
foreach (int id2 in articleIds)
{
if (id1 == id2)
{
continue;
}
if (!mutualAssociations.Any(x => x.ArticleId1 == id2))
{
int maxDisplayOrder = _relatedArticleRepository.TableUntracked
.Where(x => x.ArticleId1 == id2)
.OrderByDescending(x => x.DisplayOrder)
.Select(x => x.DisplayOrder)
.FirstOrDefault();
var newRelatedArticle = new RelatedArticle
{
ArticleId1 = id2,
ArticleId2 = id1,
DisplayOrder = maxDisplayOrder + 1
};
InsertRelatedArticle(newRelatedArticle);
++count;
}
}
}
return(count);
}
static void InsertArticle(string username, RemoteArticleInfo article, string category)
{
if (article.url.Contains("http://bestie.vn/") || article.url.Contains("http://thegioitre.vn/"))
{
article.content = System.Net.WebUtility.HtmlDecode(article.content);
article.keyword = string.Empty;
article.author = article.author.Split('-')[0].Trim();
}
if (article.author.Length > 64)
{
article.author = string.Empty;
}
if (string.IsNullOrEmpty(article.keyword))
{
article.keyword = "";
}
if (article.keyword.Length > 255)
{
string tmp = "";
for (int i = 0; i < article.keyword.Split(',').Length; i++)
{
if ((tmp + article.keyword.Split(',')[i]).Length < 255)
{
tmp += article.keyword.Split(',')[i];
}
}
article.keyword = tmp;
}
if (string.IsNullOrEmpty(article.lead_image_url) ||
string.IsNullOrEmpty(article.content) ||
string.IsNullOrEmpty(article.title) ||
string.IsNullOrWhiteSpace(article.title) ||
string.IsNullOrWhiteSpace(article.content)
)
{
return;
}
if (article.excerpt.Length > 999)
{
article.excerpt = article.excerpt.Substring(0, 999);
}
//headline re-validate
article.excerpt = RevalidateHeadline(article.excerpt);
if (string.IsNullOrEmpty(article.keyword))
{
article.keyword = string.Empty;
}
article.date_published = article.date_published.Replace(" ", " ");
article.date_published = DateTimeParse(article.date_published.Trim());
//publish time
article.date_published = DateTime.TryParse(article.date_published.Trim(), out var publishTime) ? publishTime.ToString(CultureInfo.InvariantCulture) : DateTime.Now.ToString(CultureInfo.InvariantCulture);
//author
if (!string.IsNullOrEmpty(article.author))
{
article.author = RemoveAtEnd(article.author);
}
string frendlyTitle = VietCMS.Framework.Core.Common.WebControl.ToFriendlyString(article.title);
if (_appDbContext.Articles.Count(x => x.SourceUrl.Equals(article.url) ||
x.FriendlyTitle.Equals(frendlyTitle)) == 0)
{
string tmp = "";
if (string.IsNullOrEmpty(article.keyword))
{
article.keyword = String.Empty;
}
string[] s = article.keyword.Split(',');
List <Keyword> keywords = new List <Keyword>();
for (int i = 0; i < s.Length; i++)
{
string si = s[i];
if (si.Trim().Length > 40 || si.Trim().Length < 2)
{
continue;
}
tmp += VietCMS.Framework.Core.Common.WebControl.ToFriendlyString(si);
if (i != s.Length - 1)
{
tmp += ",";
}
if (_appDbContext.Keywords.Count(x => x.Title.Equals(si.Trim())) == 0)
{
keywords.Add(_appDbContext.Keywords.Add(new Keyword()
{
Title = s[i],
CreatedAt = DateTime.Now,
FriendlyTitle = VietCMS.Framework.Core.Common.WebControl.ToFriendlyString(si),
Type = "Keyword",
VisitCount = 0
}));
}
else
{
keywords.Add(_appDbContext.Keywords.FirstOrDefault(x => x.Title.Equals(si.Trim())));
}
}
_appDbContext.SaveChanges();
List <RelatedArticle> relatedArticles = new List <RelatedArticle> ();
foreach (Keyword k in keywords)
{
List <Article> articles = _appDbContext.ArticleKeywords
.Where(y => y.Keyword.Id == k.Id)
.Take(20)
.OrderByDescending(x => x.Article.CreatedAt)
.Select(x => x.Article)
.ToList();
foreach (Article item in articles)
{
bool check = false;
foreach (RelatedArticle ra in relatedArticles)
{
if (item.Id == ra.Origin.Id)
{
ra.Index++;
check = true;
}
}
if (!check)
{
RelatedArticle relatedArticle = new RelatedArticle()
{
Origin = item,
Index = 1,
CreatedAt = DateTime.Now,
Type = "Related"
};
relatedArticles.Add(relatedArticle);
}
}
}
Article a = new Article()
{
Title = article.title.Trim(),
Thumbnail = article.lead_image_url,
Category = _appDbContext.Categories.FirstOrDefault(x => x.FriendlyName.Equals(category)),
Headlines = article.excerpt.Trim(),
Content = article.content,
Keywords = article.keyword.Trim(),
FriendlyKeywords = tmp,
Source = article.domain,
SourceUrl = article.url,
FriendlyTitle = VietCMS.Framework.Core.Common.WebControl.ToFriendlyString(article.title.Trim()),
AuthorAlias = article.author.Trim(),
Status = Const.ArticleStatusActive,
CreatedAt = DateTime.Now,
CreatedBy = _appDbContext.Users.FirstOrDefault(x => x.Username.Equals(username)),
LastModifiedBy = _appDbContext.Users.FirstOrDefault(x => x.Username.Equals(username)),
LastModifiedAt = DateTime.Now,
ViewCount = 0,
SubTitle = article.title.Trim().Length > 50 ? article.title.Trim().Substring(0, 47) + "..." : article.title.Trim()
};
foreach (Keyword keyword in keywords)
{
_appDbContext.ArticleKeywords.Add(new ArticleKeyword()
{
Article = a,
Keyword = keyword
});
}
for (int i = 0; i < relatedArticles.Count; i++)
{
var r = relatedArticles[i];
r.Related = a;
_appDbContext.RelatedArticles.Add(r);
_appDbContext.RelatedArticles.Add(new RelatedArticle()
{
CreatedAt = DateTime.Now,
Index = r.Index,
Origin = a,
Related = r.Origin,
Type = "Related"
});
}
_appDbContext.Articles.Add(a);
AddedArticles.Add(a);
_appDbContext.SaveChanges();
}
}
public static void GetSingleArticleContentUsingSeleniumAndSaveToDB(string url, int numberOfPagesForCommentsToDownload)
{
string title = null;
DateTime? createdAt = null;
string author = null;
string content = null;
int? numberOfComments = null;
int? numberOfLikes = null;
List <Tag> tags = new List <Tag>();
List <RelatedArticle> relatedArticles = new List <RelatedArticle>();
List <Comment> comments = new List <Comment>();
// initialize
var chromeOptions = new ChromeOptions();
chromeOptions.AddArguments("headless");
//chromeOptions.SetLoggingPreference(LogType.Browser, LogLevel.Severe);
IWebDriver driver = new ChromeDriver(chrome_driver_path, chromeOptions);
driver.Url = url;
#region accept policy !!! (show)
// accept policy
try
{
var accept_site_policy = driver
.FindElements(By.XPath("//button"))
.Where(x => x.Text.ToLower().Contains("przechodzę do serwisu"))
.First()
;
accept_site_policy.Click();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region title !!! (show)
try
{
var _title = driver.FindElements(By.XPath("//div[@class='single-entry__header' or @class='single-article-header']//h1"))
.First()
.Text;
title = HelperMethods.CleanText(_title);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region created at
try
{
var _createdAt = driver.FindElements(By.XPath("//div[@class='single-entry__header' or @class='single-article-header']//span[@class='time']"))
.First()
.GetAttribute("datetime");
createdAt = Convert.ToDateTime(_createdAt);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region author !!! (show)
try
{
// see, sometimes text will not return text, and we need to use innertext
var _author = driver.FindElements(By.XPath("//div[@class='single-entry__footer' or @class='slideshow__footer']//span[@class='author']"))
.First()
.GetAttribute("innerText");
author = HelperMethods.CleanText(_author);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get facebook likes number !!! (show)
try
{
// find iframe with given name
var detailFrame = driver.FindElement(By.XPath("//div[@class='single-entry__header']//iframe[@title='fb:like Facebook Social Plugin']"));
// open selected iframe
driver.SwitchTo().Frame(detailFrame);
// the iframe code in this case is a bit "encrypted"
// get all texts from all spans
// try to parse each element
// get the first one that contain number
numberOfLikes = driver.FindElements(By.XPath("//span"))
.Select(x =>
{
int value;
string old_value = x.Text;
bool success = int.TryParse(x.Text, out value);
return(new { value, old_value, success });
})
.Where(y => y.success == true)
.Select(z => z.value)
.First()
;
// switch to main window and main frame
driver.SwitchTo().DefaultContent();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region number of comments
try
{
numberOfComments = driver.FindElements(By.XPath("//div[@class='single-entry__footer' or @class='slideshow__footer']//a[@class='comments-link']"))
.First()
.GetAttribute("innerText")
.Split(' ')
.Select(x =>
{
int value;
string old_value = x;
bool success = int.TryParse(x, out value);
return(new { value, old_value, success });
})
.Where(y => y.success == true)
.Select(z => z.value)
.First();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region tags
try
{
var _tags = driver.FindElements(By.XPath("//div[@class='single-entry__header']//span[@class='inline-tags']//a"));
foreach (var _tag in _tags)
{
Tag tag = new Tag();
tag.Text = HelperMethods.CleanText(_tag.Text);
tag.Url = _tag.GetAttribute("href");
tags.Add(tag);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region related articles
try
{
var _related_article_candidates = driver.FindElements(By.XPath("//div[@class='single-entry-text bbtext']//p//a"));
foreach (var _related_article_candidate in _related_article_candidates)
{
RelatedArticle relatedArticle = new RelatedArticle();
relatedArticle.Text = HelperMethods.CleanText(_related_article_candidate.Text);
relatedArticle.Url = _related_article_candidate.GetAttribute("href");
relatedArticles.Add(relatedArticle);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region article text
try
{
var all_article_texts = driver.FindElements(By.XPath("//div[@class='single-entry-text bbtext' or @class='single-article-text']//p"));
string _content = "";
foreach (var text in all_article_texts)
{
string current_text = text.GetAttribute("innerText");
if (current_text.Length > 0)
{
_content += current_text + " ";
}
}
if (_content.Length > 0)
{
content = HelperMethods.CleanText(_content);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region comments
// parse comments
int comments_website_subpage_number = 1;
List <Comment> current_website_popular_comments = new List <Comment>();
while (comments_website_subpage_number <= numberOfPagesForCommentsToDownload)
{
string current_comments_website_subpage = url + "/" + comments_website_subpage_number.ToString() + "/#comments";
driver.Url = current_comments_website_subpage;
var current_subpage_comments = driver.FindElements(By.XPath("//div[@class='comment comment-odd' or @class='comment comment-even']"));
// ger the popular comments only onces
if (comments_website_subpage_number == 1)
{
// get higlither / most popular comments
var current_subpage_popular_comments = driver.FindElements(By.XPath("//div[@class='comments-popular']//div[@class='comment']"));
foreach (var current_popular_comment in current_subpage_popular_comments)
{
string popularCommentAuthor = null;
string popularCommentMessage = null;
try
{
popularCommentAuthor = HelperMethods.CleanText(current_popular_comment.FindElements(By.XPath(".//span[@class='comment-author']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
popularCommentMessage = HelperMethods.CleanText(current_popular_comment.FindElements(By.XPath(".//div[@class='comment-text']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
var popular_comment = new Comment();
popular_comment.Author = popularCommentAuthor;
popular_comment.Message = popularCommentMessage;
current_website_popular_comments.Add(popular_comment);
}
}
// break if there is no comments
if (current_subpage_comments.Count() == 0)
{
break;
}
else
{
foreach (var current_subpage_comment in current_subpage_comments)
{
string commentAuthor = null;
DateTime?commentCreatedAt = null;
bool? commentIsHighlited = null;
string commentMessage = null;
int? commentThumbsDown = null;
int? commentThumbsUp = null;
try
{
commentAuthor = HelperMethods.CleanText(current_subpage_comment.FindElements(By.XPath(".//span[@class='comment-author']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
commentCreatedAt = Convert.ToDateTime(current_subpage_comment.FindElements(By.XPath(".//span[@class='comment-date']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
commentThumbsUp = Convert.ToInt32(current_subpage_comment.FindElements(By.XPath(".//div[@class='comment-vote']//span[@class='plus']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
commentThumbsDown = Convert.ToInt32(current_subpage_comment.FindElements(By.XPath(".//div[@class='comment-vote']//span[@class='minus']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
commentMessage = HelperMethods.CleanText(current_subpage_comment.FindElements(By.XPath(".//div[@class='comment-text']")).First().Text);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
try
{
if (current_website_popular_comments.Where(x => x.Author == commentAuthor && x.Message == commentMessage).Count() > 0)
{
commentIsHighlited = true;
}
else
{
commentIsHighlited = false;
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
Comment comment = new Comment();
comment.Author = commentAuthor;
comment.CreatedAt = commentCreatedAt;
comment.IsHighlited = commentIsHighlited;
comment.Message = commentMessage;
comment.ThumbsDown = commentThumbsDown;
comment.ThumbsUp = commentThumbsUp;
comments.Add(comment);
}
}
comments_website_subpage_number++;
}
#endregion
#region extras !!!
// wait
//driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(5);
//var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(3));
// wait for element
//var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(timeOut)).Until(ExpectedConditions.ElementExists((By.Id(login))));
// wait rundom number of seconds...
//var rnd = new Random();
//var delay = rnd.Next(5000, 10001);
//Thread.Sleep(delay);
#endregion
// clean up the driver
driver.Quit();
#region db operations
// save results to database
using (var context = new ArticlesContext())
{
var current_article = context.Articles.Where(x => x.Url == url).First();
current_article.Title = title;
current_article.CreatedAt = createdAt;
current_article.Author = author;
current_article.Content = content;
current_article.NumberOfComments = numberOfComments;
current_article.NumberOfLikes = numberOfLikes;
current_article.IsDownloaded = true;
current_article.UpdatedAt = DateTime.UtcNow;
current_article.Tags.AddRange(tags.Where(x => !current_article.Tags.Any(y => y.Text == x.Text)));
current_article.RelatedArticles.AddRange(relatedArticles.Where(x => !current_article.RelatedArticles.Any(y => y.Text == x.Text)));
current_article.Comments.AddRange(comments.Where(x => !current_article.Comments.Any(y => y.Author == x.Author && y.Message == x.Message && x.CreatedAt == y.CreatedAt)));
context.SaveChanges();
}
#endregion
}
public void get_single_article_content_basic_article_using_selenium_and_save_to_db(string url, int current_article_id)
{
#region prepare variables
string article_title = null;
DateTime?article_created_at = null;
string article_author = null;
string article_content = null;
int? article_number_of_comments = null;
int? article_number_of_likes = null;
int? article_number_of_images = null;
int? article_number_of_bolded_text = null;
bool? article_has_slideshow = null;
bool? article_has_video = null;
var tags = new List <Tag>();
var relatedArticles = new List <RelatedArticle>();
var comments = new List <Comment>();
#endregion
// go to the artile url
browser.Url = url;
browser.Navigate();
#region accept webiste policy
try
{
var accept_site_policy = browser
.FindElements(By.XPath("//button"))
.Where(x => x.Text.ToLower().Contains("przechodzę do serwisu"))
.First()
;
accept_site_policy.Click();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get title
try
{
var _article_title = browser.FindElement(By.XPath("//div[@id='item']//h1[@class='item-title']"));
article_title = HelperMethods.CleanText(_article_title.GetAttribute("innerText").ToString());
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get author
try
{
var _article_author = browser.FindElement(By.XPath("//div[@class='item-details']//a"));
article_author = HelperMethods.CleanText(_article_author.GetAttribute("innerText").ToString());
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get creation datetime
try
{
var _article_created_at = browser.FindElement(By.XPath("//head//meta[@property='og:published_at']"));
article_created_at = Convert.ToDateTime(_article_created_at.GetAttribute("content").ToString());
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get number of comments
try
{
var _article_number_of_comments_candidate = browser.FindElement(By.XPath("//head//meta[@property='og:comments_count']"));
var _article_number_of_comments = _article_number_of_comments_candidate.GetAttribute("content");
article_number_of_comments = Convert.ToInt32(HelperMethods.CleanText(_article_number_of_comments.ToString()));
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get number of facebook likes
try
{
// find iframe with given name
var detailFrame = browser.FindElement(By.XPath("//div[@class='single-entry__header']//iframe[@title='fb:like Facebook Social Plugin']"));
// open selected iframe
browser.SwitchTo().Frame(detailFrame);
// the iframe code in this case is a bit "encrypted"
// get all texts from all spans
// try to parse each element
// get the first one that contain number
article_number_of_likes = browser.FindElements(By.XPath("//span"))
.Select(x =>
{
int value;
string old_value = x.Text;
bool success = int.TryParse(x.Text, out value);
return(new { value, old_value, success });
})
.Where(y => y.success == true)
.Select(z => z.value)
.First()
;
// switch to main window and main frame
browser.SwitchTo().DefaultContent();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get number of bolded texts
try
{
var _article_number_of_bolded_text = browser.FindElement(By.XPath("//div[@class='single-entry-text bbtext' or @class='single-article-text' or @class='article']//p"));
article_number_of_bolded_text = _article_number_of_bolded_text.ToString().Length;
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get number of images
#region get number of images (article content)
int?_number_of_images = 0;
try
{
_number_of_images = Convert.ToInt32(browser.FindElement(By.XPath("//div[@class='image-container']//img")).ToString());
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get number of images (slideshow)
article_has_slideshow = false;
int?_images_in_slideshow = 0;
try
{
var _slideshow = browser.FindElement(By.XPath("//div[@data-utm_campaign='slideshow']//span[@class='slideshow-current']"));
_images_in_slideshow = Convert.ToInt32(_slideshow.GetAttribute("innerText").ToString().Split("/")[1]);
article_has_slideshow = true;
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
// total number of images in website
article_number_of_images = (_number_of_images ?? 0) + (_images_in_slideshow ?? 0);
#endregion
#region get article text
try
{
var all_article_texts = browser.FindElements(By.XPath("//div[@class='item-content']//h2[@class='item-description']"));
var _article_content = "";
foreach (var text in all_article_texts)
{
var current_text = text.GetAttribute("innerText").ToString();
if (current_text.Length > 0)
{
_article_content += current_text + " ";
}
}
if (_article_content.Length > 0)
{
article_content = HelperMethods.CleanText(_article_content);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get tags
try
{
var _tags = browser.FindElements(By.XPath("//div[@class='item-content']//div[@class='tags']//a"));
foreach (var _tag in _tags)
{
var tag = new Tag()
{
article_id = current_article_id, tag_text = HelperMethods.CleanText(_tag.GetAttribute("innerText")), tag_url = HelperMethods.CleanText(_tag.GetAttribute("href"))
};
tags.Add(tag);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get comments
try
{
var _comments = browser.FindElements(By.XPath("//div[@class='comments-top']//div[@class='comment']"));
foreach (var _comment in _comments)
{
#region get single comment
try
{
#region get basic comment info
var _comment_author = _comment.FindElement(By.XPath(".//span[@class='comment-author']"));
var _comment_created_at = _comment.FindElement(By.XPath(".//span[@class='comment-date']"));
var comment_message = HelperMethods.CleanText(_comment.GetAttribute("innerText"));
var _comment_thumbs_up = _comment.FindElement(By.XPath(".//span[@class='yesCount']"));
var _comment_thumbs_down = _comment.FindElement(By.XPath(".//span[@class='noCount']"));
var _comment_is_highlited = true;
#endregion
#region get comment message
// get entire comment message
try
{
var _comment_info = _comment.FindElement(By.XPath(".//div[@class='comment-info']"));
var comment_info = HelperMethods.CleanText(_comment_info.GetAttribute("innerText").ToString());
comment_message = HelperMethods.CleanText(comment_message.Replace(comment_info, ""));
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
// remove any comment info from message
try
{
var _comment_options = _comment.FindElement(By.XPath(".//div[@class='comment-options']"));
var comment_options = HelperMethods.CleanText(_comment_options.GetAttribute("innerText")).ToString();
comment_message = HelperMethods.CleanText(comment_message.Replace(comment_options, ""));
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
// remove any quotes from comment message
try
{
var _comment_message_quote = _comment.FindElement(By.XPath(".//div[@class='quote']"));
var comment_message_quote = HelperMethods.CleanText(_comment_message_quote.GetAttribute("innerText"));
comment_message = HelperMethods.CleanText(comment_message.Replace(comment_message_quote, ""));
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region add comment to list
var comment = new Comment()
{
article_id = current_article_id,
comment_author = HelperMethods.CleanText(_comment_author.GetAttribute("innerText")),
comment_created_at = Convert.ToDateTime(HelperMethods.CleanText(_comment_created_at.GetAttribute("innerText"))),
comment_message = comment_message,
comment_thumbs_up = Convert.ToInt32(HelperMethods.CleanText(_comment_thumbs_up.GetAttribute("innerText"))),
comment_thumbs_down = Convert.ToInt32(HelperMethods.CleanText(_comment_thumbs_down.GetAttribute("innerText"))),
comment_is_highlited = _comment_is_highlited
};
comments.Add(comment);
#endregion
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region get related articles list
try
{
var _related_article_candidates = browser.FindElements(By.XPath("//div[@class='single-entry-text bbtext']//p//a"));
foreach (var _related_article_candidate in _related_article_candidates)
{
var relatedArticle = new RelatedArticle()
{
article_id = current_article_id,
related_article_text = HelperMethods.CleanText(_related_article_candidate.GetAttribute("innerText")),
related_article_url = _related_article_candidate.GetAttribute("href")
};
relatedArticles.Add(relatedArticle);
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
#region save article details to database
try
{
// save basic article details to database
using (var context = new ArticlesContext())
{
var current_article = context.Articles.Where(x => x.article_url == url).First();
current_article.article_title = article_title;
current_article.article_created_at = article_created_at;
current_article.article_author = article_author;
current_article.article_content = article_content;
current_article.article_number_of_comments = article_number_of_comments;
current_article.article_number_of_likes = article_number_of_likes;
current_article.article_number_of_images = article_number_of_images;
current_article.article_has_slideshow = article_has_slideshow;
current_article.article_has_video = article_has_video;
current_article.article_number_of_bolded_text = article_number_of_bolded_text;
current_article.article_is_downloaded = true;
current_article.article_updated_at = DateTime.Now;
current_article.Tags.AddRange(tags.Where(x => !current_article.Tags.Any(y => y.tag_text == x.tag_text)));
current_article.RelatedArticles.AddRange(relatedArticles.Where(x => !current_article.RelatedArticles.Any(y => y.related_article_text == x.related_article_text)));
current_article.Comments.AddRange(comments.Where(x => !current_article.Comments.Any(y => y.comment_author == x.comment_author && y.comment_message == x.comment_message && x.comment_created_at == y.comment_created_at)));
context.SaveChanges();
}
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
#endregion
}
