private static void CrawlForum(string game, bool isOldLayout)
{
game = game.ToUpper();
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
{
var boards = (from b in entities.ForumBoards
where b.BlizzArea.Game.GameAbbreviation == game.ToUpper()
&& b.IsDisabled == false
select b).ToList();
foreach (var b in boards)
{
try
{
if (isOldLayout)
{
ForumBoardCrawlerV1 forumBoardCrawler = new ForumBoardCrawlerV1(b);
forumBoardCrawler.CrawlForumBoard();
}
else
{
ForumBoardCrawler forumBoardCrawler = new ForumBoardCrawler(b);
forumBoardCrawler.CrawlForumBoard();
}
}
catch
{
Console.WriteLine("An unknown error has occurred crawling this board. Moving onto the next. ({0})", b.BoardName);
}
}
}
}
public List<GameDo> GetAllGames(string lang)
{
if (lang == null)
throw new ArgumentNullException("lang");
lang = lang.ToUpper();
List<GameDo> games = new List<GameDo>();
var db = new ForumBlogsDataContext(DbConnectionString.Value);
var allGames = (from g in db.Games
select g).ToList();
foreach (var game in allGames)
{
GameDo gameOb = new GameDo
{
AbbreviatedName = game.GameAbbreviation,
//ShowInMainMenu = game.ShowInMainMenu
};
gameOb.ExistsForCurrentLanguage = (from b in db.BlizzAreas
where b.Game.GameAbbreviation == game.GameAbbreviation
&& b.Language.LanguageAbbreviation == lang
select b).Any();
games.Add(gameOb);
}
return games;
}
public List<string> GetAvailableLanAbbrevs()
{
using (var db = new ForumBlogsDataContext(DbConnectionString.Value))
{
return (from d in db.Languages select d.LanguageAbbreviation).ToList();
}
}
private static void CrawlBlogs(string game)
{
game = game.ToUpper();
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
{
IQueryable<BlizzArea> blizzAreas;
if (game.ToUpper() == "ALL")
{
blizzAreas = from b in entities.BlizzAreas
select b;
}
else
{
blizzAreas = from b in entities.BlizzAreas
where b.Game.GameAbbreviation == game
select b;
}
foreach (var blizzArea in blizzAreas)
{
BlogEntryCrawler crawler = new BlogEntryCrawler(blizzArea.Id);
crawler.DoBlogCrawling(blizzArea.Game.GameAbbreviation, blizzArea.Region.RegionAbbreviation, blizzArea.Language.LanguageAbbreviation);
}
}
}
public void PerformCrawl(string game)
{
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
{
var topicsToCrawl = (from t in entities.Topics
where t.ForumBoard.BlizzArea.Game.GameAbbreviation == game
&& t.ToBeCrawled
select t).ToList();
foreach (var topic in topicsToCrawl)
{
topic.IsBeingCrawled = true;
}
entities.SubmitChanges();
foreach (var topic in topicsToCrawl)
{
try
{
CrawlSpecifiedTopics(topic);
entities.SubmitChanges();
}
catch (Exception e) { Console.WriteLine(e.Message); }
}
entities.SubmitChanges();
}
}
public override void PerformTwitterOperation()
{
Console.Write("Starting... ({0}) \n", "Hearthstone");
ForumBlogsDataContext db = new ForumBlogsDataContext(_connStr);
try
{
var tweetQueue = (from t in db.TweetQueues
where t.DateAdded > (DateTime.UtcNow.AddMinutes(-35))
&& !t.IsTweeted
&& t.BlizzArea.Language.LanguageAbbreviation == this.Lang
& t.BlizzArea.Game.GameAbbreviation == "HEARTHSTONE"
& (t.PostLinkNo.Value != null && (t.PostLinkNo.Value == 1 || t.PostLinkNo.Value % 2 == 0))
select t);
foreach (var tweet in tweetQueue)
{
string gameHashtag = "#Hearthstone";
string gameNormTag = "[#HS]";
string tweetedContent = String.Empty;
if (tweet.IsForumThread)
{
string link = "http://www.blizzposts.com/topic/" + tweet.ItemId + "#" + tweet.PostLinkNo;
tweetedContent = String.Format("{6} New blue post in \"{0}\" {1} - by {2} ({3}) {4}{5}",
HttpUtility.HtmlDecode(tweet.Title),
link,
tweet.NameOfThePoster.Trim(),
tweet.BlizzArea.Region.RegionAbbreviation,
gameHashtag,
"", //was bluecount - removed
gameNormTag);
}
else
{
string link = "http://www.blizzposts.com/blogentry/" + tweet.ItemId;
tweetedContent = String.Format("{4} [Blog {3}-{5}] \"{0}\" - {1} {2}",
HttpUtility.HtmlDecode(tweet.Title),
link,
gameHashtag,
tweet.BlizzArea.Region.RegionAbbreviation,
gameNormTag,
this.Lang);
}
TwitterService service = new TwitterService(CONSUMER_KEY, CONSUMER_SECRET);
service.AuthenticateWith(ACCESS_TOKEN, ACCESS_SECRET);
service.SendTweet(new SendTweetOptions { Status = tweetedContent });
tweet.TweetResult = service.Response.StatusCode + " - " + service.Response.StatusDescription;
tweet.TweetedContent = tweetedContent;
tweet.IsTweeted = true;
tweet.DateIsTweeted = DateTime.UtcNow;
Console.WriteLine("Tweeted content: {0}", tweetedContent);
Console.WriteLine("Tweet Result: {0}\n\n", service.Response.StatusCode);
db.SubmitChanges();
}
}
catch (Exception e)
{
Console.WriteLine("An error has ocurred: \n {0} \n", e.Message);
}
}
private void worker_DoWork(object sender, DoWorkEventArgs e)
{
bool isSuccess = false;
int failureCount = 0;
while (!isSuccess)
{
try
{
TrElementData trElementData = e.Argument as TrElementData;
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(trElementData.ConnStr))
{
Console.WriteLine("Initiating database connection . . .");
int threadCount = (from t in entities.Topics
where t.ThreadNumber == trElementData.threadNumber
&& t.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower() == trElementData.Game.ToLower()
&& t.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower() == trElementData.Lang.ToLower()
&& t.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower() == trElementData.Region.ToLower()
select t).Count();
trElementData.isNewThread = !(threadCount > 0);
if (trElementData.isNewThread)
{
Console.WriteLine("\nFound new thread to be marked for indexing. \n");
Topic topic = new Topic();
topic.ForumBoardId = trElementData.ForumBoardId;
topic.ThreadTitle = trElementData.threadTitle;
topic.AuthorOfThread = trElementData.threadAuthor;
topic.LastPoster = trElementData.lastPoster.Trim();
topic.ThreadNumber = trElementData.threadNumber.Value;
topic.ToBeCrawled = true; //temporary
topic.NoPages = trElementData.lastPage;
topic.LastPostDate = DateTime.Now.AddYears(-3);
topic.ThreadUpdate = trElementData.lastUpdateStamp;
entities.Topics.InsertOnSubmit(topic);
entities.SubmitChanges();
}
else
{
Topic topic = (from t in entities.Topics
where t.ThreadNumber == trElementData.threadNumber
&& t.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower() == trElementData.Game.ToLower()
&& t.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower() == trElementData.Lang.ToLower()
&& t.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower() == trElementData.Region.ToLower()
select t).ToList()[0];
if (topic.ThreadTitle != trElementData.threadTitle)
{
Console.WriteLine("\nThe thread's title was changed.\n");
topic.ThreadTitle = trElementData.threadTitle;
}
if (topic.ThreadUpdate != trElementData.lastUpdateStamp && !topic.IsBeingCrawled && !topic.ToBeCrawled)
{
Console.WriteLine("\nFound known thread that was updated. \n");
topic.ToBeCrawled = true;
topic.NoPages = trElementData.lastPage;
topic.LastPoster = trElementData.lastPoster.Trim();
topic.ThreadUpdate = trElementData.lastUpdateStamp;
}
entities.SubmitChanges();
}
}
isSuccess = true;
Console.WriteLine("Finished database operation.");
}
catch (Exception er)
{
failureCount++;
System.Threading.Thread.Sleep(2000);
if (failureCount > 10)
Console.WriteLine(er.Message);
}
}
}
/// <summary>
/// Gets the title of the specified thread.
/// </summary>
/// <param name="id">id of the thread</param>
/// <returns></returns>
public string GetTopicTitle(int id)
{
var db = new ForumBlogsDataContext(DbConnectionString.Value);
try
{
return (from t in db.Topics
where t.Id == id
select t.ThreadTitle).Single();
}
catch (InvalidOperationException)
{
return null;
}
}
/// <summary>
/// Gets the language of the specified thread.
/// </summary>
/// <param name="id">id of the thread</param>
/// <returns></returns>
public string GetTopicLanguage(int id)
{
var db = new ForumBlogsDataContext(DbConnectionString.Value);
try
{
return (from t in db.Topics
where t.Id == id
select t.ForumBoard.BlizzArea.Language.LanguageAbbreviation).Single();
}
catch (InvalidOperationException)
{
return null;
}
}
public TopicDo GetTopic(int id)
{
TopicDo topic = new TopicDo();
var topicsDbCont = new ForumBlogsDataContext(DbConnectionString.Value);
var miscDbCont = new MiscDataContext(DbConnectionString.Value);
Topic dbTopic = null;
try
{
dbTopic = (from t in topicsDbCont.Topics
where t.Id == id
select t).Single();
}
catch (InvalidOperationException)
{
return null;
}
topic.TopicId = dbTopic.Id;
topic.GameAbbrev = dbTopic.ForumBoard.BlizzArea.Game.GameAbbreviation;
topic.LanguageAbbrev = dbTopic.ForumBoard.BlizzArea.Language.LanguageAbbreviation;
topic.RegionAbbrev = dbTopic.ForumBoard.BlizzArea.Region.RegionAbbreviation;
topic.TopicTitle = dbTopic.ThreadTitle;
topic.RegionFullName = dbTopic.ForumBoard.BlizzArea.Region.RegionName;
topic.BlizzTopicId = dbTopic.ThreadNumber;
IQueryable<Post> posts = null;
IQueryable<MvpPosterBlacklist> blackList = null;
posts = from p in topicsDbCont.Posts
where p.Topic.Id == id
orderby p.Id
select p;
blackList = from b in miscDbCont.MvpPosterBlacklists
select b;
List<Post> lstPosts = posts.ToList();
List<MvpPosterBlacklist> blacklistedMvps = blackList.ToList();
List<Post> newLstPosts = new List<Post>();
foreach (var post in lstPosts)
{
try
{
string directPostLink = post.DirectPostLink;
if (directPostLink[directPostLink.Count() - 1] == '1' && directPostLink[directPostLink.Count() - 2] == '#') //Always include first post, regardless of the blaclist.
{
//we need to always include the first post, regardless of whether the MVP is blacklisted or not
newLstPosts.Add(post);
}
else if (post.PosterType == 1)
{
bool isFound = false;
if (blacklistedMvps.Count > 0)
{
foreach (var mvp in blacklistedMvps)
{
if (post.PosterName.Contains(mvp.PosterName))
isFound = true;
}
}
if (!isFound)
{
newLstPosts.Add(post);
}
}
else
{
newLstPosts.Add(post);
}
}
catch (ArgumentNullException)
{
newLstPosts.Add(post);
}
}
foreach (var post in newLstPosts)
{
PostDo postDomainObj = new PostDo();
postDomainObj.PostId = post.Id;
postDomainObj.PostContent = post.PostContent;
postDomainObj.PosterName = post.PosterName;
postDomainObj.PosterTitle = post.PosterSpecialTitle;
postDomainObj.RawDateStr = post.PostDate;
postDomainObj.SourceUri = post.DirectPostLink;
postDomainObj.Type = post.PosterType.Value;
postDomainObj.Avatar = post.AvatarLinkOfPost;
if (post.PostDateTimeUtc.HasValue)
postDomainObj.DtUtc = post.PostDateTimeUtc.Value;
topic.Posts.Add(postDomainObj);
}
//For ShowMvpPosts button.
topic.HasMvpPosts = (topic.Posts.Where(x => x.Type == 1).Count() > 0 && !(topic.Posts.Where(x => x.Type == 1).Count() == 1 && topic.Posts[0].Type == 1));
return topic;
}
private void CrawlSpecifiedTopics(Topic topic)
{
//We are going to make sure page 1 is always crawled.
List<int> pageNos = new List<int>();
pageNos.Add(1);
if (!topic.LastCrawledPage.HasValue)
{
topic.LastCrawledPage = 1;
}
for (int i = topic.LastCrawledPage.Value; i <= topic.NoPages; i++)
{
if (i != 1)
{
pageNos.Add(i);
}
}
int lastCrawledPage = 0;
for (int i = 0; i <= pageNos.Count - 1; i++)
{
string ht = WebClient.GetRawHtml(String.Format("http://{0}.battle.net/{1}/{2}/forum/topic/{3}?page={4}",
topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower(),
topic.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower(),
topic.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower(),
topic.ThreadNumber,
pageNos[i]));
HtmlDocument document = new HtmlDocument();
document.LoadHtml(ht);
var divTags = document.GetElementbyId("post-list").ChildNodes.Where(x => x.Attributes["class"] != null && x.Attributes["class"].Value.Contains("topic-post"));
foreach (var div in divTags)
{
bool indexThisPost = false;
string postContent = null;
string avatarLink = null;
string linkDirectPost = null;
string postDate = null;
string posterName = null;
string posterSpecialTitle = null;
string postEdited = null;
short posterType = 0;
if (div.HasChildNodes)
{
if (div.InnerHtml.Contains("<span id=\"1\">")) //incredibly fragile
{
indexThisPost = true;
}
if (div.Attributes["class"].Value.Contains("blizzard"))
{
indexThisPost = true;
posterType = 2;
}
else if (div.Attributes["class"].Value.Contains("mvp"))
{
indexThisPost = true;
posterType = 1;
}
if (indexThisPost)
{
#region Post Content
//*[@id="post-64426546361"]/div[1]/table/tr/td[2]/div
var postContentTag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[2]/div", div.Id));
if (postContentTag != null)
{
postContent = String.Empty;
postContent += postContentTag.InnerHtml;
}
#endregion
#region avatar
// //*[@id="post-79781202523"]
var avatarImgTag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[1]/div/div/a/img", div.Id));
if (avatarImgTag != null)
{
avatarLink = avatarImgTag.OuterHtml;
string src = avatarLink.Remove(0, avatarLink.IndexOf("src") + 1);
src = src.Remove(0, src.IndexOf("\"") + 1);
src = src.Remove(src.IndexOf("\""));
if (!src.Contains("battle.net") && !src.Contains("cms/user_avatar") && !src.Contains("media.blizzard")) //TODO: Second condition here might conflic in other regions. Check for this.
{
src = String.Format("http://{0}.battle.net/", topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower()) + src;
}
avatarLink = String.Format("<img alt=\"avatar\" src = \"{0}\" width={1} height={2}", src, "{0}", "{1} />");
}
#endregion
#region Direct Post Link
var indexATag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[3]/div/a", div.Id));
if (indexATag == null) //deleted MVP post
{
indexATag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div[1]/table/tr/td[3]/div/a", div.Id));
}
if (indexATag != null)
{
linkDirectPost = String.Format("http://{0}.battle.net/{1}/{2}/forum/topic/{3}?page={4}{5}",
topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower(),
topic.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower(),
topic.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower(),
topic.ThreadNumber,
pageNos[i],
indexATag.Attributes["href"].Value);
}
#endregion
#region Post Date
var postDateDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[3]/div/div[1]", div.Id));
if (postDateDiv != null)
{
postDate = postDateDiv.Attributes["data-tooltip"].Value.Replace(" ", " ");
}
#endregion
#region Poster Name
var posterNameSpan = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[2]/div[1]/a/span", div.Id));
//*[@id="post-66887172203"]/div[1]/table/tr/td[1]/div/div/div/a/span
if (posterNameSpan == null)
{
posterNameSpan = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div[1]/table/tr/td[1]/div/div/div/a/span", div.Id));
}
if (posterNameSpan != null)
{
posterName = posterNameSpan.InnerText;
}
#endregion
#region Poster Special Title
if (posterType == 2)
{
var posterSpecialTitleDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[2]/div[2]", div.Id));
if (posterSpecialTitleDiv != null)
{
posterSpecialTitle = posterSpecialTitleDiv.InnerText;
}
}
#endregion
#region Post Edited
var postEditedDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[2]/div[2]", div.Id));
if (postEditedDiv != null)
{
postEdited = postEditedDiv.OuterHtml;
}
#endregion
//Combining post edited with post content.
if (!String.IsNullOrEmpty(postEdited))
{
postContent += postEdited;
}
if (postContent != null)
postContent = postContent.Replace("<br>", "<br />");
//Does this post exist? Assume false for now.
bool postExists = false;
//Check if the post exists. If it does exist, make sure there weren't any updates on it.
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
{
int postCount = (from p in entities.Posts
where p.DirectPostLink == linkDirectPost
select p).Count();
postExists = (postCount > 0);
}
if (postExists)
{
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
{
Post post = (from p in entities.Posts
where p.DirectPostLink == linkDirectPost
select p).ToList()[0];
if (post.PostContent != postContent || post.AvatarLinkOfPost != avatarLink)
{
post.PostContent = postContent;
post.AvatarLinkOfPost = avatarLink;
}
entities.SubmitChanges();
}
}
else
{
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
{
if (postContent != null)
{
Post post = new Post();
post.TopicId = topic.Id;
post.DirectPostLink = linkDirectPost;
post.PosterName = posterName;
post.PostDate = postDate;
post.PosterType = posterType;
post.PosterSpecialTitle = posterSpecialTitle;
post.AvatarLinkOfPost = avatarLink;
post.PostContent = postContent;
try
{
SetPostDate(post, topic.ForumBoard.BlizzArea.Region.RegionAbbreviation);
}
catch (Exception) //This will happen if time zone is not specified
{
}
//if (post.PosterType == 2)
//{
// topic.LastPostDate = post.PostDateTimeUtc.Value;
//}
entities.Posts.InsertOnSubmit(post);
entities.SubmitChanges();
}
}
}
}
}
}
lastCrawledPage = pageNos[i];
}
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
{
Topic topicToSave = (from t in entities.Topics
where t.Id == topic.Id
select t).Single();
var lastBlue = (from p in entities.Posts
where p.TopicId == topicToSave.Id
&& p.PosterType == 2
orderby p.PostDateTimeUtc descending
select p).ToList();
if (lastBlue.Count() > 0)
topicToSave.LastPostDate = lastBlue[0].PostDateTimeUtc.Value;
topicToSave.LastCrawledPage = lastCrawledPage;
topicToSave.IsBeingCrawled = false;
topicToSave.ToBeCrawled = false;
entities.SubmitChanges();
}
}
public Domain.BlogEntryDo GetBlogEntry(int id)
{
BlogEntryDo blogEntry = new BlogEntryDo();
var db = new ForumBlogsDataContext(DbConnectionString.Value);
try
{
var entry = (from b in db.BlogEntries
where b.Id == id
select b).Single();
blogEntry.Id = entry.Id;
blogEntry.EntryNumber = entry.EntryNumber;
blogEntry.AnchoredLink = entry.BlogDirectLink;
blogEntry.BlogContent = entry.BlogContent;
blogEntry.BlogPostDate = entry.BlogDate;
//blogEntry.GameFullName = entry.BlizzArea.Game.GameName;
blogEntry.GameAbbrev = entry.BlizzArea.Game.GameAbbreviation;
blogEntry.RegionAbbrev = entry.BlizzArea.Region.RegionAbbreviation;
blogEntry.RegionFullName = entry.BlizzArea.Region.RegionName;
blogEntry.Title = entry.BlogTitle;
blogEntry.LanguageAbbrev = entry.BlizzArea.Language.LanguageAbbreviation;
}
catch (InvalidOperationException)
{
return null;
}
return blogEntry;
}
static void Main(string[] args)
{
while (true)
{
try
{
using (TrackerDataContext db = new TrackerDataContext(_connStr))
{
var trackerItemsNeedingUpdates = (from t in db.TrackerTableListItems
where t.IsPstSummaryUpdateRequired == null
|| t.IsPstSummaryUpdateRequired.Value == true
select t).ToList(); //Need to get them all in a list, because of a weird bug in the ORM.
foreach (var item in trackerItemsNeedingUpdates)
{
if (item.EntryType == 0) //ForumBoards
{
using (ForumBlogsDataContext postsContext = new ForumBlogsDataContext(_connStr))
{
try
{
var desiredPost = (from p in postsContext.Posts
where p.TopicId == item.BlogOrTopicId
&& p.DirectPostLink.EndsWith("#1")
select p).Single();
item.FirstPostSummary = GenerateSummary(desiredPost.PostContent, true);
item.IsPstSummaryUpdateRequired = false;
}
catch
{
HandleFailure(item);
}
db.SubmitChanges();
}
}
else if (item.EntryType == 1) //Blog entries
{
using (ForumBlogsDataContext blogEntriesContext = new ForumBlogsDataContext(_connStr))
{
try
{
var desiredPost = (from p in blogEntriesContext.BlogEntries
where p.Id == item.BlogOrTopicId
select p).Single();
item.FirstPostSummary = GenerateSummary(desiredPost.BlogContent, false);
item.IsPstSummaryUpdateRequired = false;
}
catch
{
HandleFailure(item);
}
db.SubmitChanges();
}
}
else if (item.EntryType == 2) //Twitter conversations
{
using (TwitterDataContext twitterContext = new TwitterDataContext(_connStr))
{
try
{
var desiredPost = (from p in twitterContext.Tweets
where p.TwitterConversationId == item.BlogOrTopicId
orderby p.Id ascending
select p).Take(1).Single();
item.FirstPostSummary = desiredPost.TweetContentNonHtml;
item.IsPstSummaryUpdateRequired = false;
}
catch
{
HandleFailure(item);
}
db.SubmitChanges();
}
}
else
{
throw new NotImplementedException("EntryType");
}
}
}
}
catch { }
//Job should run once per minute.
Console.WriteLine("\nJob done. Sleeping for a minute. - " + DateTime.Now + "\n");
System.Threading.Thread.Sleep(30 * 1000);
}
}
public TrackerDo GetTrackerByPosterName(string posterName, int page)
{
if (posterName == null)
throw new ArgumentNullException("posterName");
TrackerDo tracker = new TrackerDo();
var dbTracker = new TrackerDataContext(DbConnectionString.Value);
var dbForumBlogs = new ForumBlogsDataContext(DbConnectionString.Value);
var threadQuery = (from p in dbForumBlogs.Posts
where p.PosterName == posterName
&& p.PosterType == 2
select p.Topic).Distinct().OrderByDescending(x => x.LastPostDate);
List<int> threadsPostedIn = threadQuery.Skip((page - 1) * 50).Take(50).Select(x => x.Id).ToList();
var trackerQuery = from t in dbTracker.TrackerTableListItems
where threadsPostedIn.Contains(t.BlogOrTopicId)
&& t.EntryType == 0
orderby t.TimeOfLatestPost descending
select t;
List<TrackerTableListItem> trackerListItems = (trackerQuery).ToList();
tracker.TrackerItems = GetItemsDomainObjects(trackerListItems);
tracker.PageTitle = posterName;
tracker.TotalItems = threadQuery.Count();
return tracker;
}
public override void PerformTwitterOperation()
{
Console.Write("Starting... ({0})", this.Lang);
ForumBlogsDataContext db = new ForumBlogsDataContext(_connStr);
try
{
var tweetQueue = (from t in db.TweetQueues
where t.DateAdded > (DateTime.UtcNow.AddMinutes(-120))
&& !t.IsTweeted
&& t.BlizzArea.Language.LanguageAbbreviation == this.Lang
&& t.BlizzArea.Game.GameAbbreviation != "BDA"
select t);
foreach (var tweet in tweetQueue)
{
string gameHashtag = tweet.BlizzArea.Game.GameAbbreviation;
string gameNormTag = String.Empty;
if (gameHashtag == "WOW")
{
gameHashtag = "#Warcraft";
gameNormTag = "[#WoW]";
}
else if (gameHashtag == "D3")
{
gameHashtag = "#Diablo";
gameNormTag = "[#D3]";
}
else if (gameHashtag == "SC2")
{
gameHashtag = "#StarCraft";
gameNormTag = "[#SC2]";
}
else if (gameHashtag == "BLIZZCON")
{
gameHashtag = "#BlizzCon";
}
else if (gameHashtag == "HEARTHSTONE")
{
gameHashtag = "#Hearthstone";
}
else
{
gameHashtag = "#" + gameHashtag;
}
string tweetedContent = String.Empty;
if (tweet.IsForumThread)
{
string link = "http://www.blizzposts.com/topic/" + tweet.ItemId + "#" + tweet.PostLinkNo;
tweetedContent = String.Format("{6} Новые синие посты в \"{0}\" {1} - автор {2} ({3}) {4}{5}",
HttpUtility.HtmlDecode(tweet.Title),
link,
tweet.NameOfThePoster.Trim(),
"", //was region - removed
gameHashtag,
"", //was bluecount - removed
gameNormTag);
}
else
{
string link = "http://www.blizzposts.com/blogentry/" + tweet.ItemId;
tweetedContent = String.Format("{4} [блог {3}{5}] \"{0}\" - {1} {2}",
HttpUtility.HtmlDecode(tweet.Title),
link,
gameHashtag,
"", //was region - removed
gameNormTag,
this.Lang);
}
TwitterService service = new TwitterService(CONSUMER_KEY, CONSUMER_SECRET);
service.AuthenticateWith(ACCESS_TOKEN, ACCESS_SECRET);
service.SendTweet(new SendTweetOptions { Status = tweetedContent });
tweet.TweetResult = service.Response.StatusCode + " - " + service.Response.StatusDescription;
tweet.TweetedContent = tweetedContent;
tweet.IsTweeted = true;
tweet.DateIsTweeted = DateTime.UtcNow;
Console.WriteLine("Tweeted content: {0}", tweetedContent);
Console.WriteLine("Tweet Result: {0}\n\n", service.Response.StatusCode);
db.SubmitChanges();
}
}
catch (Exception e)
{
Console.WriteLine("An error has ocurred: \n {0} \n", e.Message);
}
}
public void DoBlogCrawling(string game, string region, string lang)
{
XmlReader reader = XmlReader.Create(String.Format("http://{0}.battle.net/{1}/{2}/feed/news", region.ToLower(), game.ToLower(), lang.ToLower()));
SyndicationFeed feed = SyndicationFeed.Load(reader);
List<MinedBlogEntry> blogEntries = (from item in feed.Items
select new MinedBlogEntry
{
Title = item.Title.Text,
PublicationDate = ConvertFromDateTimeOffset(item.PublishDate.ToUniversalTime()),
DirectLink = item.Links[0].Uri.AbsoluteUri,
TimeZone = "UTC"
}).ToList();
foreach (var blogEntry in blogEntries)
{
try
{
using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
{
blogEntry.DirectLinkWithAnchor = String.Format("<a href = \"{0}\"> {1} </a>", blogEntry.DirectLink, blogEntry.Title);
blogEntry.EntryNumber = int.Parse(blogEntry.DirectLink.Remove(0, blogEntry.DirectLink.IndexOf("blog/") + 5));
int existenceCount = (from b in entities.BlogEntries
where b.BlizzArea.Game.GameAbbreviation == game
&& b.BlizzArea.Region.RegionAbbreviation == region
&& b.BlizzArea.Language.LanguageAbbreviation == lang
&& b.EntryNumber == blogEntry.EntryNumber
select b).Count();
if (!(existenceCount > 0))
{
Console.WriteLine("Found a new blog entry.");
string blogEntryPage = WebClient.GetRawHtml(blogEntry.DirectLink);
HtmlDocument document = new HtmlDocument();
document.LoadHtml(blogEntryPage);
//article-content -> we want the contents of this div. Reason why we aren't getting it from feed directly is because we want the banner image.
string blogContent = String.Empty;
string author = String.Empty;
////*[@id="blog"]/div[2]
var blogContentDiv = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div[2]");
if (blogContentDiv != null)
{
blogContent = blogContentDiv.InnerHtml;
author = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div[1]/a[1]/span[2]").InnerText;
}
else
{
var headingImage = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[2]");
blogContentDiv = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[3]");
author = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[1]/div/a").InnerText;
if (headingImage != null)
{
blogContent += headingImage.OuterHtml;
}
if (blogContentDiv != null)
{
blogContent += blogContentDiv.InnerHtml;
}
else
{
Console.WriteLine("Cannot mine this. The div tags are not in their expected format.");
break;
}
}
blogContent = blogContent.Replace("<br>", "<br />");
TimeSpan span = DateTime.UtcNow - blogEntry.PublicationDate;
if (span.TotalMinutes > 60)
{
blogEntry.IsTweeted = true;
}
BlogEntry newBlogEntry = new BlogEntry();
newBlogEntry.BlizzAreaId = this.BlogPageId;
newBlogEntry.BlogTitle = blogEntry.Title;
newBlogEntry.BlogDirectLink = blogEntry.DirectLinkWithAnchor;
newBlogEntry.BlogContent = blogContent;
newBlogEntry.BlogDate = blogEntry.PublicationDate;
newBlogEntry.TimeZone = "UTC";
newBlogEntry.EntryNumber = blogEntry.EntryNumber;
newBlogEntry.BlogEntryAuthor = author;
entities.BlogEntries.InsertOnSubmit(newBlogEntry);
entities.SubmitChanges();
}
else
{
Console.WriteLine("Found a blog entry. This blog entry already exists.");
//TODO: We should probably check last update stamp.
}
}
}
catch (Exception e)
{
Console.WriteLine("Could not mine blog entry for following reason: \n {0}", e.Message);
}
}
}
