Exemplo n.º 1
0
        private static void CrawlForum(string game, bool isOldLayout)
        {
            game = game.ToUpper();

            using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
            {
                var boards = (from b in entities.ForumBoards
                             where b.BlizzArea.Game.GameAbbreviation == game.ToUpper()
                             && b.IsDisabled == false
                             select b).ToList();

                foreach (var b in boards)
                {
                    try
                    {

                        if (isOldLayout)
                        {
                            ForumBoardCrawlerV1 forumBoardCrawler = new ForumBoardCrawlerV1(b);
                            forumBoardCrawler.CrawlForumBoard();
                        }
                        else
                        {
                            ForumBoardCrawler forumBoardCrawler = new ForumBoardCrawler(b);
                            forumBoardCrawler.CrawlForumBoard();
                        }
                    }
                    catch
                    {
                        Console.WriteLine("An unknown error has occurred crawling this board. Moving onto the next. ({0})", b.BoardName);
                    }
                }
            }
        }
Exemplo n.º 2
0
        public List<GameDo> GetAllGames(string lang)
        {
            if (lang == null)
                throw new ArgumentNullException("lang");

            lang = lang.ToUpper();

            List<GameDo> games = new List<GameDo>();
            var db = new ForumBlogsDataContext(DbConnectionString.Value);

            var allGames = (from g in db.Games
                            select g).ToList();
            foreach (var game in allGames)
            {
                GameDo gameOb = new GameDo
                {
                    AbbreviatedName = game.GameAbbreviation,
                    //ShowInMainMenu = game.ShowInMainMenu
                };

                gameOb.ExistsForCurrentLanguage = (from b in db.BlizzAreas
                                                   where b.Game.GameAbbreviation == game.GameAbbreviation
                                                   && b.Language.LanguageAbbreviation == lang
                                                   select b).Any();

                games.Add(gameOb);
            }

            return games;
        }
Exemplo n.º 3
0
 public List<string> GetAvailableLanAbbrevs()
 {
     using (var db = new ForumBlogsDataContext(DbConnectionString.Value))
     {
         return (from d in db.Languages select d.LanguageAbbreviation).ToList();
     }
 }
Exemplo n.º 4
0
        private static void CrawlBlogs(string game)
        {
            game = game.ToUpper();

            using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
            {
                IQueryable<BlizzArea> blizzAreas;

                if (game.ToUpper() == "ALL")
                {
                    blizzAreas = from b in entities.BlizzAreas
                                select b;
                }
                else
                {
                    blizzAreas = from b in entities.BlizzAreas
                                where b.Game.GameAbbreviation == game
                                select b;
                }

                foreach (var blizzArea in blizzAreas)
                {
                    BlogEntryCrawler crawler = new BlogEntryCrawler(blizzArea.Id);
                    crawler.DoBlogCrawling(blizzArea.Game.GameAbbreviation, blizzArea.Region.RegionAbbreviation, blizzArea.Language.LanguageAbbreviation);
                }
            }
        }
Exemplo n.º 5
0
        public void PerformCrawl(string game)
        {
            using (ForumBlogsDataContext entities = new ForumBlogsDataContext(_connStr))
            {
                var topicsToCrawl = (from t in entities.Topics
                                     where t.ForumBoard.BlizzArea.Game.GameAbbreviation == game
                                     && t.ToBeCrawled
                                     select t).ToList();

                foreach (var topic in topicsToCrawl)
                {
                    topic.IsBeingCrawled = true;
                }

                entities.SubmitChanges();

                foreach (var topic in topicsToCrawl)
                {
                    try
                    {
                        CrawlSpecifiedTopics(topic);
                        entities.SubmitChanges();
                    }
                    catch (Exception e) { Console.WriteLine(e.Message); }
                }

                entities.SubmitChanges();
            }
        }
        public override void PerformTwitterOperation()
        {
            Console.Write("Starting... ({0}) \n", "Hearthstone");

            ForumBlogsDataContext db = new ForumBlogsDataContext(_connStr);

            try
            {
                var tweetQueue = (from t in db.TweetQueues
                                  where t.DateAdded > (DateTime.UtcNow.AddMinutes(-35))
                                  && !t.IsTweeted
                                  && t.BlizzArea.Language.LanguageAbbreviation == this.Lang
                                  & t.BlizzArea.Game.GameAbbreviation == "HEARTHSTONE"
                                  & (t.PostLinkNo.Value != null && (t.PostLinkNo.Value == 1 || t.PostLinkNo.Value % 2 == 0))
                                  select t);

                foreach (var tweet in tweetQueue)
                {
                    string gameHashtag = "#Hearthstone";
                    string gameNormTag = "[#HS]";

                    string tweetedContent = String.Empty;
                    if (tweet.IsForumThread)
                    {
                        string link = "http://www.blizzposts.com/topic/" + tweet.ItemId + "#" + tweet.PostLinkNo;
                        tweetedContent = String.Format("{6} New blue post in \"{0}\" {1} - by {2} ({3}) {4}{5}",
                                                            HttpUtility.HtmlDecode(tweet.Title),
                                                            link,
                                                            tweet.NameOfThePoster.Trim(),
                                                            tweet.BlizzArea.Region.RegionAbbreviation,
                                                            gameHashtag,
                                                            "", //was bluecount - removed
                                                            gameNormTag);
                    }
                    else
                    {
                        string link = "http://www.blizzposts.com/blogentry/" + tweet.ItemId;
                        tweetedContent = String.Format("{4} [Blog {3}-{5}] \"{0}\" - {1} {2}",
                                                 HttpUtility.HtmlDecode(tweet.Title),
                                                 link,
                                                 gameHashtag,
                                                 tweet.BlizzArea.Region.RegionAbbreviation,
                                                 gameNormTag,
                                                 this.Lang);

                    }

                    TwitterService service = new TwitterService(CONSUMER_KEY, CONSUMER_SECRET);
                    service.AuthenticateWith(ACCESS_TOKEN, ACCESS_SECRET);
                    service.SendTweet(new SendTweetOptions { Status = tweetedContent });

                    tweet.TweetResult = service.Response.StatusCode + " - " + service.Response.StatusDescription;
                    tweet.TweetedContent = tweetedContent;
                    tweet.IsTweeted = true;
                    tweet.DateIsTweeted = DateTime.UtcNow;

                    Console.WriteLine("Tweeted content: {0}", tweetedContent);
                    Console.WriteLine("Tweet Result: {0}\n\n", service.Response.StatusCode);

                    db.SubmitChanges();

                }

            }
            catch (Exception e)
            {
                Console.WriteLine("An error has ocurred: \n {0} \n", e.Message);
            }
        }
Exemplo n.º 7
0
        private void worker_DoWork(object sender, DoWorkEventArgs e)
        {
            bool isSuccess = false;
            int failureCount = 0;
            while (!isSuccess)
            {
                try
                {
                    TrElementData trElementData = e.Argument as TrElementData;

                    using (ForumBlogsDataContext entities = new ForumBlogsDataContext(trElementData.ConnStr))
                    {
                        Console.WriteLine("Initiating database connection . . .");

                        int threadCount = (from t in entities.Topics
                                           where t.ThreadNumber == trElementData.threadNumber
                                           && t.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower() == trElementData.Game.ToLower()
                                           && t.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower() == trElementData.Lang.ToLower()
                                           && t.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower() == trElementData.Region.ToLower()
                                           select t).Count();

                        trElementData.isNewThread = !(threadCount > 0);

                        if (trElementData.isNewThread)
                        {
                            Console.WriteLine("\nFound new thread to be marked for indexing. \n");

                            Topic topic = new Topic();
                            topic.ForumBoardId = trElementData.ForumBoardId;
                            topic.ThreadTitle = trElementData.threadTitle;
                            topic.AuthorOfThread = trElementData.threadAuthor;
                            topic.LastPoster = trElementData.lastPoster.Trim();
                            topic.ThreadNumber = trElementData.threadNumber.Value;
                            topic.ToBeCrawled = true; //temporary
                            topic.NoPages = trElementData.lastPage;
                            topic.LastPostDate = DateTime.Now.AddYears(-3);
                            topic.ThreadUpdate = trElementData.lastUpdateStamp;

                            entities.Topics.InsertOnSubmit(topic);
                            entities.SubmitChanges();
                        }
                        else
                        {
                            Topic topic = (from t in entities.Topics
                                           where t.ThreadNumber == trElementData.threadNumber
                                           && t.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower() == trElementData.Game.ToLower()
                                           && t.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower() == trElementData.Lang.ToLower()
                                           && t.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower() == trElementData.Region.ToLower()
                                           select t).ToList()[0];

                            if (topic.ThreadTitle != trElementData.threadTitle)
                            {
                                Console.WriteLine("\nThe thread's title was changed.\n");
                                topic.ThreadTitle = trElementData.threadTitle;
                            }

                            if (topic.ThreadUpdate != trElementData.lastUpdateStamp && !topic.IsBeingCrawled && !topic.ToBeCrawled)
                            {
                                Console.WriteLine("\nFound known thread that was updated. \n");
                                topic.ToBeCrawled = true;
                                topic.NoPages = trElementData.lastPage;
                                topic.LastPoster = trElementData.lastPoster.Trim();
                                topic.ThreadUpdate = trElementData.lastUpdateStamp;
                            }

                            entities.SubmitChanges();

                        }
                    }
                    isSuccess = true;
                    Console.WriteLine("Finished database operation.");
                }
                catch (Exception er)
                {
                    failureCount++;
                    System.Threading.Thread.Sleep(2000);

                    if (failureCount > 10)
                        Console.WriteLine(er.Message);
                }
            }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Gets the title of the specified thread.
        /// </summary>
        /// <param name="id">id of the thread</param>
        /// <returns></returns>
        public string GetTopicTitle(int id)
        {
            var db = new ForumBlogsDataContext(DbConnectionString.Value);

            try
            {
                return (from t in db.Topics
                        where t.Id == id
                        select t.ThreadTitle).Single();
            }
            catch (InvalidOperationException)
            {
                return null;
            }
        }
Exemplo n.º 9
0
        /// <summary>
        /// Gets the language of the specified thread.
        /// </summary>
        /// <param name="id">id of the thread</param>
        /// <returns></returns>
        public string GetTopicLanguage(int id)
        {
            var db = new ForumBlogsDataContext(DbConnectionString.Value);

            try
            {
                return (from t in db.Topics
                        where t.Id == id
                        select t.ForumBoard.BlizzArea.Language.LanguageAbbreviation).Single();
            }
            catch (InvalidOperationException)
            {
                return null;
            }
        }
Exemplo n.º 10
0
        public TopicDo GetTopic(int id)
        {
            TopicDo topic = new TopicDo();
            var topicsDbCont = new ForumBlogsDataContext(DbConnectionString.Value);
            var miscDbCont = new MiscDataContext(DbConnectionString.Value);

            Topic dbTopic = null;
            try
            {
                dbTopic = (from t in topicsDbCont.Topics
                           where t.Id == id
                           select t).Single();
            }
            catch (InvalidOperationException)
            {
                return null;
            }

            topic.TopicId = dbTopic.Id;
            topic.GameAbbrev = dbTopic.ForumBoard.BlizzArea.Game.GameAbbreviation;
            topic.LanguageAbbrev = dbTopic.ForumBoard.BlizzArea.Language.LanguageAbbreviation;
            topic.RegionAbbrev = dbTopic.ForumBoard.BlizzArea.Region.RegionAbbreviation;
            topic.TopicTitle = dbTopic.ThreadTitle;
            topic.RegionFullName = dbTopic.ForumBoard.BlizzArea.Region.RegionName;
            topic.BlizzTopicId = dbTopic.ThreadNumber;

            IQueryable<Post> posts = null;
            IQueryable<MvpPosterBlacklist> blackList = null;

            posts = from p in topicsDbCont.Posts
                    where p.Topic.Id == id
                    orderby p.Id
                    select p;
            blackList = from b in miscDbCont.MvpPosterBlacklists
                        select b;

            List<Post> lstPosts = posts.ToList();
            List<MvpPosterBlacklist> blacklistedMvps = blackList.ToList();
            List<Post> newLstPosts = new List<Post>();

            foreach (var post in lstPosts)
            {
                try
                {
                    string directPostLink = post.DirectPostLink;
                    if (directPostLink[directPostLink.Count() - 1] == '1' && directPostLink[directPostLink.Count() - 2] == '#') //Always include first post, regardless of the blaclist.
                    {
                        //we need to always include the first post, regardless of whether the MVP is blacklisted or not
                        newLstPosts.Add(post);
                    }
                    else if (post.PosterType == 1)
                    {
                        bool isFound = false;
                        if (blacklistedMvps.Count > 0)
                        {
                            foreach (var mvp in blacklistedMvps)
                            {
                                if (post.PosterName.Contains(mvp.PosterName))
                                    isFound = true;
                            }
                        }
                        if (!isFound)
                        {
                            newLstPosts.Add(post);
                        }
                    }
                    else
                    {
                        newLstPosts.Add(post);
                    }
                }
                catch (ArgumentNullException)
                {
                    newLstPosts.Add(post);
                }
            }

            foreach (var post in newLstPosts)
            {
                PostDo postDomainObj = new PostDo();

                postDomainObj.PostId = post.Id;
                postDomainObj.PostContent = post.PostContent;
                postDomainObj.PosterName = post.PosterName;
                postDomainObj.PosterTitle = post.PosterSpecialTitle;
                postDomainObj.RawDateStr = post.PostDate;
                postDomainObj.SourceUri = post.DirectPostLink;
                postDomainObj.Type = post.PosterType.Value;
                postDomainObj.Avatar = post.AvatarLinkOfPost;

                if (post.PostDateTimeUtc.HasValue)
                    postDomainObj.DtUtc = post.PostDateTimeUtc.Value;

                topic.Posts.Add(postDomainObj);

            }
            //For ShowMvpPosts button.
            topic.HasMvpPosts = (topic.Posts.Where(x => x.Type == 1).Count() > 0 && !(topic.Posts.Where(x => x.Type == 1).Count() == 1 && topic.Posts[0].Type == 1));

            return topic;
        }
Exemplo n.º 11
0
        private void CrawlSpecifiedTopics(Topic topic)
        {
            //We are going to make sure page 1 is always crawled.
            List<int> pageNos = new List<int>();
            pageNos.Add(1);
            if (!topic.LastCrawledPage.HasValue)
            {
                topic.LastCrawledPage = 1;
            }

            for (int i = topic.LastCrawledPage.Value; i <= topic.NoPages; i++)
            {
                if (i != 1)
                {
                    pageNos.Add(i);
                }
            }

            int lastCrawledPage = 0;

            for (int i = 0; i <= pageNos.Count - 1; i++)
            {
                string ht = WebClient.GetRawHtml(String.Format("http://{0}.battle.net/{1}/{2}/forum/topic/{3}?page={4}",
                                                   topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower(),
                                                   topic.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower(),
                                                   topic.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower(),
                                                   topic.ThreadNumber,
                                                   pageNos[i]));
                HtmlDocument document = new HtmlDocument();
                document.LoadHtml(ht);

                var divTags = document.GetElementbyId("post-list").ChildNodes.Where(x => x.Attributes["class"] != null && x.Attributes["class"].Value.Contains("topic-post"));

                foreach (var div in divTags)
                {
                    bool indexThisPost = false;
                    string postContent = null;
                    string avatarLink = null;
                    string linkDirectPost = null;
                    string postDate = null;
                    string posterName = null;
                    string posterSpecialTitle = null;
                    string postEdited = null;
                    short posterType = 0;

                    if (div.HasChildNodes)
                    {

                        if (div.InnerHtml.Contains("<span id=\"1\">")) //incredibly fragile
                        {
                            indexThisPost = true;
                        }

                        if (div.Attributes["class"].Value.Contains("blizzard"))
                        {
                            indexThisPost = true;
                            posterType = 2;
                        }
                        else if (div.Attributes["class"].Value.Contains("mvp"))
                        {
                            indexThisPost = true;
                            posterType = 1;
                        }

                        if (indexThisPost)
                        {
                            #region Post Content
                            //*[@id="post-64426546361"]/div[1]/table/tr/td[2]/div
                            var postContentTag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[2]/div", div.Id));

                            if (postContentTag != null)
                            {
                                postContent = String.Empty;

                                postContent += postContentTag.InnerHtml;
                            }
                            #endregion

                            #region avatar

                            // //*[@id="post-79781202523"]

                            var avatarImgTag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[1]/div/div/a/img", div.Id));

                            if (avatarImgTag != null)
                            {
                                avatarLink = avatarImgTag.OuterHtml;

                                string src = avatarLink.Remove(0, avatarLink.IndexOf("src") + 1);
                                src = src.Remove(0, src.IndexOf("\"") + 1);
                                src = src.Remove(src.IndexOf("\""));

                                if (!src.Contains("battle.net") && !src.Contains("cms/user_avatar") && !src.Contains("media.blizzard")) //TODO: Second condition here might conflic in other regions. Check for this.
                                {
                                    src = String.Format("http://{0}.battle.net/", topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower()) + src;
                                }

                                avatarLink = String.Format("<img alt=\"avatar\" src = \"{0}\" width={1} height={2}", src, "{0}", "{1} />");
                            }

                            #endregion

                            #region Direct Post Link

                            var indexATag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[3]/div/a", div.Id));

                            if (indexATag == null) //deleted MVP post
                            {
                                indexATag = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div[1]/table/tr/td[3]/div/a", div.Id));
                            }

                            if (indexATag != null)
                            {
                                linkDirectPost = String.Format("http://{0}.battle.net/{1}/{2}/forum/topic/{3}?page={4}{5}",
                                                               topic.ForumBoard.BlizzArea.Region.RegionAbbreviation.ToLower(),
                                                               topic.ForumBoard.BlizzArea.Game.GameAbbreviation.ToLower(),
                                                               topic.ForumBoard.BlizzArea.Language.LanguageAbbreviation.ToLower(),
                                                               topic.ThreadNumber,
                                                               pageNos[i],
                                                               indexATag.Attributes["href"].Value);
                            }

                            #endregion

                            #region Post Date

                            var postDateDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[3]/div/div[1]", div.Id));

                            if (postDateDiv != null)
                            {
                                postDate = postDateDiv.Attributes["data-tooltip"].Value.Replace("&nbsp;", " ");
                            }

                            #endregion

                            #region Poster Name

                            var posterNameSpan = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[2]/div[1]/a/span", div.Id));

                            //*[@id="post-66887172203"]/div[1]/table/tr/td[1]/div/div/div/a/span
                            if (posterNameSpan == null)
                            {
                                posterNameSpan = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div[1]/table/tr/td[1]/div/div/div/a/span", div.Id));
                            }

                            if (posterNameSpan != null)
                            {
                                posterName = posterNameSpan.InnerText;
                            }

                            #endregion

                            #region Poster Special Title

                            if (posterType == 2)
                            {
                                var posterSpecialTitleDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[1]/div/div[2]/div[2]", div.Id));

                                if (posterSpecialTitleDiv != null)
                                {
                                    posterSpecialTitle = posterSpecialTitleDiv.InnerText;
                                }
                            }

                            #endregion

                            #region Post Edited

                            var postEditedDiv = document.DocumentNode.SelectSingleNode(String.Format("//*[@id=\"{0}\"]/div/table/tr/td[2]/div[2]", div.Id));

                            if (postEditedDiv != null)
                            {
                                postEdited = postEditedDiv.OuterHtml;
                            }
                            #endregion

                            //Combining post edited with post content.
                            if (!String.IsNullOrEmpty(postEdited))
                            {
                                postContent += postEdited;
                            }

                            if (postContent != null)
                                postContent = postContent.Replace("<br>", "<br />");

                            //Does this post exist? Assume false for now.
                            bool postExists = false;

                            //Check if the post exists. If it does exist, make sure there weren't any updates on it.
                            using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
                            {
                                int postCount = (from p in entities.Posts
                                                 where p.DirectPostLink == linkDirectPost
                                                 select p).Count();

                                postExists = (postCount > 0);

                            }

                            if (postExists)
                            {
                                using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
                                {
                                    Post post = (from p in entities.Posts
                                                 where p.DirectPostLink == linkDirectPost
                                                 select p).ToList()[0];

                                    if (post.PostContent != postContent || post.AvatarLinkOfPost != avatarLink)
                                    {
                                        post.PostContent = postContent;
                                        post.AvatarLinkOfPost = avatarLink;
                                    }

                                    entities.SubmitChanges();
                                }
                            }
                            else
                            {
                                using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
                                {
                                    if (postContent != null)
                                    {
                                        Post post = new Post();
                                        post.TopicId = topic.Id;
                                        post.DirectPostLink = linkDirectPost;
                                        post.PosterName = posterName;
                                        post.PostDate = postDate;
                                        post.PosterType = posterType;
                                        post.PosterSpecialTitle = posterSpecialTitle;
                                        post.AvatarLinkOfPost = avatarLink;
                                        post.PostContent = postContent;

                                        try
                                        {
                                            SetPostDate(post, topic.ForumBoard.BlizzArea.Region.RegionAbbreviation);
                                        }
                                        catch (Exception) //This will happen if time zone is not specified
                                        {
                                        }

                                        //if (post.PosterType == 2)
                                        //{
                                        //    topic.LastPostDate = post.PostDateTimeUtc.Value;
                                        //}

                                        entities.Posts.InsertOnSubmit(post);
                                        entities.SubmitChanges();
                                    }

                                }
                            }

                        }
                    }
                }

                lastCrawledPage = pageNos[i];

            }

            using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
            {
                Topic topicToSave = (from t in entities.Topics
                                     where t.Id == topic.Id
                                     select t).Single();

                var lastBlue = (from p in entities.Posts
                                where p.TopicId == topicToSave.Id
                                && p.PosterType == 2
                                orderby p.PostDateTimeUtc descending
                                select p).ToList();

                if (lastBlue.Count() > 0)
                    topicToSave.LastPostDate = lastBlue[0].PostDateTimeUtc.Value;

                topicToSave.LastCrawledPage = lastCrawledPage;
                topicToSave.IsBeingCrawled = false;
                topicToSave.ToBeCrawled = false;
                entities.SubmitChanges();
            }
        }
Exemplo n.º 12
0
        public Domain.BlogEntryDo GetBlogEntry(int id)
        {
            BlogEntryDo blogEntry = new BlogEntryDo();
            var db = new ForumBlogsDataContext(DbConnectionString.Value);

            try
            {
                var entry = (from b in db.BlogEntries
                             where b.Id == id
                             select b).Single();

                blogEntry.Id = entry.Id;
                blogEntry.EntryNumber = entry.EntryNumber;
                blogEntry.AnchoredLink = entry.BlogDirectLink;
                blogEntry.BlogContent = entry.BlogContent;
                blogEntry.BlogPostDate = entry.BlogDate;
                //blogEntry.GameFullName = entry.BlizzArea.Game.GameName;
                blogEntry.GameAbbrev = entry.BlizzArea.Game.GameAbbreviation;
                blogEntry.RegionAbbrev = entry.BlizzArea.Region.RegionAbbreviation;
                blogEntry.RegionFullName = entry.BlizzArea.Region.RegionName;
                blogEntry.Title = entry.BlogTitle;
                blogEntry.LanguageAbbrev = entry.BlizzArea.Language.LanguageAbbreviation;

            }
            catch (InvalidOperationException)
            {
                return null;
            }

            return blogEntry;
        }
Exemplo n.º 13
0
        static void Main(string[] args)
        {
            while (true)
            {
                try
                {
                    using (TrackerDataContext db = new TrackerDataContext(_connStr))
                    {
                        var trackerItemsNeedingUpdates = (from t in db.TrackerTableListItems
                                                          where t.IsPstSummaryUpdateRequired == null
                                                          || t.IsPstSummaryUpdateRequired.Value == true
                                                          select t).ToList(); //Need to get them all in a list, because of a weird bug in the ORM.
                        foreach (var item in trackerItemsNeedingUpdates)
                        {
                            if (item.EntryType == 0) //ForumBoards
                            {
                                using (ForumBlogsDataContext postsContext = new ForumBlogsDataContext(_connStr))
                                {
                                    try
                                    {
                                        var desiredPost = (from p in postsContext.Posts
                                                           where p.TopicId == item.BlogOrTopicId
                                                           && p.DirectPostLink.EndsWith("#1")
                                                           select p).Single();

                                        item.FirstPostSummary = GenerateSummary(desiredPost.PostContent, true);
                                        item.IsPstSummaryUpdateRequired = false;
                                    }
                                    catch
                                    {
                                        HandleFailure(item);
                                    }
                                    db.SubmitChanges();
                                }
                            }
                            else if (item.EntryType == 1) //Blog entries
                            {
                                using (ForumBlogsDataContext blogEntriesContext = new ForumBlogsDataContext(_connStr))
                                {
                                    try
                                    {
                                        var desiredPost = (from p in blogEntriesContext.BlogEntries
                                                           where p.Id == item.BlogOrTopicId
                                                           select p).Single();

                                        item.FirstPostSummary = GenerateSummary(desiredPost.BlogContent, false);
                                        item.IsPstSummaryUpdateRequired = false;
                                    }
                                    catch
                                    {
                                        HandleFailure(item);
                                    }
                                    db.SubmitChanges();
                                }
                            }
                            else if (item.EntryType == 2) //Twitter conversations
                            {
                                using (TwitterDataContext twitterContext = new TwitterDataContext(_connStr))
                                {
                                    try
                                    {
                                        var desiredPost = (from p in twitterContext.Tweets
                                                           where p.TwitterConversationId == item.BlogOrTopicId
                                                           orderby p.Id ascending
                                                           select p).Take(1).Single();

                                        item.FirstPostSummary = desiredPost.TweetContentNonHtml;
                                        item.IsPstSummaryUpdateRequired = false;
                                    }
                                    catch
                                    {
                                        HandleFailure(item);
                                    }
                                    db.SubmitChanges();
                                }
                            }
                            else
                            {
                                throw new NotImplementedException("EntryType");
                            }
                        }
                    }
                }
                catch { }

                //Job should run once per minute.
                Console.WriteLine("\nJob done. Sleeping for a minute. - " + DateTime.Now + "\n");
                System.Threading.Thread.Sleep(30 * 1000);
            }
        }
Exemplo n.º 14
0
        public TrackerDo GetTrackerByPosterName(string posterName, int page)
        {
            if (posterName == null)
                throw new ArgumentNullException("posterName");

            TrackerDo tracker = new TrackerDo();
            var dbTracker = new TrackerDataContext(DbConnectionString.Value);
            var dbForumBlogs = new ForumBlogsDataContext(DbConnectionString.Value);

            var threadQuery = (from p in dbForumBlogs.Posts
                               where p.PosterName == posterName
                               && p.PosterType == 2
                               select p.Topic).Distinct().OrderByDescending(x => x.LastPostDate);

            List<int> threadsPostedIn = threadQuery.Skip((page - 1) * 50).Take(50).Select(x => x.Id).ToList();

            var trackerQuery = from t in dbTracker.TrackerTableListItems
                               where threadsPostedIn.Contains(t.BlogOrTopicId)
                               && t.EntryType == 0
                               orderby t.TimeOfLatestPost descending
                               select t;

            List<TrackerTableListItem> trackerListItems = (trackerQuery).ToList();

            tracker.TrackerItems = GetItemsDomainObjects(trackerListItems);
            tracker.PageTitle = posterName;
            tracker.TotalItems = threadQuery.Count();

            return tracker;
        }
Exemplo n.º 15
0
        public override void PerformTwitterOperation()
        {
            Console.Write("Starting... ({0})", this.Lang);

            ForumBlogsDataContext db = new ForumBlogsDataContext(_connStr);

            try
            {
                var tweetQueue = (from t in db.TweetQueues
                                  where t.DateAdded > (DateTime.UtcNow.AddMinutes(-120))
                                  && !t.IsTweeted
                                  && t.BlizzArea.Language.LanguageAbbreviation == this.Lang
                                  && t.BlizzArea.Game.GameAbbreviation != "BDA"
                                  select t);

                foreach (var tweet in tweetQueue)
                {
                    string gameHashtag = tweet.BlizzArea.Game.GameAbbreviation;

                    string gameNormTag = String.Empty;
                    if (gameHashtag == "WOW")
                    {
                        gameHashtag = "#Warcraft";
                        gameNormTag = "[#WoW]";
                    }
                    else if (gameHashtag == "D3")
                    {
                        gameHashtag = "#Diablo";
                        gameNormTag = "[#D3]";
                    }
                    else if (gameHashtag == "SC2")
                    {
                        gameHashtag = "#StarCraft";
                        gameNormTag = "[#SC2]";
                    }
                    else if (gameHashtag == "BLIZZCON")
                    {
                        gameHashtag = "#BlizzCon";
                    }
                    else if (gameHashtag == "HEARTHSTONE")
                    {
                        gameHashtag = "#Hearthstone";
                    }
                    else
                    {
                        gameHashtag = "#" + gameHashtag;
                    }
                    string tweetedContent = String.Empty;
                    if (tweet.IsForumThread)
                    {
                        string link = "http://www.blizzposts.com/topic/" + tweet.ItemId + "#" + tweet.PostLinkNo;
                        tweetedContent = String.Format("{6} Новые синие посты в \"{0}\" {1} - автор {2} ({3}) {4}{5}",
                                                            HttpUtility.HtmlDecode(tweet.Title),
                                                            link,
                                                            tweet.NameOfThePoster.Trim(),
                                                            "", //was region - removed
                                                            gameHashtag,
                                                            "", //was bluecount - removed
                                                            gameNormTag);
                    }
                    else
                    {
                        string link = "http://www.blizzposts.com/blogentry/" + tweet.ItemId;
                        tweetedContent = String.Format("{4} [блог {3}{5}] \"{0}\" - {1} {2}",
                                                 HttpUtility.HtmlDecode(tweet.Title),
                                                 link,
                                                 gameHashtag,
                                                 "", //was region - removed
                                                 gameNormTag,
                                                 this.Lang);

                    }

                    TwitterService service = new TwitterService(CONSUMER_KEY, CONSUMER_SECRET);
                    service.AuthenticateWith(ACCESS_TOKEN, ACCESS_SECRET);
                    service.SendTweet(new SendTweetOptions { Status = tweetedContent });

                    tweet.TweetResult = service.Response.StatusCode + " - " + service.Response.StatusDescription;
                    tweet.TweetedContent = tweetedContent;
                    tweet.IsTweeted = true;
                    tweet.DateIsTweeted = DateTime.UtcNow;

                    Console.WriteLine("Tweeted content: {0}", tweetedContent);
                    Console.WriteLine("Tweet Result: {0}\n\n", service.Response.StatusCode);

                    db.SubmitChanges();

                }

            }
            catch (Exception e)
            {
                Console.WriteLine("An error has ocurred: \n {0} \n", e.Message);
            }
        }
Exemplo n.º 16
0
        public void DoBlogCrawling(string game, string region, string lang)
        {
            XmlReader reader = XmlReader.Create(String.Format("http://{0}.battle.net/{1}/{2}/feed/news", region.ToLower(), game.ToLower(), lang.ToLower()));
            SyndicationFeed feed = SyndicationFeed.Load(reader);
            List<MinedBlogEntry> blogEntries = (from item in feed.Items
                                                select new MinedBlogEntry
                                                {
                                                    Title = item.Title.Text,
                                                    PublicationDate = ConvertFromDateTimeOffset(item.PublishDate.ToUniversalTime()),
                                                    DirectLink = item.Links[0].Uri.AbsoluteUri,
                                                    TimeZone = "UTC"
                                                }).ToList();

            foreach (var blogEntry in blogEntries)
            {
                try
                {
                    using (ForumBlogsDataContext entities = new ForumBlogsDataContext(this._connStr))
                    {

                        blogEntry.DirectLinkWithAnchor = String.Format("<a href = \"{0}\"> {1} </a>", blogEntry.DirectLink, blogEntry.Title);
                        blogEntry.EntryNumber = int.Parse(blogEntry.DirectLink.Remove(0, blogEntry.DirectLink.IndexOf("blog/") + 5));

                        int existenceCount = (from b in entities.BlogEntries
                                              where b.BlizzArea.Game.GameAbbreviation == game
                                              && b.BlizzArea.Region.RegionAbbreviation == region
                                              && b.BlizzArea.Language.LanguageAbbreviation == lang
                                              && b.EntryNumber == blogEntry.EntryNumber
                                              select b).Count();

                        if (!(existenceCount > 0))
                        {
                            Console.WriteLine("Found a new blog entry.");

                            string blogEntryPage = WebClient.GetRawHtml(blogEntry.DirectLink);

                            HtmlDocument document = new HtmlDocument();
                            document.LoadHtml(blogEntryPage);

                            //article-content -> we want the contents of this div. Reason why we aren't getting it from feed directly is because we want the banner image.

                            string blogContent = String.Empty;
                            string author = String.Empty;
                            ////*[@id="blog"]/div[2]

                            var blogContentDiv = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div[2]");

                            if (blogContentDiv != null)
                            {
                                blogContent = blogContentDiv.InnerHtml;
                                author = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div[1]/a[1]/span[2]").InnerText;
                            }
                            else
                            {
                                var headingImage = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[2]");
                                blogContentDiv = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[3]");
                                author = document.DocumentNode.SelectSingleNode("//*[@id=\"blog\"]/div/div[1]/div/a").InnerText;

                                if (headingImage != null)
                                {
                                    blogContent += headingImage.OuterHtml;
                                }
                                if (blogContentDiv != null)
                                {
                                    blogContent += blogContentDiv.InnerHtml;
                                }
                                else
                                {
                                    Console.WriteLine("Cannot mine this. The div tags are not in their expected format.");
                                    break;
                                }

                            }

                            blogContent = blogContent.Replace("<br>", "<br />");

                            TimeSpan span = DateTime.UtcNow - blogEntry.PublicationDate;
                            if (span.TotalMinutes > 60)
                            {
                                blogEntry.IsTweeted = true;
                            }

                            BlogEntry newBlogEntry = new BlogEntry();
                            newBlogEntry.BlizzAreaId = this.BlogPageId;
                            newBlogEntry.BlogTitle = blogEntry.Title;
                            newBlogEntry.BlogDirectLink = blogEntry.DirectLinkWithAnchor;
                            newBlogEntry.BlogContent = blogContent;
                            newBlogEntry.BlogDate = blogEntry.PublicationDate;
                            newBlogEntry.TimeZone = "UTC";
                            newBlogEntry.EntryNumber = blogEntry.EntryNumber;
                            newBlogEntry.BlogEntryAuthor = author;

                            entities.BlogEntries.InsertOnSubmit(newBlogEntry);
                            entities.SubmitChanges();
                        }
                        else
                        {
                            Console.WriteLine("Found a blog entry. This blog entry already exists.");
                            //TODO: We should probably check last update stamp.
                        }

                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine("Could not mine blog entry for following reason: \n {0}", e.Message);
                }

            }
        }