public static bool ScanRSSFeeds()
        {
            var fact = RSSFactory.Initilize();
            var feeds = fact.PullAllFeedsToScan(10000, 0);

            for (int i = 0; i < feeds.Count; i++)
            {
                int postCount = 0;
                try
                {
                    if (!String.IsNullOrEmpty(feeds[i].RSSUrl))
                    {
                        XmlReaderSettings settings = new XmlReaderSettings();
                        settings.ProhibitDtd = false;
                        XmlReader reader = XmlReader.Create(feeds[i].RSSUrl, settings);
                        SyndicationFeed feed = SyndicationFeed.Load(reader);
                        string authorName = ServerConfig.DEFAULT_ADMIN_EMAIL_ADMIN;

                        if (!String.IsNullOrEmpty(feeds[i].AuthorUserName))
                        {
                            authorName = feeds[i].AuthorUserName;
                        }

                        var latestPosts = feed.Items.Where(x => x.PublishDate >= feeds[i].LastChecked).ToList();
                        for (int j = 0; j < latestPosts.Count; j++)
                        {
                            try
                            {
                                PostDetail detail = new PostDetail();

                                var p = latestPosts[j];

                                detail.Author = authorName;

                                foreach (SyndicationElementExtension ext in p.ElementExtensions)
                                {
                                    if (ext.GetObject<XElement>().Name.LocalName == "encoded")
                                        detail.Content = ext.GetObject<XElement>().Value;
                                }

                                if (p.Content != null)
                                    detail.Content = (p.Content as TextSyndicationContent).Text;
                                if (p.Summary != null)
                                {
                                    detail.Description = (p.Summary as TextSyndicationContent).Text;
                                    if (String.IsNullOrEmpty(detail.Content))
                                        detail.Content = detail.Description;
                                }


                                detail.Title = p.Title.Text;
                                detail.DateCreated = DateTime.UtcNow.ToString();
                                detail.IsSavedForApproval = true;
                                detail.Tags = new List<TagItem>();
                                detail.Categories = new List<CategoryItem>();
                                detail.HasCommentsEnabled = true;


                                foreach (var cat in feeds[i].Categories)
                                {
                                    detail.Categories.Add(new CategoryItem()
                                    {
                                        Id = cat.CategoryRNId,
                                        Title = Category.GetCategory(cat.CategoryRNId).Title
                                    });
                                }
                                foreach (var tag in feeds[i].Tags)
                                {
                                    detail.Tags.Add(new TagItem()
                                    {
                                        TagName = tag.TagName
                                    });
                                }


                                detail.MainImageUrl = feeds[i].MainImageUrl;
                                detail.InitialImageUrl = feeds[i].InitialImageUrl;
                                detail.FeedId = feeds[i].FeedId;

                                PostRepository repo = new PostRepository();
                                detail = repo.Add(detail, "RNFeedSubmittal");
                                PostManager.AddPostForFeed(detail.Id, feeds[i].FeedId);
                                postCount += 1;
                            }
                            catch (Exception exception)
                            {

                            }
                        }
                    }



                }
                catch (Exception exception)
                {
                    if (!String.IsNullOrEmpty(exception.Message) && !exception.Message.Contains("The operation has timed out") && !exception.Message.Contains(@"c:\windows\system32\inetsrv\NA") && !exception.Message.Contains("Unable to connect to the remote serve") && !exception.Message.Contains("remote name could not be resolved"))
                        ErrorDatabaseManager.AddException(exception, exception.GetType(), additionalInformation: feeds[i].RSSUrl);
                }
                fact.FinishFeedPolling(feeds[i].FeedId, postCount);
            }



            return true;

        }
예제 #2
0
        public void ScanAllPages()
        {

            ILogger log = LoggerManager.GetLogger(GetType().Assembly, this.GetType());
            for (int i = 2528; i < 7776; i++)
            {

                log.Log(GetType(), Level.Info, i, null);

                try
                {

                    WebClient client = new WebClient();
                    string html = client.DownloadString("http://www.derbynews.net/?p=" + i);

                    HtmlDocument doc = new HtmlDocument();
                    //doc.Load("http://www.derbynews.net/?p=" + i);


                    doc.LoadHtml(html);

                    var content = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'blog-wrapper')]");
                    if (content != null)
                    {
                        if (!String.IsNullOrEmpty(content.InnerHtml))
                        {
                            var header = doc.DocumentNode.SelectSingleNode("//h2[contains(@class, 'post-header')]");
                            PostDetail detail = new PostDetail();
                            detail.IsSavedForApproval = true;
                            detail.Tags = new List<TagItem>();
                            detail.Title = header.InnerText.Replace("&#8217;", "").Replace("&#038;","");
                            detail.Author = ServerConfig.DEFAULT_ADMIN_EMAIL_ADMIN;
                            detail.Categories = new List<CategoryItem>();
                            detail.HasCommentsEnabled = true;

                            int imgNumber = 0;

                            var images = doc.DocumentNode.SelectNodes("//img");
                            if (images != null)
                            {
                                foreach (var image in images)
                                {
                                    try
                                    {
                                        string src = image.Attributes["src"].Value;


                                        log.Log(GetType(), Level.Info, src, null);
                                        if (!String.IsNullOrEmpty(src) && !src.Contains("gravatar") && !src.Contains("http://scontent-a.cdninstagram") && !src.Contains("disquscdn"))
                                        {
                                            WebClient dlFile = new WebClient();
                                            dlFile.DownloadFile(src, HttpContext.Current.Server.MapPath(BlogService.GetDirectory("/").FullPath + "/DNNImages/" + i + "-" + imgNumber + VirtualPathUtility.GetExtension(src)));
                                            //dlFile.DownloadFile(src, "C:/Personal/" + i + "-" + imgNumber + VirtualPathUtility.GetExtension(src));
                                        }
                                            var newNodeStr = "<!-- IMG[" + i + "-" + imgNumber + "] -->";
                                            var newNode = HtmlNode.CreateNode(newNodeStr);
                                            doc.DocumentNode.InnerHtml = doc.DocumentNode.InnerHtml.Replace(image.OuterHtml, newNode.InnerHtml);
                                            //image.Remove();
                                        

                                    }
                                    catch (Exception exception)
                                    {

                                    }
                                }
                            }

                            var mainContent = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'main-content')]");


                            detail.Content = mainContent.InnerHtml;

                            var date = doc.DocumentNode.SelectSingleNode("//span[contains(@class, 'date-span')]");
                            //detail.DateCreated = date.InnerText.Replace("\n", "").Replace("\t", "").Trim();
                            var dateLink = date.SelectSingleNode("a");
                            DateTime dateCre = DateTime.UtcNow;
                            var datePulled = dateLink.InnerText;
                            var split = datePulled.Split(' ');
                            string day = split[0];
                            day = day.Replace("rd", "").Replace("th", "").Replace("nd", "").Replace("st", "");
                            datePulled = day + " " + split[1] + " " + split[2];


                            DateTime.TryParse(datePulled, out dateCre);

                            //if (DateTime.TryParse(datePulled, "dd MMMM yyyy", DateTimeStyles.None, out dateCre)) ;
                            detail.DateCreated = dateCre.ToString("yyyy-MM-dd HH:mm");
                            var authorForDNN = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-bio-content')]");
                            var dnnAuthorName = authorForDNN.SelectSingleNode("h4");
                            detail.BottomLineForConstribution = "By " + dnnAuthorName.InnerText + " for Derby News Network";

                            var tagCloud = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'tagcloud')]");
                            var tags = tagCloud.SelectNodes("a");
                            if (tags != null)
                                foreach (var tag in tags)
                                {
                                    detail.Tags.Add(new TagItem()
                                    {
                                        TagName = tag.InnerText
                                    });
                                }
                            detail.Tags.Add(new TagItem()
                            {
                                TagName = "Derby News Network"
                            });


                            detail.MainImageUrl = "http://rollinnews.com/FILES%2f2014%2f09%2f11%2fDerby-News-Network.jpg.axdx";
                            detail.InitialImageUrl = "http://rollinnews.com/FILES%2f2014%2f09%2f11%2fDerby-News-Network1.jpg.axdx";

                            PostRepository repo = new PostRepository();
                            detail = repo.Add(detail, "DNNSubmittal");

                        }
                    }

                }
                catch (Exception exception)
                {

                    log.Log(GetType(), Level.Error, "Error occured", exception);

                }




            }

        }