public static bool ScanRSSFeeds()
{
var fact = RSSFactory.Initilize();
var feeds = fact.PullAllFeedsToScan(10000, 0);
for (int i = 0; i < feeds.Count; i++)
{
int postCount = 0;
try
{
if (!String.IsNullOrEmpty(feeds[i].RSSUrl))
{
XmlReaderSettings settings = new XmlReaderSettings();
settings.ProhibitDtd = false;
XmlReader reader = XmlReader.Create(feeds[i].RSSUrl, settings);
SyndicationFeed feed = SyndicationFeed.Load(reader);
string authorName = ServerConfig.DEFAULT_ADMIN_EMAIL_ADMIN;
if (!String.IsNullOrEmpty(feeds[i].AuthorUserName))
{
authorName = feeds[i].AuthorUserName;
}
var latestPosts = feed.Items.Where(x => x.PublishDate >= feeds[i].LastChecked).ToList();
for (int j = 0; j < latestPosts.Count; j++)
{
try
{
PostDetail detail = new PostDetail();
var p = latestPosts[j];
detail.Author = authorName;
foreach (SyndicationElementExtension ext in p.ElementExtensions)
{
if (ext.GetObject<XElement>().Name.LocalName == "encoded")
detail.Content = ext.GetObject<XElement>().Value;
}
if (p.Content != null)
detail.Content = (p.Content as TextSyndicationContent).Text;
if (p.Summary != null)
{
detail.Description = (p.Summary as TextSyndicationContent).Text;
if (String.IsNullOrEmpty(detail.Content))
detail.Content = detail.Description;
}
detail.Title = p.Title.Text;
detail.DateCreated = DateTime.UtcNow.ToString();
detail.IsSavedForApproval = true;
detail.Tags = new List<TagItem>();
detail.Categories = new List<CategoryItem>();
detail.HasCommentsEnabled = true;
foreach (var cat in feeds[i].Categories)
{
detail.Categories.Add(new CategoryItem()
{
Id = cat.CategoryRNId,
Title = Category.GetCategory(cat.CategoryRNId).Title
});
}
foreach (var tag in feeds[i].Tags)
{
detail.Tags.Add(new TagItem()
{
TagName = tag.TagName
});
}
detail.MainImageUrl = feeds[i].MainImageUrl;
detail.InitialImageUrl = feeds[i].InitialImageUrl;
detail.FeedId = feeds[i].FeedId;
PostRepository repo = new PostRepository();
detail = repo.Add(detail, "RNFeedSubmittal");
PostManager.AddPostForFeed(detail.Id, feeds[i].FeedId);
postCount += 1;
}
catch (Exception exception)
{
}
}
}
}
catch (Exception exception)
{
if (!String.IsNullOrEmpty(exception.Message) && !exception.Message.Contains("The operation has timed out") && !exception.Message.Contains(@"c:\windows\system32\inetsrv\NA") && !exception.Message.Contains("Unable to connect to the remote serve") && !exception.Message.Contains("remote name could not be resolved"))
ErrorDatabaseManager.AddException(exception, exception.GetType(), additionalInformation: feeds[i].RSSUrl);
}
fact.FinishFeedPolling(feeds[i].FeedId, postCount);
}
return true;
}
public void ScanAllPages()
{
ILogger log = LoggerManager.GetLogger(GetType().Assembly, this.GetType());
for (int i = 2528; i < 7776; i++)
{
log.Log(GetType(), Level.Info, i, null);
try
{
WebClient client = new WebClient();
string html = client.DownloadString("http://www.derbynews.net/?p=" + i);
HtmlDocument doc = new HtmlDocument();
//doc.Load("http://www.derbynews.net/?p=" + i);
doc.LoadHtml(html);
var content = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'blog-wrapper')]");
if (content != null)
{
if (!String.IsNullOrEmpty(content.InnerHtml))
{
var header = doc.DocumentNode.SelectSingleNode("//h2[contains(@class, 'post-header')]");
PostDetail detail = new PostDetail();
detail.IsSavedForApproval = true;
detail.Tags = new List<TagItem>();
detail.Title = header.InnerText.Replace("’", "").Replace("&","");
detail.Author = ServerConfig.DEFAULT_ADMIN_EMAIL_ADMIN;
detail.Categories = new List<CategoryItem>();
detail.HasCommentsEnabled = true;
int imgNumber = 0;
var images = doc.DocumentNode.SelectNodes("//img");
if (images != null)
{
foreach (var image in images)
{
try
{
string src = image.Attributes["src"].Value;
log.Log(GetType(), Level.Info, src, null);
if (!String.IsNullOrEmpty(src) && !src.Contains("gravatar") && !src.Contains("http://scontent-a.cdninstagram") && !src.Contains("disquscdn"))
{
WebClient dlFile = new WebClient();
dlFile.DownloadFile(src, HttpContext.Current.Server.MapPath(BlogService.GetDirectory("/").FullPath + "/DNNImages/" + i + "-" + imgNumber + VirtualPathUtility.GetExtension(src)));
//dlFile.DownloadFile(src, "C:/Personal/" + i + "-" + imgNumber + VirtualPathUtility.GetExtension(src));
}
var newNodeStr = "<!-- IMG[" + i + "-" + imgNumber + "] -->";
var newNode = HtmlNode.CreateNode(newNodeStr);
doc.DocumentNode.InnerHtml = doc.DocumentNode.InnerHtml.Replace(image.OuterHtml, newNode.InnerHtml);
//image.Remove();
}
catch (Exception exception)
{
}
}
}
var mainContent = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'main-content')]");
detail.Content = mainContent.InnerHtml;
var date = doc.DocumentNode.SelectSingleNode("//span[contains(@class, 'date-span')]");
//detail.DateCreated = date.InnerText.Replace("\n", "").Replace("\t", "").Trim();
var dateLink = date.SelectSingleNode("a");
DateTime dateCre = DateTime.UtcNow;
var datePulled = dateLink.InnerText;
var split = datePulled.Split(' ');
string day = split[0];
day = day.Replace("rd", "").Replace("th", "").Replace("nd", "").Replace("st", "");
datePulled = day + " " + split[1] + " " + split[2];
DateTime.TryParse(datePulled, out dateCre);
//if (DateTime.TryParse(datePulled, "dd MMMM yyyy", DateTimeStyles.None, out dateCre)) ;
detail.DateCreated = dateCre.ToString("yyyy-MM-dd HH:mm");
var authorForDNN = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'author-bio-content')]");
var dnnAuthorName = authorForDNN.SelectSingleNode("h4");
detail.BottomLineForConstribution = "By " + dnnAuthorName.InnerText + " for Derby News Network";
var tagCloud = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'tagcloud')]");
var tags = tagCloud.SelectNodes("a");
if (tags != null)
foreach (var tag in tags)
{
detail.Tags.Add(new TagItem()
{
TagName = tag.InnerText
});
}
detail.Tags.Add(new TagItem()
{
TagName = "Derby News Network"
});
detail.MainImageUrl = "http://rollinnews.com/FILES%2f2014%2f09%2f11%2fDerby-News-Network.jpg.axdx";
detail.InitialImageUrl = "http://rollinnews.com/FILES%2f2014%2f09%2f11%2fDerby-News-Network1.jpg.axdx";
PostRepository repo = new PostRepository();
detail = repo.Add(detail, "DNNSubmittal");
}
}
}
catch (Exception exception)
{
log.Log(GetType(), Level.Error, "Error occured", exception);
}
}
}