public virtual ActionResult FeedsCheck()
{
var badFeeds = new List <FeedContract>();
var baseserver = new BaseServer(_appConfigBiz, _feedBusiness, _feedItemBusiness, _updaterDurationBusiness);
var feeds = _feedBusiness.GetList().Where(f => f.Deleted == DeleteStatus.Temporary && f.UpdatingErrorCount < 7).ToList();
ViewBag.OldCount = feeds.Count;
foreach (var feed in feeds)
{
FeedContract feedContract = new FeedContract();
try
{
feedContract = feed.ToViewModel <FeedContract>();
feedContract = (new ClientUpdater(baseserver, _feedBusiness, _appConfigBiz, true)).FeedUpdateAsService(feedContract, new List <string>());
if (!feedContract.FeedItems.Any())
{
badFeeds.Add(feedContract);
}
feed.Deleted = DeleteStatus.Active;
feed.UpdatingErrorCount = 0;
}
catch (Exception ex)
{
feed.UpdatingErrorCount = (byte)(feed.UpdatingErrorCount + 1);
feedContract.SiteTitle = ex.Message.SubstringETC(0, 200) + (ex.InnerException != null && ex.InnerException.Message != null ? ex.InnerException.Message.SubstringETC(0, 200) : string.Empty);
badFeeds.Add(feedContract);
}
_feedBusiness.Edit(feed);
}
ViewBag.NewCount = feeds.Count - badFeeds.Count;
return(View(badFeeds));
}
private FeedContract getNewItems(FeedContract feedAsService)
{
var insertedItems = new List <FeedItemSP>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
RssFeed feed = RssFeed.Read(feedAsService.Link);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (feed.Channels[0].Items.LatestPubDate() != feed.Channels[0].Items[0].PubDate)
{
RssItems = feed.Channels[0].ItemsSorted;
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = FeedItemsOperation.RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = insertedItems;
return(feedAsService);
}
public List <FeedItem> RssItemCollectionToFeedItemsContract(RssItemCollection items, FeedContract feed)
{
var listReturnBack = new List <FeedItem>();
foreach (RssItem item in items)
{
if (!string.IsNullOrEmpty(feed.LastFeedItemUrl) && feed.LastFeedItemUrl.SubstringX(0, 399).Equals(item.Link.ToString().SubstringX(0, 399)))
{
return(listReturnBack);
}
if (listReturnBack.Any(x => x.Link == item.Link.ToString()))
{
continue;
}
if (!Utility.HasFaWord(item.Title))
{
continue;
}
var itemcontract = new FeedItem();
itemcontract.Title = Helper.HtmlRemoval.StripTagsRegex(item.Title).Replace("\t", "").Replace("\n", "").Replace("\r", "");
itemcontract.Link = item.Link.ToString();
itemcontract.Description = HtmlRemoval.StripTagsRegex(item.Description).Replace("\t", "").Replace("\n", "").Replace("\r", "");
//-------------------------Baray DB koochiK!!-----------------
itemcontract.Description = itemcontract.Description.SubstringX(0, _appConfigBiz.MaxDescriptionLength());
if (item.PubDate.Year > 1350 && item.PubDate < DateTime.Now.AddDays(2))
{
itemcontract.PubDate = item.PubDate;
}
else
{
break;
}
itemcontract.FeedId = feed.Id;
itemcontract.SiteId = feed.SiteId;
itemcontract.SiteUrl = feed.SiteUrl;
itemcontract.SiteTitle = feed.SiteTitle;
listReturnBack.Add(itemcontract);
}
return(listReturnBack);
}
public FeedContract FeedUpdateAsService(FeedContract feedAsService, List <string> listRes)
{
var insertedItems = new List <FeedItem>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
HttpWebRequest request = HttpWebRequest.CreateHttp(feedAsService.Link);
request.Timeout = RequestTimeOut;
var feed = RssFeed.Read(request);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (!feed.Channels[0].Items.LatestPubDate().Equals(feed.Channels[0].Items[0].PubDate))
{
if (feedAsService.Cats.Contains("27"))
{
var RssItemsT = feed.Channels[0].Items;
int count = 0;
foreach (RssItem ritem in RssItemsT)
{
if (count >= 10)
{
break;
}
RssItems.Insert(count++, ritem);
}
}
else
{
RssItems = feed.Channels[0].ItemsSorted;
}
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
//XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom;
WebRequest request = WebRequest.Create(feedAsService.Link);
request.Timeout = RequestTimeOut;
using (WebResponse response = request.GetResponse())
using (XmlReader reader = XmlReader.Create(response.GetResponseStream()))
{
atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = new FeedItemsOperation(_appConfigBiz).RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = listRes != null?insertedItems.Where(x => !listRes.Any(l => l == x.Link)).ToList() : insertedItems;
return(feedAsService);
}
