private FeedContract getNewItems(FeedContract feedAsService)
{
var insertedItems = new List <FeedItemSP>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
RssFeed feed = RssFeed.Read(feedAsService.Link);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (feed.Channels[0].Items.LatestPubDate() != feed.Channels[0].Items[0].PubDate)
{
RssItems = feed.Channels[0].ItemsSorted;
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = FeedItemsOperation.RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = insertedItems;
return(feedAsService);
}
public FeedContract FeedUpdateAsService(FeedContract feedAsService, List <string> listRes)
{
var insertedItems = new List <FeedItem>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
HttpWebRequest request = HttpWebRequest.CreateHttp(feedAsService.Link);
request.Timeout = RequestTimeOut;
var feed = RssFeed.Read(request);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (!feed.Channels[0].Items.LatestPubDate().Equals(feed.Channels[0].Items[0].PubDate))
{
if (feedAsService.Cats.Contains("27"))
{
var RssItemsT = feed.Channels[0].Items;
int count = 0;
foreach (RssItem ritem in RssItemsT)
{
if (count >= 10)
{
break;
}
RssItems.Insert(count++, ritem);
}
}
else
{
RssItems = feed.Channels[0].ItemsSorted;
}
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
//XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom;
WebRequest request = WebRequest.Create(feedAsService.Link);
request.Timeout = RequestTimeOut;
using (WebResponse response = request.GetResponse())
using (XmlReader reader = XmlReader.Create(response.GetResponseStream()))
{
atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = new FeedItemsOperation(_appConfigBiz).RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = listRes != null?insertedItems.Where(x => !listRes.Any(l => l == x.Link)).ToList() : insertedItems;
return(feedAsService);
}
//one call is fuctionality for update duration that has many feeds
private Feed UpdatingFeed(long feedId)
{
int NumberOfNewItem = 0;
var entiti = new TazehaContext();
Feed dbfeed = entiti.Feeds.Single <Feed>(x => x.Id == feedId);
dbfeed.UpdatingCount = dbfeed.UpdatingCount == null ? 1 : dbfeed.UpdatingCount + 1;
dbfeed.LastUpdaterVisit = DateTime.Now;
try
{
if (dbfeed.FeedType == 0 || !dbfeed.FeedType.HasValue)
{
#region Feed
RssFeed feed = RssFeed.Read(dbfeed.Link);
//-----------shart check kardane inke feed aslan update shode dar site marja ya kheir------------
if (feed == null)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotWork;
}
else if (feed.Channels.Count > 0)
{
bool Exist = false;
if (Exist)
{
GeneralLogs.WriteLog("NoUpdated(last item exist) feed: " + feedId, TypeOfLog.Info);
}
else
{
//--------Feed has new items-----------
if (feed.Channels.Count > 0)
{
RssChannel channel = (RssChannel)feed.Channels[0];
List <FeedItemSP> listReturnBack;
if (channel.Items.LatestPubDate() != channel.Items[0].PubDate)
{
listReturnBack = FeedItemsOperation.InsertFeedItems(channel.ItemsSorted, dbfeed);
}
else
{
listReturnBack = FeedItemsOperation.InsertFeedItems(channel.Items, dbfeed);
}
GeneralLogs.WriteLog("Updating feed " + dbfeed.Id + " Num:" + listReturnBack.Count + " " + dbfeed.Link, TypeOfLog.OK);
}
}
}
#endregion
}
else if (dbfeed.FeedType.HasValue && dbfeed.FeedType.Value == Common.Share.FeedType.Atom)
{
#region Atom
//-----------------atom feed--------------
XmlReader reader = XmlReader.Create(dbfeed.Link);
SyndicationFeed atomfeed = SyndicationFeed.Load(reader);
int i = 0;
if (atomfeed == null)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotWork;
}
else if (atomfeed.Items.Any())
{
foreach (SyndicationItem item in atomfeed.Items)
{
i++;
break;
}
if (i > 0)
{
List <FeedItem> listReturnBack = FeedItemsOperation.InsertFeedItems(atomfeed.Items, dbfeed);
GeneralLogs.WriteLog("OK updating atom " + dbfeed.Id + " Num:" + listReturnBack.Count + " " + dbfeed.Link);
}
}
#endregion
}
}
catch (Exception ex)
{
#region Exception
if (ex.Message.IndexOf("404") > 0)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotFound;
}
else if (ex.Message.IndexOf("403") > 0)
{
dbfeed.Deleted = Common.Share.DeleteStatus.Forbidden;
}
else if (ex.Message.IndexOfX("timed out") > 0)
{
//------request time out
dbfeed.Deleted = Common.Share.DeleteStatus.RequestTimeOut;
}
//-----------log error-----
if (ex.Message.IndexOfX("Inner Exception") > 0)
{
CrawlerLog.FailLog(dbfeed, ex.InnerException.Message.SubstringX(0, 1020));
}
else
{
GeneralLogs.WriteLog("Info " + ex.Message);
}
#endregion
}
#region LASTFLOW
try
{
CheckForChangeDuration(dbfeed, NumberOfNewItem > 0 ? true : false);
CrawlerLog.SuccessLog(dbfeed, NumberOfNewItem);
entiti.SaveChanges();
}
catch (Exception ex)
{
GeneralLogs.WriteLog(ex.Message);
}
#endregion
return(dbfeed);
}
public int UpdatingFeed(Feed dbfeed, bool saveItems = true)
{
int NumberOfNewItem = 0;
var items = new List <FeedItem>();
var log = new FeedLog()
{
FeedId = dbfeed.Id
};
try
{
if (dbfeed.FeedType == 0 || !dbfeed.FeedType.HasValue)
{
#region Feed
RssFeed feed = RssFeed.Read(dbfeed.Link);
//-----------shart check kardane inke feed aslan update shode dar site marja ya kheir------------
if (feed == null)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotWork;
}
else if (feed.Channels.Count > 0)
{
//--------Feed has new items-----------
if (feed.Channels.Count > 0)
{
RssChannel channel = (RssChannel)feed.Channels[0];
if (channel.Items.LatestPubDate() != channel.Items[0].PubDate)
{
items = new FeedItemsOperation(_appConfigBiz).RssItemsToFeedItems(channel.ItemsSorted, dbfeed);
}
else
{
items = new FeedItemsOperation(_appConfigBiz).RssItemsToFeedItems(channel.Items, dbfeed);
}
//----------Visual Items---------
//NumberOfNewItem = listReturnBack.Count;
//if (NumberOfNewItem > 0)
//{
// if (dbfeed.InIndex == 1)
// {
// if (dbfeed.Site.HasSocialTag.HasValue && dbfeed.Site.HasSocialTag.Value)
// Updater.FeedItemImage.setFeedItemsImage(dbfeed, listReturnBack, 5);
// //else
// // FeedItemsOperation.insertFeedItemsIndex(dbfeed, listReturnBack, 5);
// }
// entiti.SaveChanges();
//}
}
}
#endregion
}
else if (dbfeed.FeedType.HasValue && dbfeed.FeedType.Value == Common.Share.FeedType.Atom)
{
#region Atom
XmlReader reader = XmlReader.Create(dbfeed.Link);
SyndicationFeed atomfeed = SyndicationFeed.Load(reader);
int i = 0;
if (atomfeed == null)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotWork;
}
else if (atomfeed.Items.Any())
{
foreach (SyndicationItem item in atomfeed.Items)
{
i++;
break;
}
if (i > 0)
{
items = new FeedItemsOperation(_appConfigBiz).AtomItemsToFeedItems(atomfeed.Items, dbfeed);
}
}
#endregion
}
if (items.Any())
{
if (dbfeed.Site.HasImage != HasImage.NotSupport)
{
new FeedItemImage(_appConfigBiz).SetItemsImage(items, dbfeed);
}
if (saveItems)
{
NumberOfNewItem = ItemBiz.AddItems(items);
if (NumberOfNewItem > 0)
{
dbfeed.LastUpdatedItemUrl = items.Last().Link.SubstringM(0, 400);
}
}
GeneralLogs.WriteLog("Updating feed " + dbfeed.Id + " Items:" + NumberOfNewItem + " New:" + NumberOfNewItem + " " + dbfeed.Link.Replace("http://", ""), TypeOfLog.OK, typeof(FeedUpdater));
}
}
catch (Exception ex)
{
if (ex.Message.IndexOf("404") > 0)
{
dbfeed.Deleted = Common.Share.DeleteStatus.NotFound;
}
else if (ex.Message.IndexOf("403") > 0)
{
dbfeed.Deleted = Common.Share.DeleteStatus.Forbidden;
}
else if (ex.Message.IndexOfX("timed out") > 0)
{
dbfeed.Deleted = Common.Share.DeleteStatus.RequestTimeOut;
}
else
{
dbfeed.Deleted = Common.Share.DeleteStatus.Temporary;
}
log.HasError = true;
log.Message = "Error:" + (ex.InnerException != null ? ex.InnerException.Message.SubstringM(0, 512) : ex.Message.SubstringM(0, 512));
FeedBiz.CreateFeedLog(log);
var msg = "FeedId " + dbfeed.Id + " " + dbfeed.Link.SubstringM(0, 40) + " Error:" + (ex.InnerException != null ? ex.InnerException.Message.SubstringM(0, 512) : ex.Message.SubstringM(0, 512));
GeneralLogs.WriteLog(msg, TypeOfLog.Error, typeof(FeedUpdater));
FeedBiz.Edit(dbfeed);
return(NumberOfNewItem);
}
dbfeed.LastUpdaterVisit = DateTime.Now;
if (saveItems)
{
FeedBiz.CheckForChangeDuration(dbfeed, NumberOfNewItem > 0 ? true : false);
if (NumberOfNewItem > 0)
{
dbfeed.LastUpdateDateTime = DateTime.Now;
}
}
var res = FeedBiz.Edit(dbfeed);
if (saveItems && res.Status)
{
log.HasError = false;
log.Message = "Successfull Updat Feed " + dbfeed.Title.SubstringM(0, 16) + " " + dbfeed.Link.Replace("http://", "").SubstringM(0, 24);
log.ItemsCount = NumberOfNewItem;
FeedBiz.CreateFeedLog(log);
}
return(saveItems ? NumberOfNewItem : items.Count);
}
