public RssChannel()
{
XElement = new XElement("channel");
Items = new RssItemCollection(XElement);
GenerateParentElements();
}
/// <summary>Gets index of specified RSS Item in the collection.</summary>
/// <param name="items">Target collection.</param>
/// <param name="item">Target item.</param>
/// <returns>Index if found, -1 otherwise.</returns>
public static int IndexOf(RssItemCollection items, RssItem item)
{
int idx = 0;
foreach (RssItem it in items)
{
if ((item.Guid == null || item.Guid.Name == null || item.Guid.Name.Length <= 0) &&
(it.Guid == null || it.Guid.Name == null || it.Guid.Name.Length <= 0))
{
if (item.Title == it.Title && item.Link == it.Link && item.PubDate == it.PubDate)
{
return(idx);
}
}
else if ((item.Guid != null && it.Guid != null) &&
(item.Guid.Name != null && item.Guid.Name.Length > 0) &&
(item.Guid.Name == it.Guid.Name && item.PubDate == it.PubDate))
{
return(idx);
}
idx++;
}
return(-1);
}
//-----------------console--------------------------
#region feeds
public static RssItemCollection GetRssItemCollection(this SyndicationFeed atomfeed)
{
int i = 0;
List <RssItem> retlist = new List <RssItem>();
RssItemCollection colection = new RssItemCollection();
if (atomfeed.Items.Any())
{
foreach (SyndicationItem item in atomfeed.Items)
{
i++;
break;
}
}
if (i > 0)
{
retlist = atomfeed.Items.Select(x => new RssItem {
Title = x.Title.Text, Description = x.Summary.Text, Link = x.Links[0].Uri, PubDate = x.PublishDate.DateTime
}).ToList();
}
foreach (var item in retlist.OrderBy(x => x.PubDate))
{
colection.Add(item);
}
return(colection);
}
internal RssChannel(XElement element)
{
XElement = element;
Items = new RssItemCollection(XElement);
if (this.Title == null)
{
throw new Exception();
}
}
public void Save(RssItemCollection items, string filePath)
{
if (items == null)
{
throw new ArgumentNullException("items");
}
// mbr - 18-08-2006 - defer...
this.Save(this.CreateChannelFromLegacySettings(items), filePath);
}
public List <FeedItem> InsertItemsSqlLucene(RssItemCollection items, Feed feed)
{
var entiti = new TazehaContext();
List <FeedItem> listReturnBack = new List <FeedItem>();
IRepositorySaver RepositorySql = new SqlRepository();
IRepositorySaver RepositoryLucene = new LuceneRepositoryAsService();
int erroroccur = 0;
foreach (RssItem item in items)
{
if (erroroccur > 2)
{
return(listReturnBack);
}
if (!Utility.HasFaWord(item.Title))
{
continue;
}
FeedItem dbitem = new FeedItem();
dbitem.Title = HtmlRemoval.StripTagsRegex(item.Title).Replace("\t", "").Replace("\n", "").Replace("\r", "");
dbitem.Link = item.Link.ToString();
dbitem.Description = HtmlRemoval.StripTagsRegex(item.Description).Replace("\t", "").Replace("\n", "").Replace("\r", "");
//-------------------------Baray DB koochiK!!-----------------
dbitem.Description = dbitem.Description.SubstringX(0, _appConfigBiz.MaxDescriptionLength());
dbitem.SiteId = feed.SiteId;
dbitem.FeedId = feed.Id;
if (item.PubDate.Year > 1350 && item.PubDate < DateTime.Now.AddDays(2))
{
dbitem.PubDate = item.PubDate;
}
else
{
break;
}
dbitem.CreateDate = DateTime.Now;
//dbitem.Cats = feed.Categories.Select(x => x.Id).ToList();
dbitem.SiteTitle = feed.Site.SiteTitle;
dbitem.SiteUrl = feed.Site.SiteUrl;
dbitem.SiteId = feed.SiteId;
dbitem.IndexedType = dbitem.IndexedType.HasValue ? dbitem.IndexedType + 1 : 1;
var feedItem = new FeedItem {
Link = dbitem.Link, Title = dbitem.Title, PubDate = dbitem.PubDate, CreateDate = DateTime.Now, FeedId = feed.Id
};
if (RepositorySql.AddItem(feedItem))
{
RepositoryLucene.AddItem(dbitem);
listReturnBack.Add(dbitem);
Indexer.Indexer.FirstIndexing(dbitem);
}
}
return(listReturnBack);
}
/// <summary>
/// Gets the collection as a collection of RSS items.
/// </summary>
/// <returns></returns>
public RssItemCollection ToRssItemCollection()
{
RssItemCollection items = new RssItemCollection();
foreach (object entity in this.InnerList)
{
items.Add(((IRssItem)entity).ToRssItem());
}
// return...
return(items);
}
public void Save(RssItemCollection items, TextWriter writer)
{
if (items == null)
{
throw new ArgumentNullException("items");
}
if (writer == null)
{
throw new ArgumentNullException("writer");
}
// defer...
this.Save(this.CreateChannelFromLegacySettings(items), writer);
}
public void Save(RssItemCollection items, Stream stream)
{
if (items == null)
{
throw new ArgumentNullException("items");
}
if (stream == null)
{
throw new ArgumentNullException("stream");
}
// defer...
this.Save(this.CreateChannelFromLegacySettings(items), stream);
}
//public static void InsertFeedItemsRss(object param)
//{
// object[] array = param as object[];
// RssItemCollection items = (RssItemCollection)array[0];
// decimal feedId = (decimal)array[1];
// InsertFeedItems(items, feedId);
//}
public List <FeedItem> RssItemsToFeedItems(RssItemCollection items, Feed feed)
{
var listReturnBack = new List <FeedItem>();
foreach (RssItem item in items)
{
if (!string.IsNullOrEmpty(feed.LastUpdatedItemUrl) && feed.LastUpdatedItemUrl.SubstringX(0, 399).Equals(item.Link.ToString().SubstringX(0, 399)))
{
return(listReturnBack);
}
if (listReturnBack.Any(x => x.Link == item.Link.ToString()))
{
continue;
}
if (!Utility.HasFaWord(item.Title))
{
continue;
}
var dbitem = new FeedItem();
dbitem.Id = Guid.NewGuid();
dbitem.Title = HtmlRemoval.StripTagsRegex(item.Title).Replace("\t", "").Replace("\n", "").Replace("\r", "");
dbitem.Link = item.Link.ToString();
dbitem.Description = HtmlRemoval.StripTagsRegex(item.Description).Replace("\t", "").Replace("\n", "").Replace("\r", "");
//-------------------------Baray DB koochiK!!-----------------
dbitem.Description = dbitem.Description.SubstringX(0, MaxDescLength);
dbitem.SiteId = feed.SiteId;
dbitem.FeedId = feed.Id;
if (item.PubDate.Year > 1350 && item.PubDate < DateTime.Now.AddHours(1))
{
dbitem.PubDate = item.PubDate.AddMinutes(feed.Site.TimeDifference);
}
else
{
break;
}
dbitem.CreateDate = DateTime.Now;
//dbitem.Cats = feed.Categories.Select(x => x.Id).ToList();
dbitem.SiteTitle = feed.Site.SiteTitle;
dbitem.SiteUrl = feed.Site.SiteUrl;
dbitem.SiteId = feed.SiteId;
dbitem.ShowContentType = feed.Site.ShowContentType;
dbitem.IndexedType = dbitem.IndexedType.HasValue ? dbitem.IndexedType + 1 : 1;
var feedItem = new FeedItem {
Link = dbitem.Link, Title = dbitem.Title, PubDate = dbitem.PubDate, CreateDate = DateTime.Now, FeedId = feed.Id
};
listReturnBack.Add(dbitem);
}
return(listReturnBack);
}
private FeedContract getNewItems(FeedContract feedAsService)
{
var insertedItems = new List <FeedItemSP>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
RssFeed feed = RssFeed.Read(feedAsService.Link);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (feed.Channels[0].Items.LatestPubDate() != feed.Channels[0].Items[0].PubDate)
{
RssItems = feed.Channels[0].ItemsSorted;
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = FeedItemsOperation.RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = insertedItems;
return(feedAsService);
}
private RssChannel CreateChannelFromLegacySettings(RssItemCollection items)
{
if (items == null)
{
throw new ArgumentNullException("items");
}
// create...
RssChannel channel = new RssChannel();
channel.Title = _title;
channel.Items.AddRange(items);
// return...
return(channel);
}
public List <FeedItem> RssItemCollectionToFeedItemsContract(RssItemCollection items, FeedContract feed)
{
var listReturnBack = new List <FeedItem>();
foreach (RssItem item in items)
{
if (!string.IsNullOrEmpty(feed.LastFeedItemUrl) && feed.LastFeedItemUrl.SubstringX(0, 399).Equals(item.Link.ToString().SubstringX(0, 399)))
{
return(listReturnBack);
}
if (listReturnBack.Any(x => x.Link == item.Link.ToString()))
{
continue;
}
if (!Utility.HasFaWord(item.Title))
{
continue;
}
var itemcontract = new FeedItem();
itemcontract.Title = Helper.HtmlRemoval.StripTagsRegex(item.Title).Replace("\t", "").Replace("\n", "").Replace("\r", "");
itemcontract.Link = item.Link.ToString();
itemcontract.Description = HtmlRemoval.StripTagsRegex(item.Description).Replace("\t", "").Replace("\n", "").Replace("\r", "");
//-------------------------Baray DB koochiK!!-----------------
itemcontract.Description = itemcontract.Description.SubstringX(0, _appConfigBiz.MaxDescriptionLength());
if (item.PubDate.Year > 1350 && item.PubDate < DateTime.Now.AddDays(2))
{
itemcontract.PubDate = item.PubDate;
}
else
{
break;
}
itemcontract.FeedId = feed.Id;
itemcontract.SiteId = feed.SiteId;
itemcontract.SiteUrl = feed.SiteUrl;
itemcontract.SiteTitle = feed.SiteTitle;
listReturnBack.Add(itemcontract);
}
return(listReturnBack);
}
public void Display(string url)
{
if (string.IsNullOrEmpty(url))
{
throw new ArgumentException("Null RssFeedUrl");
}
RssFeed feed = null;
if (HttpContext.Current.Cache.Get(RSS_CACHE_NAME) == null)
{
feed = RssFeed.Read(url);
HttpContext.Current.Cache.Add(RSS_CACHE_NAME, feed, null, DateTime.Now.AddDays(1),
TimeSpan.Zero, System.Web.Caching.CacheItemPriority.Default, null);
}
else
{
feed = HttpContext.Current.Cache.Get(RSS_CACHE_NAME) as RssFeed;
}
// Limit feed to the first 4 elements:
if ((feed.Channels != null) && (feed.Channels.Count > 0))
{
for (int i = feed.Channels.Count - 1; i > 3; --i)
{
feed.Channels.RemoveAt(i);
}
RssItemCollection items = feed.Channels[0].Items;
if ((items != null) && (items.Count > 0))
{
for (int i = items.Count - 1; i > 3; --i)
{
RssElement element = items[i];
items.RemoveAt(i);
}
}
}
listChannels.DataSource = feed.Channels;
listChannels.DataBind();
}
public XmlDocument Generate(RssItemCollection items)
{
return(this.Generate(this.CreateChannelFromLegacySettings(items)));
}
/// <summary>Does read the target URL.</summary>
/// <param name="state"></param>
public void DoRead(Object state)
{
_mutex.WaitOne();
if (!_activated)
{
_mutex.ReleaseMutex();
return;
}
Log.AddInfo(@"RssWatcher.DoRead() : " + this._targetInfo.Title);
string url = _targetInfo.Url;
#if ZEPTAIR
if (_targetInfo.IsZeptDist)
{
Zeptair.Lib.Common.ConfParam zeptConf = TaskMain.Instance.ZeptConf(false);
if (zeptConf.SpecifyAdminNames &&
zeptConf.AdminNames != null &&
zeptConf.AdminNames.Length > 0)
{
if (url.IndexOf('?') >= 0)
{
url += @"&";
}
else
{
url += @"?";
}
url += @"admins=" + zeptConf.AdminNames;
}
}
#endif
RssFeed feed = null;
try
{
ServicePointManager.ServerCertificateValidationCallback = ThetisCore.Lib.EasyTrustPolicy.CheckValidationResult;
HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(url);
if (_targetInfo.UserName != null && _targetInfo.UserName.Length > 0)
{
webReq.Credentials = new NetworkCredential(_targetInfo.UserName, _targetInfo.Password);
}
if (_lastFeed == null)
{
feed = RssFeed.Read(webReq);
}
else
{
feed = RssFeed.Read(webReq, _lastFeed);
}
_lastFeed = feed;
}
catch (WebException we)
{
Log.AddError(" " + we.Message + "\n" + we.StackTrace);
}
if (feed != null && !feed.Cached)
{
bool modified = false;
ArrayList postArray = new ArrayList();
if (_feedHistory == null)
{
_feedHistory = feed;
foreach (RssChannel channel in feed.Channels)
{
foreach (RssItem item in channel.Items)
{
postArray.Add(InfoItem.Create(item, channel.Title, _targetInfo));
}
}
modified = true;
}
else
{
foreach (RssChannel channel in feed.Channels)
{
int historyIdx = RssHelper.IndexOf(_feedHistory.Channels, channel);
if (historyIdx < 0)
{
_feedHistory.Channels.Add(channel);
foreach (RssItem item in channel.Items)
{
postArray.Add(InfoItem.Create(item, channel.Title, _targetInfo));
}
modified = true;
}
else
{
RssItemCollection historyItems = _feedHistory.Channels[historyIdx].Items;
foreach (RssItem item in channel.Items)
{
if (!RssHelper.Contains(historyItems, item))
{
historyItems.Add(item);
postArray.Add(InfoItem.Create(item, channel.Title, _targetInfo));
modified = true;
}
}
}
}
}
if (modified)
{
_feedHistory.LastModified = feed.LastModified;
_targetInfo.SaveFeedHistory(_feedHistory);
_manager.PostItems(postArray, _targetInfo);
postArray.Clear();
}
postArray = null;
}
_mutex.ReleaseMutex();
}
protected void Page_Load(object sender, EventArgs e)
{
string decodedURL = HttpUtility.HtmlDecode(Request.QueryString["url"]);
string decodedFilterString = HttpUtility.HtmlDecode(Request.QueryString["q"]);
string[] filterStrings = decodedFilterString.Split(new char[] { ' ', '�@' }, StringSplitOptions.RemoveEmptyEntries);
bool checkDescription = Request.QueryString["Description"] != null;
bool checkTitle = true;
if (checkDescription)
{
checkTitle = Request.QueryString["title"] != null;
}
RssFeed feed;
try
{
feed = RssFeed.Read(decodedURL);
}
catch (Exception ex)
{
Response.ContentType = "text/plain";
Response.Write(ex.Message);
Response.End();
return;
}
if (feed.Channels.Count == 0)
{
Response.ContentType = "text/plain";
Response.Write("This feed has no channel.");
Response.End();
return;
}
//Feed��o�^
lock (Application["feedsDT"])
{
FeedsDS.FeedsDTDataTable feedsDT = Application["feedsDT"] as FeedsDS.FeedsDTDataTable;
FeedsDS.FeedsDTRow row = feedsDT.FindByFilterStringFeedURL(decodedFilterString, decodedURL);
if (null == row)
{
// insert
feedsDT.AddFeedsDTRow(decodedURL, decodedFilterString, checkTitle, checkDescription, Request.Url.AbsoluteUri, DateTime.Now);
}
else
{
//update
row.TargetTitle = checkTitle;
row.TargetDescription = checkDescription;
row.ResistDate = DateTime.Now;
}
WriteFeedXml(feedsDT);
}
//main from here
foreach (RssChannel channel in feed.Channels)
{
RssItemCollection newItems = new RssItemCollection();
foreach (RssItem item in channel.Items)
{
StringBuilder targetString = new StringBuilder();
if (checkTitle)
{
targetString.Append(item.Title);
}
if (checkDescription)
{
targetString.Append(item.Description);
}
if (!IsFilteredEntry(targetString.ToString(), filterStrings))
{
newItems.Add(item);
}
}
if (newItems.Count == 0)
{
Response.ContentType = "text/plain";
Response.Write("Channel must contain at least one item.");
Response.End();
return;
}
channel.Items.Clear();
foreach (RssItem item in newItems)
{
channel.Items.Add(item);
}
}
feed.Encoding = feed.Encoding ?? Encoding.UTF8;
Response.ContentType = "text/xml";
feed.Write(Response.OutputStream);
Response.End();
}
bool IsFilteredEntry(string targetString, string[] filterStrings)
{
string targetStringLow = targetString.ToLower();
foreach (string filterString in filterStrings)
{
string filterStringLow = filterString.ToLower();
if (filterStringLow.StartsWith("-"))
{
if (targetStringLow.Contains(filterStringLow.Substring(1)))
{
return(true);
}
}
else
{
if (!targetStringLow.Contains(filterStringLow))
{
return(true);
}
}
}
return(false);
}
void WriteFeedXml(FeedsDS.FeedsDTDataTable feedsDT)
{
string feedsFileName = Path.Combine(Server.MapPath("./"), ConfigurationManager.AppSettings["FeedsFilePath"]);
if (!File.Exists(feedsFileName))
{
Directory.CreateDirectory(Path.GetDirectoryName(feedsFileName));
}
feedsDT.WriteXml(feedsFileName);
}
} // end of class
/// <summary>Gets if the specified RSS Item is in the collection.</summary>
/// <param name="items">Target collection.</param>
/// <param name="item">Target item.</param>
/// <returns>true if found, false otherwise.</returns>
public static bool Contains(RssItemCollection items, RssItem item)
{
return(IndexOf(items, item) >= 0);
}
public FeedContract FeedUpdateAsService(FeedContract feedAsService, List <string> listRes)
{
var insertedItems = new List <FeedItem>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
HttpWebRequest request = HttpWebRequest.CreateHttp(feedAsService.Link);
request.Timeout = RequestTimeOut;
var feed = RssFeed.Read(request);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (!feed.Channels[0].Items.LatestPubDate().Equals(feed.Channels[0].Items[0].PubDate))
{
if (feedAsService.Cats.Contains("27"))
{
var RssItemsT = feed.Channels[0].Items;
int count = 0;
foreach (RssItem ritem in RssItemsT)
{
if (count >= 10)
{
break;
}
RssItems.Insert(count++, ritem);
}
}
else
{
RssItems = feed.Channels[0].ItemsSorted;
}
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
//XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom;
WebRequest request = WebRequest.Create(feedAsService.Link);
request.Timeout = RequestTimeOut;
using (WebResponse response = request.GetResponse())
using (XmlReader reader = XmlReader.Create(response.GetResponseStream()))
{
atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = new FeedItemsOperation(_appConfigBiz).RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = listRes != null?insertedItems.Where(x => !listRes.Any(l => l == x.Link)).ToList() : insertedItems;
return(feedAsService);
}
