//public static void InsertRSSFeed(RssFeed feed, Site Site)
//{
// RssChannel channel = (RssChannel)feed.Channels[0];
// var entiti = new TazehaContext();
// Feed dbfeed = new Feed();
// dbfeed.CopyRight = feed.Channels[0].Copyright;
// dbfeed.Description = feed.Channels[0].Description.SubstringX(0, 100);
// dbfeed.Link = feed.Url.SubstringX(0, 400);// feed.Channels[0].Link.AbsolutePath;
// dbfeed.Title = feed.Channels[0].Title.SubstringX(0, 148);
// dbfeed.Id = ServiceFactory.Get<IAppConfigBiz>().DefaultSearchDuration();
// dbfeed.Id = Site.Id;
// dbfeed.FeedType = 0;
// dbfeed.CreationDate = DateTime.Now;
// entiti.Feeds.Add(dbfeed);
// entiti.SaveChanges();
// GeneralLogs.WriteLog("OK HasRSS " + Site.SiteUrl);
// try
// {
// ThreadPool.QueueUserWorkItem(FeedItemsOperation.InsertFeedItemsRss, new object[] { channel.Items, dbfeed.Id });
// }
// catch
// { }
//}
public void InsertAtomFeed(SyndicationFeed atomfeed, Site Site)
{
var entiti = new TazehaContext();
var dbfeed = new Feed();
if (atomfeed.Copyright != null)
{
dbfeed.CopyRight = atomfeed.Copyright.Text;
}
if (atomfeed.Description != null)
{
dbfeed.Description = atomfeed.Description.Text.SubstringX(0, 999);
}
dbfeed.Link = atomfeed.BaseUri.ToString();// feed.Channels[0].Link.AbsolutePath;
dbfeed.Title = atomfeed.Title.Text.SubstringX(0, 148);
dbfeed.UpdateDurationId = _appConfigBiz.DefaultSearchDuration();
dbfeed.SiteId = Site.Id;
dbfeed.FeedType = FeedType.Atom;//-----for Atom----------
dbfeed.CreationDate = DateTime.Now;
entiti.Feeds.Add(dbfeed);
entiti.SaveChanges();
GeneralLogs.WriteLog("OK HasAtom " + Site.SiteUrl);
try
{
new FeedItemsOperation(_appConfigBiz).InsertFeedItemsAtom(new object[] { atomfeed.Items, dbfeed.Id });
}
catch { }
}
public bool UpdateFeeds(List <FeedContract> feeds)
{
var ids = feeds.Select(x => x.FeedId).ToList();
var dbFeeds = context.Feeds.Where(x => ids.Any(f => f == x.FeedId));
foreach (var dbfeed in dbFeeds)
{
dbfeed.LastUpdatedItemUrl = feeds.FirstOrDefault(x => x.FeedId == dbfeed.FeedId).LastFeedItemUrl;
if (!string.IsNullOrEmpty(dbfeed.LastUpdatedItemUrl) && feeds.SingleOrDefault(f => f.FeedId == dbfeed.FeedId).FeedItems.Count > 0)
{
dbfeed.UpdatingCount = dbfeed.UpdatingCount == null ? 1 : dbfeed.UpdatingCount + 1;
dbfeed.LastUpdateDateTime = DateTime.Now;
BaseUpdater.CheckForChangeDuration(dbfeed, true);
}
else
{
BaseUpdater.CheckForChangeDuration(dbfeed, false);
}
dbfeed.LastUpdaterVisit = DateTime.Now;
}
context.SaveChanges();
GeneralLogs.WriteLog("UpdateFeeds[AsService] : " + string.Join("[br /]", dbFeeds.Select(x => x.Link)), TypeOfLog.OK);
return(true);
}
public void AutoUpdater()
{
StopUpdater = false;
if (Config.GetConfig("DisableUpdater") == "1")
{
GeneralLogs.WriteLog("Updater is Disable", TypeOfLog.Info);
return;
}
//------Starting after 10 min
//System.Threading.Thread.Sleep(1000 * 60 * 10);
while (!StopUpdater)
{
DateTime datetime = DateTime.Now.AddHours(double.Parse(Config.GetConfig("Hour_UTC")));
if (datetime.Hour >= StartOfEndDate && datetime.Hour < EndOfEndDate)
{
StopUpdater = true;
return;
}
LastHour = datetime;
UpdateIsParting();
var now = DateTime.Now.AddHours(double.Parse(Config.GetConfig("Hour_UTC")));
var delay = now.Subtract(datetime).Minutes;
if (delay > 0 && delay < 20)
{
IsPause = true;
System.Threading.Thread.Sleep(1000 * 60 * (Config.GetTimeInterval() - delay));//1 sec >> 1 min >> 10 min
IsPause = false;
}
}
}
public static void sendEmailByLocalServer(List <string> emailsTo, string subject, string body)
{
MailMessage mail = new MailMessage();
SmtpClient SmtpServer = new SmtpClient("127.0.0.1");//"smtp.live.com"
mail.From = new MailAddress("*****@*****.**", "خبرنامه سایت تازه یاب");
mail.Sender = new MailAddress("*****@*****.**");
mail.Body = body;
mail.Subject = subject;
mail.IsBodyHtml = true;
SmtpServer.Port = 25;
SmtpServer.Host = "www.tazeyab.com";
//SmtpServer.EnableSsl = true;
//SmtpServer.Timeout = 300000;//10min
foreach (var email in emailsTo)
{
if (ValidEmail(email))
{
mail.Bcc.Add(email);
}
}
try
{
SmtpServer.Send(mail);
GeneralLogs.WriteLog(">OK sended Email to " + emailsTo.Count);
}
catch (Exception ex)
{
GeneralLogs.WriteLog(">Error sending email to " + emailsTo.Count + ex.Message);
}
}
public void Optimize()
{
LuceneBase _LuceneRepository = new LuceneSaverRepository();
_LuceneRepository.Optimize();
GeneralLogs.WriteLogInDB("Optimize[AsService] ", TypeOfLog.Info);
}
public string Execution(CommandList command)
{
try
{
if (command == CommandList.StartUpdater)
{
//string remoteUpdater = Config.getConfig<string>("RemoteUpdater");
if (EndPoint.GetType() == typeof(string))
{
var remoteUpdater = EndPoint as string;
GeneralLogs.WriteLog("Poke Client " + remoteUpdater, TypeOfLog.Start);
WebClient client = new WebClient();
client.DownloadData(remoteUpdater + "Execution?command=" + command.ToString());
GeneralLogs.WriteLog("Poke Client", TypeOfLog.OK);
return("Start Updater");
}
else if (EndPoint.GetType().IsClass)
{
(EndPoint as BaseUpdaterClient).Poke();
}
}
}
catch (Exception ex)
{
GeneralLogs.WriteLog(ex.Message, TypeOfLog.Error);
}
return(string.Empty);
}
public static void sendEmailByExternalServer(List <string> emailsTo, string subject, string body, SmtpServers server)
{
MailMessage mail = new MailMessage();
SmtpClient SmtpServer = new SmtpServerConfig(server);
mail.From = new MailAddress("*****@*****.**", "خبرنامه تازه یاب");
mail.Body = body;
mail.Subject = subject;
mail.IsBodyHtml = true;
foreach (var email in emailsTo)
{
if (ValidEmail(email))
{
mail.Bcc.Add(email);
}
}
try
{
SmtpServer.Send(mail);
GeneralLogs.WriteLogInDB(">OK sended Email to " + emailsTo.Count);
}
catch (Exception ex)
{
GeneralLogs.WriteLogInDB(">Error sending email to " + emailsTo.Count + ex.Message);
}
}
public static void TagsTableSaveChanges()
{
try
{
var entiti = new TazehaContext();
var TagsTableMain = entiti.Tags.ToList <Tag>();
TagsTable.ForEach(x => x.RepeatCount = x.RepeatCount == null ? 0 : x.RepeatCount);
TagsTableMain.ForEach(x => x.RepeatCount = x.RepeatCount == null ? 0 : x.RepeatCount);
int NumberOfChanges = 0;
for (int i = 0; i < TagsTableMain.Count; i++)
{
if (TagsTableMain[i].RepeatCount < TagsTable[i].RepeatCount)
{
TagsTableMain[i].RepeatCount = TagsTable[i].RepeatCount;
NumberOfChanges++;
}
}
entiti.SaveChanges();
GeneralLogs.WriteLogInDB(">OK TagsTableSaveChanges NumberOfChanges:" + NumberOfChanges);
}
catch
{
GeneralLogs.WriteLogInDB(">Error TagsTableSaveChanges ");
}
}
public static void SetSitePageRank(int StartIndex = 0)
{
var context = new TazehaContext();
int TopCount = 100;
var sites = context.Sites.Where(x => !x.PageRank.HasValue && !x.IsBlog).OrderBy(x => x.Id).Skip(StartIndex).Take(TopCount).Select(x => new { x.Id, x.SiteUrl }).ToDictionary(x => x.Id, x => x.SiteUrl);
foreach (var site in sites)
{
try
{
string siteUrl = site.Value;
siteUrl = siteUrl.IndexOfX("www.") > -1 || siteUrl.IndexOfX("http://") > -1 ? siteUrl : "www." + siteUrl;
siteUrl = siteUrl.IndexOfX("http://") > -1 ? siteUrl : "http://" + siteUrl;
byte PageRank = GooglePageRank.GetPageRank(siteUrl);
setPageRank(site.Key, PageRank);
GeneralLogs.WriteLog("OK @SetSitePageRank " + siteUrl + " " + PageRank);
}
catch (Exception ex)
{
setPageRank(site.Key, 0);
GeneralLogs.WriteLog("Error @setPageRank SiteId:" + site.Key + " " + ex.Message);
}
}
if (context.Sites.Where(x => x.PageRank == null && !x.IsBlog).OrderBy(x => x.Id).Skip(StartIndex).Count() > 0)
{
SetSitePageRank(StartIndex + TopCount);
}
}
public void SpeedUP(Feed feed)
{
feed.LastUpdateDateTime = DateTime.Now;
if (feed.UpdateSpeed > 5)
{
//------feed ro saritar pooyesh kon(TAghire level)----
var newdurations = _dbContext.Set <UpdateDuration>().Where(x => x.PriorityLevel < feed.UpdateDuration.PriorityLevel);
if (!newdurations.Any())
{
//-------agar be kamtarin baze updater resid---1Hour-----
feed.UpdateSpeed = 0;
}
else
{
UpdateDuration newduration = newdurations.OrderByDescending(x => x.PriorityLevel).First();
feed.UpdateDurationId = newduration.Id;
feed.UpdateSpeed = -4;
GeneralLogs.WriteLogInDB("Change Duration(+) of Feed:" + feed.Id + " Link:" + feed.Link + " NewDuration:" + newduration.Id);
}
}
else
{
feed.UpdateSpeed = feed.UpdateSpeed + 1;
}
GeneralLogs.WriteLog("SpeedUp Feed:" + feed.Id + " Link:" + feed.Link, TypeOfLog.Info);
}
public void AutoUpdater()
{
StopUpdater = false;
if (Config.GetConfig <int>("DisableUpdater") == 1)
{
GeneralLogs.WriteLog("Updater is Disable", TypeOfLog.Info);
return;
}
//------Starting after 10 min
//System.Threading.Thread.Sleep(1000 * 60 * 10);
while (!StopUpdater)
{
DateTime startTime = Config.GetServerNow();
if (startTime.Hour >= StartOfEndDate && startTime.Hour < EndOfEndDate)
{
StopUpdater = true;
return;
}
UpdateIsParting();
var delay = Config.GetServerNow().Subtract(startTime);
if (((delay.Minutes == 0 && delay.Seconds > 0) || delay.Minutes > 0) && delay.Minutes < Config.GetTimeInterval())
{
GeneralLogs.WriteLog("I'm Sleeping for " + (Config.GetTimeInterval() - delay.Minutes) + " minutes", TypeOfLog.End, typeof(FeedUpdater));
IsPause = true;
System.Threading.Thread.Sleep(1000 * 60 * (Config.GetTimeInterval() - delay.Minutes));//1 sec >> 1 min >> 10 min
IsPause = false;
}
}
}
public void UpdateFeedsPerDuration(UpdateDuration duration)
{
var delaytime = TimeSpan.Parse(duration.DelayTime);
var Partnumber = delaytime.Hours * 60 / Config.GetTimeInterval();//20 min intervall
var TopCount = (duration.FeedsCount / Partnumber) != 0 ? (duration.FeedsCount / Partnumber) : (duration.FeedsCount % Partnumber);
var feeds = FeedBiz.GetList().Where(x => x.UpdateDurationId.Value == duration.Id &&
((int)x.Deleted < 1 || (int)x.Deleted > 10)).OrderBy(feed => feed.Id).Skip(duration.StartIndex).Take(TopCount).ToList();
if (TopCount == 0)
{
return;
}
GeneralLogs.WriteLog("Start updating duration " + duration.Code + " Start at:" + duration.StartIndex, TypeOfLog.Start, typeof(FeedUpdater));
foreach (var feed in feeds)
{
GeneralLogs.WriteLog("Feed updating. Id:" + feed.Id, TypeOfLog.Start, typeof(FeedUpdater));
UpdatingFeed(feed);
}
duration.FeedsCount = FeedBiz.GetList().Where(x => x.UpdateDurationId.Value == duration.Id && ((int)x.Deleted < 1 || (int)x.Deleted > 10)).Count();
duration.StartIndex = (duration.StartIndex + TopCount) >= duration.FeedsCount ? 0 : (duration.StartIndex + TopCount);
var res = DurationBiz.Edit(duration);
if (res.Status)
{
GeneralLogs.WriteLog("Duration updated. Id:" + duration.Id + " Start Index: " + duration.StartIndex, TypeOfLog.OK, typeof(FeedUpdater));
}
}
public void AutoUpdateFromServer()
{
StopUpdater = false;
var MaxFeedCountAsService = Config.GetConfig <int>("MaxFeedCountAsService");
//GeneralLogs.WriteLog("Start AutoUpdate as service", TypeOfLog.Start);
while (!StopUpdater)
{
try
{
var feeds = server.getLatestFeeds(MaxFeedCountAsService, IsLocaly);
if (feeds.Count > 0)
{
GeneralLogs.WriteLog(string.Format("feeds.Count {0} > 0 So start FeedsUpdat", feeds.Count), TypeOfLog.Start);
FeedsUpdat(feeds);
}
else
{
StopUpdater = true;
GeneralLogs.WriteLog("End AutoUpdate as service count of feeds ", TypeOfLog.End);
break;
}
}
catch (Exception ex)
{
GeneralLogs.WriteLog("exption getLatestFeeds service: " + ex.Message, TypeOfLog.Error);
StopUpdater = true;
break;
}
}
}
public void NoneStopUpdateFromServer(string DurationCode)
{
var MaxFeedCountAsService = Config.GetConfig <int>("MaxFeedCountAsService");
//GeneralLogs.WriteLog("Start AutoUpdate as service", TypeOfLog.Start);
while (true)
{
if (DateTime.UtcNow.AddHours(Config.GetConfig <double>("UTCDelay")).Hour > Config.GetConfig <int>("StartNightly") &&
DateTime.UtcNow.AddHours(Config.GetConfig <double>("UTCDelay")).Hour < Config.GetConfig <int>("EndNightly"))
{
return;
}
try
{
var feeds = server.getLatestFeedsByDuration(DurationCode, MaxFeedCountAsService, false);
if (feeds.Count > 0)
{
GeneralLogs.WriteLog(string.Format("feeds.Count {0} > 0 So start FeedsUpdat", feeds.Count), TypeOfLog.Start);
FeedsUpdat(feeds);
System.Threading.Thread.Sleep(1000 * 2);
}
else
{
GeneralLogs.WriteLog("End AutoUpdate as service count of feeds ", TypeOfLog.End);
break;
}
}
catch (Exception ex)
{
GeneralLogs.WriteLog("exption getLatestFeeds service: " + ex.Message, TypeOfLog.Error);
}
}
}
private string AddItemsToDataBase(List <FeedItem> Items)
{
var res = string.Empty;
try
{
using (var con = new SqlConnection(ConfigurationManager.ConnectionStrings["TazehaContext"].ConnectionString))
{
con.Open();
using (var cmd = new SqlCommand("exec sp_FeedItemsCreate @list", con))
{
var table = new DataTable();
table.Columns.Add("FeedItemId", typeof(string));
table.Columns.Add("Title", typeof(string));
table.Columns.Add("Link", typeof(string));
table.Columns.Add("HasPhoto", typeof(bool));
table.Columns.Add("Description", typeof(string));
table.Columns.Add("PubDate", typeof(DateTime));
table.Columns.Add("FeedId", typeof(int));
table.Columns.Add("SiteId", typeof(long));
table.Columns.Add("SiteUrl", typeof(string));
table.Columns.Add("SiteTitle", typeof(string));
table.Columns.Add("VisitsCount", typeof(int));
table.Columns.Add("ShowContentType", typeof(byte));
foreach (var item in Items)
{
table.Rows.Add(item.Id,
item.Title.SubstringM(0, 256).ApplyUnifiedYeKe(),
HttpUtility.UrlDecode(item.Link.SubstringM(0, 256)).ApplyUnifiedYeKe(),
item.HasPhoto,
item.Description.SubstringM(0, 512),
item.PubDate,
item.FeedId,
item.SiteId,
item.SiteUrl.SubstringM(0, 64),
item.SiteTitle.SubstringM(0, 64).ApplyUnifiedYeKe(),
item.VisitsCount,
item.ShowContentType);
}
var pList = new SqlParameter("@list", SqlDbType.Structured);
pList.TypeName = "dbo.FeedItemDictionary1";
pList.Value = table;
cmd.Parameters.Add(pList);
res = cmd.ExecuteScalar().ToString();
if (!string.IsNullOrEmpty(res))
{
res = res.Remove(0, 1);
}
}
con.Close();
}
}
catch (Exception ex)
{
GeneralLogs.WriteLogInDB(ex, typeof(FeedItemBusiness));
}
return(res);
}
protected void Application_End()
{
try
{
LuceneBase _LuceneRepository = new LuceneSaverRepository();
_LuceneRepository.Optimize();
GeneralLogs.WriteLogInDB(">INFO application ending" + DateTime.Now.ToString("yyyMMddHH"));
}
catch { }
}
private void AfterOneCall(StartUp inputParams, UpdateDuration duration)
{
#region NoIsParting
TazehaContext context = new TazehaContext();
int ItemCountPriorityCode = context.Feeds.Where <Feed>(x => x.UpdateDurationId.Value == duration.Id && x.Site.IsBlog == inputParams.IsBlog && (x.Deleted == 0 || (int)x.Deleted > 10)).Count();
int LastCount = context.Feeds.Where <Feed>(x => x.UpdateDurationId.Value == duration.Id && x.Site.IsBlog == inputParams.IsBlog && (x.Deleted == 0 || (int)x.Deleted > 10)).OrderBy(x => x.Id).Skip(inputParams.StartIndex).Take <Feed>(inputParams.TopCount).Count();
int NextCount = context.Feeds.Where <Feed>(x => x.UpdateDurationId.Value == duration.Id && x.Site.IsBlog == inputParams.IsBlog && (x.Deleted == 0 || (int)x.Deleted > 10)).OrderBy(x => x.Id).Skip(inputParams.StartIndex + inputParams.TopCount).Take <Feed>(inputParams.TopCount).Count();
if (NextCount > 0)
{
var LastUpdateIndex = context.ProjectSetups.SingleOrDefault(x => x.Title == "LastUpdat:" + inputParams.StartUpConfig);
if (LastUpdateIndex != null)
{
//LastUpdateIndex.Value = (InputParams.StartIndex + InputParams.TopCount + InputParams.TopCount).ToString();
LastUpdateIndex.Value = (inputParams.StartIndex + inputParams.TopCount).ToString();
context.SaveChanges();
}
else
{
//entiti.ProjectSetup.AddObject(new ProjectSetup { Title = "LastUpdat:" + InputParams.StartUpConfig, Value = (InputParams.StartIndex + InputParams.TopCount + InputParams.TopCount).ToString(), Meaning = "last index updater of feed of priority" });
context.ProjectSetups.Add(new ProjectSetup {
Title = "LastUpdat:" + inputParams.StartUpConfig, Value = (inputParams.StartIndex + inputParams.TopCount).ToString(), Meaning = "last index updater of feed of priority"
});
context.SaveChanges();
}
inputParams.StartIndex += inputParams.TopCount;
if (!duration.IsParting.HasValue || !duration.IsParting.Value)
{
StartByDuration(inputParams, null, 0);
}
}
else
{
//-----------------------When all items updated------------------
if (duration != null)
{
var LastUpdateIndex = context.ProjectSetups.SingleOrDefault(x => x.Title == "LastUpdat:" + inputParams.StartUpConfig);
if (LastUpdateIndex != null)
{
LastUpdateIndex.Value = "0";
context.SaveChanges();
}
else
{
context.ProjectSetups.Add(new ProjectSetup {
Title = "LastUpdat:" + inputParams.StartUpConfig, Value = "0", Meaning = "last index updater of feed of priority"
});
context.SaveChanges();
}
GeneralLogs.WriteLogInDB(">OK UpdaterSleeping... duration:" + duration.Code);
///for test task with windows----
}
}
#endregion
}
public virtual ActionResult Start(string param)
{
var status = Mn.NewsCms.Common.Updater.BaseUpdater.UpdatersIsRun();
if (!status.HasFlag(UpdaterList.UpdaterClient))
{
GeneralLogs.WriteLog("status != UpdaterList.UpdaterClient", TypeOfLog.Info);
AppUpdater.RunServerWithClientUpdater(_appConfigBiz, _feedBusiness, _feedItemBusiness, _updaterDurationBusiness);
}
//AppUpdater.RunServerWithClientUpdater();
return(View());
}
public virtual ActionResult ClearCache(bool indb)
{
if (indb)
{
_logsBusiness.DeleteAll();
}
else
{
GeneralLogs.ClearCache();
}
return(RedirectToAction(MVC.Dashboard.Logs.Index(indb)));
}
private bool UpdateItemFeeds(IEnumerable <FeedContract> feeds)
{
#region send items
List <FeedItemSP> items = feeds.SelectMany(feed => feed.FeedItems.Select(item => new FeedItemSP
{
Cats = !string.IsNullOrEmpty(feed.Cats) ? feed.Cats.Split(' ').Select(x => int.Parse(x)) : null,
CreateDate = DateTime.Now,
Description = item.Description,
Link = item.Link,
PubDate = item.PubDate,
FeedId = feed.Id,
SiteId = feed.SiteId,
SiteTitle = feed.SiteTitle,
SiteUrl = feed.SiteUrl,
Title = item.Title
})).ToList();
IRepositorySaver saver = new LuceneSaverRepository();
saver.AddItems(items);
NumberOfNewItemsToday += items.Count;
if (NumberOfNewItemsToday > 100)
{
Optimize();
}
#endregion
#region UpdateFeed
var ids = feeds.Select(x => x.Id).ToList();
var dbFeeds = Context.Feeds.Where(x => ids.Any(f => f == x.Id));
foreach (var dbfeed in dbFeeds)
{
dbfeed.LastUpdatedItemUrl = feeds.FirstOrDefault(x => x.Id == dbfeed.Id).LastFeedItemUrl;
if (!string.IsNullOrEmpty(dbfeed.LastUpdatedItemUrl))
{
dbfeed.UpdatingCount = dbfeed.UpdatingCount == null ? 1 : dbfeed.UpdatingCount + 1;
dbfeed.LastUpdaterVisit = DateTime.Now;
CheckForChangeDuration(dbfeed, true);
}
else
{
CheckForChangeDuration(dbfeed, false);
}
dbfeed.LastUpdaterVisit = DateTime.Now;
}
Context.SaveChanges();
GeneralLogs.WriteLog("UpdateFeeds[" + UpdaterName + "] : " + string.Join("[br /]", dbFeeds.Select(x => x.Link)), TypeOfLog.OK);
#endregion
return(true);
}
//[OutputCache(Duration = 100)]
public virtual ActionResult Start(string pass)
{
if (pass != "777")
{
ViewBag.UpdaterStatus = "Password incorrect!";
return(View());
}
var subc = new TimeSpan(0, 10, 0);
if (HttpContext.Application["LastStartDateTime"] == null || DateTime.Parse(HttpContext.Application["LastStartDateTime"].ToString()).AddMinutes(10) < DateTime.Now)
{
HttpContext.Application["LastStartDateTime"] = DateTime.Now;
if (DateTime.Now.NowHour() > _appConfigBiz.GetConfig <int>("StartNightly") &&
DateTime.Now.NowHour() < _appConfigBiz.GetConfig <int>("EndNightly"))
{
//-----Nightly-----
LuceneBase.TodayItemsCount = 0;
ViewBag.UpdaterStatus = "Nightly";
}
else
{
try
{
//-------Daily----
Mn.NewsCms.Common.EventsLog.GeneralLogs.WriteLog("-----Start Updater Ping----", TypeOfLog.Start);
var status = Mn.NewsCms.Common.Updater.BaseUpdater.UpdatersIsRun();
if (!status.HasFlag(UpdaterList.UpdaterClient))
{
Mn.NewsCms.Common.EventsLog.GeneralLogs.WriteLog("status != UpdaterList.UpdaterClient", TypeOfLog.Info);
AppUpdater.RunServerWithClientUpdater(_appConfigBiz, _feedBusiness, _feedItemBusiness, _updaterDurationBusiness);
ViewBag.UpdaterStatus = "Now Start";
}
else
{
ViewBag.UpdaterStatus = "Is Running...";
}
}
catch (Exception ex)
{
GeneralLogs.WriteLogInDB("UpdaterController " + ex.Message, TypeOfLog.Error);
}
}
}
else
{
ViewBag.UpdaterStatus = "Is Running";
}
return(View());
}
public void AddItems(List <FeedItem> items)
{
//ServiceFactory<IBaseService>.Create().SendFeedItems(items);
base.AddFeedItems(items);
if (++CallOptimize > 5)
{
//ServiceFactory<IBaseService>.Create().Optimize();
base.Optimize();
CallOptimize = 0;
GeneralLogs.WriteLog("Optimize data" + DateTime.Now, TypeOfLog.OK);
//-----------Save tags changes---------
// Indexer.Indexer.TagsTableSaveChanges();
}
}
private FeedContract getNewItems(FeedContract feedAsService)
{
var insertedItems = new List <FeedItemSP>();
RssItemCollection RssItems = new RssItemCollection();
if (!feedAsService.IsAtom)
{
RssFeed feed = RssFeed.Read(feedAsService.Link);
if (feed == null)
{
feedAsService.IsNull = true;
}
if (feed.Channels[0].Items.LatestPubDate() != feed.Channels[0].Items[0].PubDate)
{
RssItems = feed.Channels[0].ItemsSorted;
}
else
{
RssItems = feed.Channels[0].Items;
}
}
else
{
XmlReader reader = XmlReader.Create(feedAsService.Link);
SyndicationFeed atom = SyndicationFeed.Load(reader);
if (atom == null)
{
feedAsService.IsNull = true;
}
RssItems = atom.GetRssItemCollection();
}
//--------Feed has new items-----------
if (RssItems.Count > 0)
{
insertedItems = FeedItemsOperation.RssItemCollectionToFeedItemsContract(RssItems, feedAsService);
if (insertedItems.Count() > 0)
{
feedAsService.LastFeedItemUrl = insertedItems[0].Link.SubstringX(0, 399);// RssItems[0].Link.ToString();
}
GeneralLogs.WriteLog("OK updating feed " + feedAsService.Id + " Num:" + RssItems.Count + " " + feedAsService.Link);
}
//CrawlerLog.SuccessLog(feedAsService, insertedItems.Count);
feedAsService.FeedItems = insertedItems;
return(feedAsService);
}
private static Uri GetFaviconUrl(string siteURL)
{
siteURL = siteURL.IndexOfX("www.") > -1 || siteURL.IndexOfX("http://") > -1 ? siteURL : "www." + siteURL;
siteURL = siteURL.IndexOfX("http://") > -1 ? siteURL : "http://" + siteURL;
var url = new Uri(siteURL);
var faviconUrl = new Uri(string.Format("{0}://{1}/favicon.ico", url.Scheme, url.Host));
try
{
using (var httpWebResponse = WebRequest.Create(faviconUrl).GetResponse() as HttpWebResponse)
{
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
// Log("Found a /favicon.ico file for {0}", url);
return(faviconUrl);
}
}
}
catch (WebException)
{
}
String htmlText = " ";
try
{
using (var httpWebResponse = WebRequest.Create(siteURL).GetResponse() as HttpWebResponse)
{
if (httpWebResponse != null && httpWebResponse.StatusCode == HttpStatusCode.OK)
{
Stream stream = httpWebResponse.GetResponseStream();
StreamReader reader = new StreamReader(stream);
htmlText = reader.ReadToEnd();
//site.IndexPageText = htmlText;
}
}
}
catch (Exception ex)
{
GeneralLogs.WriteLog("Error @SetIcons GetWebText" + ex.Message.SubstringX(0, 150));
}
foreach (Match match in Regex.Matches(htmlText, "<link .*? href=\"(.*?.ico)\""))
{
siteURL = siteURL[siteURL.Length - 1] == '/' ? siteURL.Remove(siteURL.Length - 1, 1) : siteURL;
return(new Uri(siteURL + "/" + match.Groups[1].Value.Replace("~/", "")));
}
return(null);
}
static void Main(string[] args)
{
container = Bootstrapper.Initialise();
if (args == null || !args.Any() || args[0].Contains("updater"))
{
new FeedUpdater(container.Resolve <IAppConfigBiz>(), container.Resolve <IFeedBusiness>(), container.Resolve <IFeedItemBusiness>(), container.Resolve <IUpdaterDurationBusiness>()).AutoUpdater();
GeneralLogs.WriteLogInDB("Bye.... ", TypeOfLog.End, typeof(FeedUpdater));
}
else if (args[0].ToLower().Contains("feedschecker"))
{
GeneralLogs.WriteLogInDB("Starttrr Check Feed ", TypeOfLog.Start, "FeedHealth");
var bads = FeedCheck();
GeneralLogs.WriteLogInDB("Number of bad feeds " + bads, TypeOfLog.End, "FeedHealth");
}
}
public static string GetTagAttribute(HtmlDocument doc, string Pattern, string AttributeName)
{
try
{
HtmlNode Tag = doc.DocumentNode.SelectSingleNode("//" + Pattern);
if (Tag.GetAttributeValue(AttributeName, string.Empty) != null)
{
return(Tag.GetAttributeValue(AttributeName, string.Empty));
}
}
catch (Exception ex)
{
GeneralLogs.WriteLog("Error GetImageStreamByPattern" + ex.Message);
}
return(null);
}
public List <FeedContract> getLatestFeedsByDuration(string DurationCode, int MaxSize, bool IsBlog)
{
var duration = _updaterDurationBusiness.GetLast(DurationCode, MaxSize);
var context = new TazehaContext();
GeneralLogs.WriteLog("getLatestFeedsByDuration :" + DurationCode + " StartIndex:" + duration.StartIndex, TypeOfLog.Info);
var arr = new List <Feed>();
arr = context.Feeds.Where <Feed>(x => x.UpdateDurationId.Value == duration.Id &&
(x.Site.IsBlog == IsBlog) &&
(x.Deleted == 0 || (int)x.Deleted > 10)).OrderBy(x => x.Id).Skip(duration.StartIndex).Take <Feed>(MaxSize).ToList();
var res = arr.ConvertToFeedContract().ToList();
System.Web.HttpRuntime.Cache.AddToChache_Hours("Duration_" + DurationCode, duration.StartIndex, 12);
return(res);
}
public List <FeedContract> getLatestFeedsByDuration(string DurationCode, int MaxSize, int IsBlog)
{
UpdateDuration duration = UpdateDurationManager.getLast(DurationCode, MaxSize);
TazehaContext context = new TazehaContext();
GeneralLogs.WriteLog("getLatestFeedsByDuration :" + DurationCode + " StartIndex:" + duration.StartIndex, TypeOfLog.Info);
List <Feed> arr = new List <Feed>();
arr = context.Feeds.Where <Feed>(x => x.UpdateDurationId.Value == duration.UpdateDurationId &&
(x.Site.IsBlog == IsBlog || IsBlog == 2) &&
(x.Deleted == 0 || x.Deleted > 10)).OrderBy(x => x.FeedId).Skip(duration.StartIndex).Take <Feed>(MaxSize).ToList();
var res = arr.ConvertToFeedContract().ToList();
//GeneralLogs.WriteLogInDB(string.Format("Return {0} Feed to remote client,start index {1}", res.Count(), duration.StartIndex), TypeOfLog.Info);
//System.Web.HttpRuntime.Cache.AddToChache_Hours("Duration_" + DurationCode, duration.StartIndex, 12);
return(res);
}
private void setTitle()
{
try
{
int startingIndex = Content.IndexOfX("<Title>");
if (startingIndex > -1)
{
int indexOfClosingQuotationMark = Content.IndexOfX("</Title>");
_fulltitle = Content.Substring(startingIndex + 7, indexOfClosingQuotationMark - (startingIndex + 7));
_fulltitle = Helper.HtmlRemoval.StripTagsRegex(_fulltitle);
char[] splitor = { '_', '*', '-', '/', '\\', '&', '@', '#', '~', '(', ')', '+', '=', ':' };
string[] arr = _fulltitle.Split(splitor);
_title = arr[0].SubstringX(0, 150).Length > 5 ? arr[0].SubstringX(0, 150) : _fulltitle.SubstringX(0, 150);
}
}
catch (Exception ex) {
GeneralLogs.WriteLog(ex.Message);
}
}
public virtual JsonResult Logs_Read([DataSourceRequest] DataSourceRequest request, bool indb = true)
{
IQueryable <LogsBuffer> res;
if (!request.Sorts.Any())
{
request.Sorts.Add(new Kendo.Mvc.SortDescriptor("Id", System.ComponentModel.ListSortDirection.Descending));
}
if (indb)
{
res = _logsBusiness.GetList();
return(Json(res.ToDataSourceResult(request), JsonRequestBehavior.AllowGet));
}
else
{
return(Json(GeneralLogs.getLogs().ToDataSourceResult(request), JsonRequestBehavior.AllowGet));
}
}
