private static RssFeed read(string url, HttpWebRequest request, RssFeed oldFeed)
{
// ***** Marked for substantial improvement
RssFeed feed = new RssFeed();
RssElement element;
Stream stream = null;
Uri uri = new Uri(url);
feed.url = url;
switch (uri.Scheme)
{
case "file":
feed.lastModified = File.GetLastWriteTime(url);
if ((oldFeed != null) && (feed.LastModified == oldFeed.LastModified))
{
oldFeed.cached = true;
return oldFeed;
}
stream = new FileStream(url, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
break;
case "https":
goto case "http";
case "http":
if (request == null)
request = (HttpWebRequest) WebRequest.Create(uri);
if (oldFeed != null)
{
request.IfModifiedSince = oldFeed.LastModified;
request.Headers.Add("If-None-Match", oldFeed.ETag);
}
try
{
HttpWebResponse response = (HttpWebResponse) request.GetResponse();
feed.lastModified = response.LastModified;
feed.etag = response.Headers["ETag"];
try
{
if (response.ContentEncoding != "")
feed.encoding = Encoding.GetEncoding(response.ContentEncoding);
}
catch
{
}
stream = response.GetResponseStream();
}
catch (WebException we)
{
if (oldFeed != null)
{
oldFeed.cached = true;
return oldFeed;
}
else throw we; // bad
}
break;
}
if (stream != null)
{
RssReader reader = null;
try
{
reader = new RssReader(stream);
do
{
element = reader.Read();
if (element is DSD.Site.UtilityClasses.RSS.NET.RssChannel.RssChannel)
feed.Channels.Add((DSD.Site.UtilityClasses.RSS.NET.RssChannel.RssChannel) element);
} while (element != null);
feed.rssVersion = reader.Version;
}
finally
{
feed.exceptions = reader.Exceptions;
reader.Close();
}
}
else
throw new ApplicationException("Not a valid Url");
return feed;
}
private static void CheckOneFeed(string readerUrl, string persistAsKey, int orgIdForTemplate)
{
string persistenceKey = String.Format("Pressrelease-Highwater-{0}", persistAsKey);
DateTime highWaterMark = DateTime.MinValue;
RssReader reader = null;
try
{
string highWaterMarkString = Persistence.Key[persistenceKey];
if (string.IsNullOrEmpty(highWaterMarkString))
{
//Initialize highwatermark if never used
highWaterMark = DateTime.Now;
Persistence.Key[persistenceKey] = DateTime.Now.ToString();
}
else
{
try
{
highWaterMark = DateTime.Parse(highWaterMarkString);
}
catch (Exception ex)
{
HeartBeater.Instance.SuggestRestart();
throw new Exception(
"Triggered restart. Unable to read/parse old highwater mark from database in PressReleaseChecker.Run(), from key:" +
persistenceKey + ", loaded string was '" + highWaterMarkString + "' expected format is " +
DateTime.Now, ex);
}
}
DateTime storedHighWaterMark = highWaterMark;
reader = new RssReader(readerUrl);
Rss rss = reader.Read();
foreach (RssChannelItem item in rss.Channel.Items)
{
// Ignore any items older than the highwater mark.
// Also ignore if older than two days
if (item.PubDate < highWaterMark || item.PubDate < DateTime.Now.AddDays(-2))
{
continue;
}
// This is an item we should publish.
// Set highwater datetime mark. We do this first, BEFORE processing, as a defense against mail floods,
// if should something go wrong and unexpected exceptions happen.
// We used to add 70 minutes as a defense against mistakes on DST switch in spring and fall (yes, it has happened), but have reduced to two.
if (item.PubDate > storedHighWaterMark)
{
Persistence.Key[persistenceKey] = item.PubDate.AddMinutes(2).ToString();
storedHighWaterMark = item.PubDate.AddMinutes(2);
// Verify that it was written correctly to database. This is defensive programming to avoid a mail flood,
// in case we can't write to the database for some reason.
string newStoredHighWaterString = "";
try
{
newStoredHighWaterString = Persistence.Key[persistenceKey];
DateTime temp = DateTime.Parse(newStoredHighWaterString);
}
catch (Exception ex)
{
throw new Exception(
"Unable to commit/parse new highwater mark to database in PressReleaseChecker.Run(), loaded string was '" +
newStoredHighWaterString + "'", ex);
}
if (DateTime.Parse(Persistence.Key[persistenceKey]) < item.PubDate)
{
throw new Exception(
"Unable to commit new highwater mark to database in PressReleaseChecker.Run()");
}
}
bool allReporters = false;
bool international = false;
MediaCategories categories = new MediaCategories();
foreach (RssCategory category in item.Categories)
{
if (category.Name == "Alla")
{
allReporters = true;
}
else if (category.Name == "Uncategorized")
{
}
else
{
try
{
MediaCategory mediaCategory = MediaCategory.FromName(category.Name);
categories.Add(mediaCategory);
if (category.Name.StartsWith("International"))
{
international = true;
}
}
catch (Exception)
{
ExceptionMail.Send(
new Exception("Unrecognized media category in press release: " + category.Name));
}
}
}
string mailText = Blog2Mail(item.Content);
// Create recipient list of relevant reporters
Reporters reporters = null;
if (allReporters)
{
reporters = Reporters.GetAll();
}
else
{
reporters = Reporters.FromMediaCategories(categories);
}
// Add officers if not int'l
People officers = new People();
Dictionary <int, bool> officerLookup = new Dictionary <int, bool>();
if (!international)
{
int[] officerIds = Roles.GetAllDownwardRoles(1, 1);
foreach (int officerId in officerIds)
{
officerLookup[officerId] = true;
}
}
else
{
officerLookup[1] = true;
}
// Send press release
//TODO: hardcoded geo ... using World
Organization org = Organization.FromIdentity(orgIdForTemplate);
Geography geo = Geography.Root;
PressReleaseMail pressreleasemail = new PressReleaseMail();
pressreleasemail.pSubject = item.Title;
pressreleasemail.pDate = DateTime.Now;
pressreleasemail.pBodyContent = Blog2Mail(item.Content);
pressreleasemail.pOrgName = org.MailPrefixInherited;
if (allReporters)
{
pressreleasemail.pPostedToCategories = "Alla"; // TODO: TRANSLATE
}
else if (international)
{
pressreleasemail.pPostedToCategories = "International/English"; // TODO: THIS IS HARDCODED
}
else
{
pressreleasemail.pPostedToCategories =
PressReleaseMail.GetConcatenatedCategoryString(categories);
}
OutboundMail newMail = pressreleasemail.CreateFunctionalOutboundMail(MailAuthorType.PressService,
OutboundMail.PriorityHighest, org, geo);
int recipientCount = 0;
foreach (Reporter recipient in reporters)
{
if (!Formatting.ValidateEmailFormat(recipient.Email))
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient);
}
foreach (int key in officerLookup.Keys)
{
Person recipient = Person.FromIdentity(key);
if (!Formatting.ValidateEmailFormat(recipient.Mail))
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient, true);
}
newMail.SetRecipientCount(recipientCount);
newMail.SetResolved();
newMail.SetReadyForPickup();
}
}
catch (Exception ex)
{
ExceptionMail.Send(
new Exception("PressReleaseChecker failed:" + ex.Message + "\r\nwhen checking " + readerUrl, ex));
}
finally
{
reader.Close();
}
}
private static void CheckOneFeed (string readerUrl, string persistAsKey, int orgIdForTemplate)
{
string persistenceKey = String.Format("Pressrelease-Highwater-{0}", persistAsKey);
DateTime highWaterMark = DateTime.MinValue;
RssReader reader = null;
try
{
string highWaterMarkString = Persistence.Key[persistenceKey];
if (string.IsNullOrEmpty(highWaterMarkString))
{
//Initialize highwatermark if never used
highWaterMark = DateTime.Now;
Persistence.Key[persistenceKey] = DateTime.Now.ToString();
}
else
{
try
{
highWaterMark = DateTime.Parse(highWaterMarkString);
}
catch (Exception ex)
{
HeartBeater.Instance.SuggestRestart();
throw new Exception(
"Triggered restart. Unable to read/parse old highwater mark from database in PressReleaseChecker.Run(), from key:" + persistenceKey + ", loaded string was '" + highWaterMarkString + "' expected format is " + DateTime.Now.ToString(), ex);
}
}
DateTime storedHighWaterMark = highWaterMark;
reader = new RssReader(readerUrl);
Rss rss = reader.Read();
foreach (RssChannelItem item in rss.Channel.Items)
{
// Ignore any items older than the highwater mark.
// Also ignore if older than two days
if (item.PubDate < highWaterMark || item.PubDate < DateTime.Now.AddDays(-2))
{
continue;
}
// This is an item we should publish.
// Set highwater datetime mark. We do this first, BEFORE processing, as a defense against mail floods,
// if should something go wrong and unexpected exceptions happen.
// We used to add 70 minutes as a defense against mistakes on DST switch in spring and fall (yes, it has happened), but have reduced to two.
if (item.PubDate > storedHighWaterMark)
{
Persistence.Key[persistenceKey] = item.PubDate.AddMinutes(2).ToString();
storedHighWaterMark = item.PubDate.AddMinutes(2);
// Verify that it was written correctly to database. This is defensive programming to avoid a mail flood,
// in case we can't write to the database for some reason.
string newStoredHighWaterString = "";
try
{
newStoredHighWaterString = Persistence.Key[persistenceKey];
DateTime temp = DateTime.Parse(newStoredHighWaterString);
}
catch (Exception ex)
{
throw new Exception(
"Unable to commit/parse new highwater mark to database in PressReleaseChecker.Run(), loaded string was '" + newStoredHighWaterString + "'", ex);
}
if (DateTime.Parse(Persistence.Key[persistenceKey]) < item.PubDate)
{
throw new Exception(
"Unable to commit new highwater mark to database in PressReleaseChecker.Run()");
}
}
bool allReporters = false;
bool international = false;
MediaCategories categories = new MediaCategories();
foreach (RssCategory category in item.Categories)
{
if (category.Name == "Alla")
{
allReporters = true;
}
else if (category.Name == "Uncategorized")
{
continue;
}
else
{
try
{
MediaCategory mediaCategory = MediaCategory.FromName(category.Name);
categories.Add(mediaCategory);
if (category.Name.StartsWith("International"))
{
international = true;
}
}
catch (Exception)
{
ExceptionMail.Send(new Exception("Unrecognized media category in press release: " + category.Name));
}
}
}
string mailText = Blog2Mail(item.Content);
// Create recipient list of relevant reporters
Reporters reporters = null;
if (allReporters)
{
reporters = Reporters.GetAll();
}
else
{
reporters = Reporters.FromMediaCategories(categories);
}
// Add officers if not int'l
People officers = new People();
Dictionary<int, bool> officerLookup = new Dictionary<int, bool>();
if (!international)
{
int[] officerIds = Roles.GetAllDownwardRoles(1, 1);
foreach (int officerId in officerIds)
{
officerLookup[officerId] = true;
}
}
else
{
officerLookup[1] = true;
}
// Send press release
//TODO: hardcoded geo ... using World
Organization org = Organization.FromIdentity(orgIdForTemplate);
Geography geo = Geography.Root;
PressReleaseMail pressreleasemail = new PressReleaseMail();
pressreleasemail.pSubject = item.Title;
pressreleasemail.pDate = DateTime.Now;
pressreleasemail.pBodyContent = Blog2Mail(item.Content);
pressreleasemail.pOrgName = org.MailPrefixInherited;
if (allReporters)
{
pressreleasemail.pPostedToCategories = "Alla"; // TODO: TRANSLATE
}
else if (international)
{
pressreleasemail.pPostedToCategories = "International/English"; // TODO: THIS IS HARDCODED
}
else
{
pressreleasemail.pPostedToCategories = PressReleaseMail.GetConcatenatedCategoryString(categories);
}
OutboundMail newMail = pressreleasemail.CreateFunctionalOutboundMail(MailAuthorType.PressService, OutboundMail.PriorityHighest, org, geo);
int recipientCount = 0;
foreach (Reporter recipient in reporters)
{
if (!Formatting.ValidateEmailFormat(recipient.Email))
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient);
}
foreach (int key in officerLookup.Keys)
{
Person recipient = Person.FromIdentity(key);
if (!Formatting.ValidateEmailFormat(recipient.Mail))
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient, true);
}
newMail.SetRecipientCount(recipientCount);
newMail.SetResolved();
newMail.SetReadyForPickup();
}
}
catch (Exception ex)
{
ExceptionMail.Send(new Exception("PressReleaseChecker failed:" + ex.Message + "\r\nwhen checking " + readerUrl, ex));
}
finally
{
reader.Close();
}
}
private static void CheckOneBlog(string readerFeedUrl, int personId)
{
try
{
Person sender = Person.FromIdentity(personId);
string senderName = sender.Name + " (Piratpartiet)";
string senderAddress = sender.PartyEmail;
if (personId == 5)
{
senderName = "=?utf-8?Q?Christian_Engstr=C3=B6m_(Piratpartiet)?="; // compensate for Mono bug
}
if (personId == 1)
{
senderAddress = "*****@*****.**";
}
RssReader reader = new RssReader(readerFeedUrl);
People recipients = null;
DateTime highWaterMark = DateTime.MinValue;
string persistenceKey = "Newsletter-Highwater-" + personId.ToString();
string highWaterMarkString = Persistence.Key[persistenceKey];
try
{
highWaterMark = DateTime.Parse(highWaterMarkString);
}
catch (FormatException)
{
highWaterMark = DateTime.MinValue;
}
catch (Exception e)
{
throw new ReaderException("feed:" + readerFeedUrl, e);
}
try
{
Rss rss = reader.Read();
// TODO: Read the high water mark from db
foreach (RssChannelItem item in rss.Channel.Items)
{
// Ignore any items older than the highwater mark.
if (item.PubDate < highWaterMark)
{
continue;
}
// For each item, look for the "Nyhetsbrev" category.
bool publish = false;
foreach (RssCategory category in item.Categories)
{
if (category.Name.ToLowerInvariant() == "nyhetsbrev")
{
publish = true;
}
}
if (publish)
{
// Set highwater datetime mark. We do this first as a defense against mail floods, should something go wrong.
Persistence.Key[persistenceKey] = item.PubDate.AddMinutes(5).ToString();
// Verify that it was written correctly to database. This is defensive programming to avoid a mail flood.
if (DateTime.Parse(Persistence.Key[persistenceKey]) < item.PubDate)
{
throw new Exception(
"Unable to commit new highwater mark to database in NewsletterChecker.Run()");
}
bool testMode = false;
if (item.Title.ToLower().Contains("test"))
{
// Newsletter blog entry contains "test" in title -> testmode
testMode = true;
}
// Post to forum
string forumText = Blog2Forum(item.Content);
string mailText = Blog2Mail(item.Content);
int forumPostId = 0;
try
{
forumPostId = SwedishForumDatabase.GetDatabase().CreateNewPost(
testMode ? ForumIdTestPost : ForumIdNewsletter, sender,
"Nyhetsbrev " + DateTime.Today.ToString("yyyy-MM-dd"),
item.Title, forumText);
}
catch (Exception ex)
{
if (!System.Diagnostics.Debugger.IsAttached)
{ // ignore when debugging
throw ex;
}
}
// Establish people to send to, if not already done.
if (recipients == null || recipients.Count == 1)
{
recipients = People.FromNewsletterFeed(NewsletterFeed.TypeID.ChairmanBlog);
}
/*
* Disabled sending to activists -- this was done leading up to the election in 2009
*/
// Add activists (HACK)
// Should probably be better to select by organization, not geography.
/*
* People activists = Activists.FromGeography(Country.FromCode("SE").Geography).People;
*
* recipients = recipients.LogicalOr(activists);*/
// OVERRIDE: If this is a TEST newsletter, send ONLY to the originator
if (testMode)
{
recipients = People.FromSingle(sender);
item.Title += " [TEST MODE]";
}
//TODO: hardcoded Org & geo ... using PP & World
Organization org = Organization.PPSE;
Geography geo = Geography.Root;
NewsletterMail newslettermail = new NewsletterMail();
newslettermail.pSubject = item.Title;
newslettermail.pDate = DateTime.Now;
newslettermail.pForumPostUrl =
String.Format("http://vbulletin.piratpartiet.se/showthread.php?t={0}", forumPostId);
newslettermail.pBodyContent = Blog2Mail(item.Content);
newslettermail.pOrgName = org.MailPrefixInherited;
OutboundMail newMail = newslettermail.CreateOutboundMail(sender, OutboundMail.PriorityLow, org, geo);
int recipientCount = 0;
foreach (Person recipient in recipients)
{
if (!Formatting.ValidateEmailFormat(recipient.Mail) ||
recipient.MailUnreachable ||
recipient.NeverMail)
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient, false);
}
newMail.SetRecipientCount(recipientCount);
newMail.SetResolved();
newMail.SetReadyForPickup();
}
}
}
finally
{
reader.Close();
}
}
catch (Exception ex)
{
lock (feedErrorSignaled)
{
if (!feedErrorSignaled.ContainsKey(readerFeedUrl) || feedErrorSignaled[readerFeedUrl].AddMinutes(60) < DateTime.Now)
{
feedErrorSignaled[readerFeedUrl] = DateTime.Now;
throw new ReaderException("NewsletterChecker got error " + ex.Message + "\n when checking feed:" + readerFeedUrl + "\n(feed will be continually checked, bu will not signal error again for an hour)", ex);
}
}
}
}
private static void CheckOneBlog (string readerFeedUrl, int personId)
{
try
{
Person sender = Person.FromIdentity(personId);
string senderName = sender.Name + " (Piratpartiet)";
string senderAddress = sender.PartyEmail;
if (personId == 5)
{
senderName = "=?utf-8?Q?Christian_Engstr=C3=B6m_(Piratpartiet)?="; // compensate for Mono bug
}
if (personId == 1)
{
senderAddress = "*****@*****.**";
}
RssReader reader = new RssReader(readerFeedUrl);
People recipients = null;
DateTime highWaterMark = DateTime.MinValue;
string persistenceKey = "Newsletter-Highwater-" + personId.ToString();
string highWaterMarkString = Persistence.Key[persistenceKey];
try
{
highWaterMark = DateTime.Parse(highWaterMarkString);
}
catch (FormatException)
{
highWaterMark = DateTime.MinValue;
}
catch (Exception e)
{
throw new ReaderException("feed:" + readerFeedUrl, e);
}
try
{
Rss rss = reader.Read();
// TODO: Read the high water mark from db
foreach (RssChannelItem item in rss.Channel.Items)
{
// Ignore any items older than the highwater mark.
if (item.PubDate < highWaterMark)
{
continue;
}
// For each item, look for the "Nyhetsbrev" category.
bool publish = false;
foreach (RssCategory category in item.Categories)
{
if (category.Name.ToLowerInvariant() == "nyhetsbrev")
{
publish = true;
}
}
if (publish)
{
// Set highwater datetime mark. We do this first as a defense against mail floods, should something go wrong.
Persistence.Key[persistenceKey] = item.PubDate.AddMinutes(5).ToString();
// Verify that it was written correctly to database. This is defensive programming to avoid a mail flood.
if (DateTime.Parse(Persistence.Key[persistenceKey]) < item.PubDate)
{
throw new Exception(
"Unable to commit new highwater mark to database in NewsletterChecker.Run()");
}
bool testMode = false;
if (item.Title.ToLower().Contains("test"))
{
// Newsletter blog entry contains "test" in title -> testmode
testMode = true;
}
// Post to forum
string forumText = Blog2Forum(item.Content);
string mailText = Blog2Mail(item.Content);
int forumPostId = 0;
try
{
forumPostId = SwedishForumDatabase.GetDatabase().CreateNewPost(
testMode ? ForumIdTestPost : ForumIdNewsletter, sender,
"Nyhetsbrev " + DateTime.Today.ToString("yyyy-MM-dd"),
item.Title, forumText);
}
catch (Exception ex)
{
if (!System.Diagnostics.Debugger.IsAttached)
{ // ignore when debugging
throw ex;
}
}
// Establish people to send to, if not already done.
if (recipients == null || recipients.Count == 1)
{
recipients = People.FromNewsletterFeed(NewsletterFeed.TypeID.ChairmanBlog);
}
/*
* Disabled sending to activists -- this was done leading up to the election in 2009
*/
// Add activists (HACK)
// Should probably be better to select by organization, not geography.
/*
People activists = Activists.FromGeography(Country.FromCode("SE").Geography).People;
recipients = recipients.LogicalOr(activists);*/
// OVERRIDE: If this is a TEST newsletter, send ONLY to the originator
if (testMode)
{
recipients = People.FromSingle(sender);
item.Title += " [TEST MODE]";
}
//TODO: hardcoded Org & geo ... using PP & World
Organization org = Organization.PPSE;
Geography geo = Geography.Root;
NewsletterMail newslettermail = new NewsletterMail();
newslettermail.pSubject = item.Title;
newslettermail.pDate = DateTime.Now;
newslettermail.pForumPostUrl =
String.Format("http://vbulletin.piratpartiet.se/showthread.php?t={0}", forumPostId);
newslettermail.pBodyContent = Blog2Mail(item.Content);
newslettermail.pOrgName = org.MailPrefixInherited;
OutboundMail newMail = newslettermail.CreateOutboundMail(sender, OutboundMail.PriorityLow, org, geo);
int recipientCount = 0;
foreach (Person recipient in recipients)
{
if (!Formatting.ValidateEmailFormat(recipient.Mail)
|| recipient.MailUnreachable
|| recipient.NeverMail)
{
continue;
}
++recipientCount;
newMail.AddRecipient(recipient, false);
}
newMail.SetRecipientCount(recipientCount);
newMail.SetResolved();
newMail.SetReadyForPickup();
}
}
}
finally
{
reader.Close();
}
}
catch (Exception ex)
{
lock (feedErrorSignaled)
{
if (!feedErrorSignaled.ContainsKey(readerFeedUrl) || feedErrorSignaled[readerFeedUrl].AddMinutes(60) < DateTime.Now)
{
feedErrorSignaled[readerFeedUrl] = DateTime.Now;
throw new ReaderException("NewsletterChecker got error " + ex.Message + "\n when checking feed:" + readerFeedUrl + "\n(feed will be continually checked, bu will not signal error again for an hour)", ex);
}
}
}
}