//private Dictionary<string, string> linkToPageContent = new Dictionary<string, string>();
public void OnRSSPageDownloaded(object sender, PageCompletedEventArgs e)
{
try {
if (AQPublisher == null && !DownloadTextOnly) {
AddEvent("Active MQ publisher is null. Unable to post the event to the queue.");
return;
}
if (!CanRun)
return;
int currentForumId = AccountInfoList[_currentAccountIndex].ForumId;
XmlDocument doc = new HtmlAgilityPack.XmlDocument();
doc.LoadXml(e.PageContent);
foreach (HtmlNode item in doc.DocumentNode.SelectNodes("//channel/item") ?? new HtmlNodeCollection(null)) {
string link = item.Element("link").InnerText.DecodeXMLString();
if (link.LastIndexOf('#') >= 0)
link = link.Substring(0, link.LastIndexOf('#'));
Dictionary<string, string> nameValueCollection = Utility.ParseQueryArguments(link);
string itemId = "";
if (!nameValueCollection.ContainsKey("p")) {
AddEvent("A post in the RSS feed didn't contain post id. Ignoring the post. RSS feed: " + e.SmallUri.GetString());
continue;
}
itemId = nameValueCollection["p"];
lock (_postHashLock) {
if (_forumIdToImportedIdH[currentForumId].Contains(itemId))
continue;
}
WebGetter.DownloadPageAsync(this, item, new GenLib.Web.SmallUri(link), OnPostPageDownloaded);
}
//AddEvent("Parsed RSS page: " + e.SmallUri.GetString());
//Trace.WriteLine("Parsed RSS page: " + e.SmallUri.GetString());
GenLib.Log.LogService.LogInfo("Parsed RSS page " + e.SmallUri.GetString());
if (CanRun)
DownloadNextRSSPage();
}
catch (Exception ex) {
AddEvent("Exception while publishing new data: " + ex.Message);
GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex);
}
}
private void Foo(object sender, PageCompletedEventArgs e)
{
}
public void OnPostPageDownloaded(object sender, PageCompletedEventArgs e)
{
if (!CanRun)
return;
try {
_processedCountSinceLastCheck++;
HtmlNode item = e.UserState as HtmlNode;
string link = e.SmallUri.GetString();
Dictionary<string, string> nameValueCollection = GenLib.Web.Utility.ParseQueryArguments(link);
int postId = -1;
int threadId = -1;
int forumId = -1;
if (nameValueCollection.ContainsKey("p"))
postId = int.Parse(nameValueCollection["p"]);
if (nameValueCollection.ContainsKey("t"))
threadId = int.Parse(nameValueCollection["t"]);
if (nameValueCollection.ContainsKey("f"))
forumId = int.Parse(nameValueCollection["f"]);
//if (!linkToPageContent.ContainsKey(link))
// linkToPageContent[link] = e.PageContent;
HtmlAgilityPack.XmlDocument doc = new HtmlAgilityPack.XmlDocument();
doc.LoadXml(e.PageContent);
var bodyNode = doc.DocumentNode.SelectSingleNode(String.Format("//div[@id='article{0}']", postId));
if (bodyNode == null) {
ReportError("Body for post page was null. Error downloading post information.");
return;
}
string body = bodyNode.InnerHtml;
body = body.StripHtml(); // todo: test if this removes everything ok
//body = body.Replace("\n", " ");
//body = Regex.Replace(body, "<br[ ]*[/]?>", Environment.NewLine);
//body = body.Replace(" \r\n", "\r\n");
//body = body.DecodeXMLString();
//body = RemoveBlockQuotes(body);
//body = System.Web.HttpUtility.HtmlDecode(body);
// use item and body to create an event that is sent
string subject = item.Element("title").InnerText;
string author = item.Element("author").InnerText;
string category = item.Element("category").InnerText;
string dateStr = item.Element("pubDate").InnerText;
string postIdStr = nameValueCollection["p"];
DateTime date = DateTime.Parse(dateStr);
//AddEvent("Parsed post with id " + postId);
//Trace.WriteLine("Parsed post " + postId);
AccountInfo account = AccountInfoList[_currentAccountIndex];
string eventData = Defaults.BuildNewForumItem(account.ForumId, account.ForumName, forumId, postId, threadId, link, date, subject, body, author, category);
if (DownloadTextOnly) {
string fileName = Path.Combine(DownloadPath, postId.ToString());
File.WriteAllText(fileName, eventData);
}
else {
lock (_publisherLock) {
StoreEventData(_activeMQSettings.TopicNameForumSensorPublishForumPost, eventData, false);
AQPublisher.SendMessage(eventData);
}
GenLib.Log.LogService.LogInfo("Parsed post with id " + postId);
}
lock (_postHashLock) {
_forumIdToImportedIdH[account.ForumId].Add(postIdStr);
}
if (postId % 10 == 0)
AddEvent("downloaded post with id " + postId);
}
catch (Exception ex) {
AddEvent("Exception while publishing new data: " + ex.Message);
GenLib.Log.LogService.LogException("ForumSensorDialog.OnPostPageDownloaded Exception while publishing new data: ", ex);
}
//string currentFolder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
//File.WriteAllText(Path.Combine(currentFolder, itemIdStr + ".xml"), eventData);
_totalSentCount++;
//this.Invoke((Action)(() => { LabelStatus.Text = String.Format("Sent {0} posts", _totalSentCount); }));
}
