async Task <HtmlDocument> LoadItem(ForumItem item)
{
var response = await Client.GetAsync(item.Link);
// Limit maximum throughput to 100 Kbps by delaying based on content length.
var wait = (int)(SecondsToMilliseconds * response.Content.Headers.ContentLength.Value / MaxBandwidthBytesPerSec);
await Task.Delay(wait);
var document = new HtmlDocument();
document.Load(await response.Content.ReadAsStreamAsync());
return(document);
}
async Task <bool> ProcessForum(ForumItem forum)
{
if (forum == null)
{
return(true);
}
var updated = true;
var forumIndex = 0;
var topicIndex = 0;
while (true)
{
Console.WriteLine($" Processing {forum}...");
var document = await LoadItem(forum);
var forumItems = document.DocumentNode.SelectNodes(Configuration["Forums:Item"] ?? ".//fake_no_match");
if (forumItems != null)
{
foreach (var forumItem in forumItems)
{
forumIndex++;
forumItem.SetAttributeValue("__forum_scanner_forum_index__", forumIndex.ToString());
updated &= await ProcessForum(await CheckItemIsUpdated(ForumItemType.Forum, forumItem));
}
}
var topicItems = document.DocumentNode.SelectNodes(Configuration["Topics:Item"]);
if (topicItems != null)
{
foreach (var topicItem in topicItems)
{
topicIndex++;
topicItem.SetAttributeValue("__forum_scanner_topic_index__", topicIndex.ToString());
updated &= await ProcessTopic(forum, await CheckItemIsUpdated(ForumItemType.Topic, topicItem));
}
}
var nextLink = GetHtmlValue(document.DocumentNode, Configuration.GetSection("Forums:Next"));
if (nextLink.StartsWith("<default:"))
{
break;
}
forum = new ForumItem(forum.Type, forum.Id, nextLink, forum.Updated);
}
return(await SetItemUpdated(forum, updated));
}
async Task <bool> SetItemUpdated(ForumItem item, bool updated)
{
if (updated)
{
await Storage.ExecuteNonQueryAsync($"INSERT OR REPLACE INTO {item.Type}s ({item.Type}Id, Updated) VALUES (@Param0, @Param1)", item.Id, item.Updated);
if ((DateTimeOffset.Now - StartTime).TotalMinutes >= MaxEmailMinutes)
{
throw new MaximumTimeLimitException();
}
if (EmailsSent >= MaxEmailCount)
{
throw new MaximumEmailLimitException();
}
}
return(updated);
}
async Task <bool> ProcessTopic(ForumItem forum, ForumItem topic)
{
if (topic == null)
{
return(true);
}
var updated = true;
var postIndex = 0;
while (true)
{
Console.WriteLine($" Processing {topic}...");
var document = await LoadItem(topic);
var postItems = document.DocumentNode.SelectNodes(Configuration["Posts:Item"]);
if (postItems != null)
{
foreach (var postItem in postItems)
{
postIndex++;
postItem.SetAttributeValue("__forum_scanner_post_index__", postIndex.ToString());
updated &= await ProcessPost(forum, topic, await CheckItemIsUpdated(ForumItemType.Post, postItem));
}
}
var nextLink = GetHtmlValue(document.DocumentNode, Configuration.GetSection("Topics:Next"));
if (nextLink.StartsWith("<default:"))
{
break;
}
topic = new ForumItem(topic.Type, topic.Id, nextLink, topic.Updated);
}
return(await SetItemUpdated(topic, updated));
}
static string GetTemplateResult(string template, ForumItem forum, ForumItem topic, ForumPostItem post)
{
return(GetTemplateResult(template, forum.Id, topic.Id, post.Id, post.ForumName, post.TopicName, post.Index.ToString()));
}
static string GetSubject(IConfigurationSection config, ForumItem forum, ForumItem topic, ForumPostItem post)
{
return(GetTemplateResult(post.Index == 1 ? config["SubjectOP"] : config["SubjectRE"], forum, topic, post));
}
async Task <bool> ProcessPost(ForumItem forum, ForumItem topic, ForumItem item)
{
if (item == null || !(item is ForumPostItem))
{
return(true);
}
var updated = false;
var rootDomainName = GetUrlDomainName.Replace(Configuration["RootUrl"], "$1");
var post = item as ForumPostItem;
post.ApplyTemplate(template => GetTemplateResult(template, forum, topic, post));
Console.WriteLine($" Processing {post}...");
var message = new MimeMessage();
message.MessageId = $"{topic.Id}/{post.Index}@{rootDomainName}";
if (post.Index >= 2)
{
message.InReplyTo = $"{topic.Id}/1@{rootDomainName}";
}
message.Headers["X-ForumScanner-ForumId"] = forum.Id;
message.Headers["X-ForumScanner-ForumName"] = post.ForumName;
message.Headers["X-ForumScanner-TopicId"] = topic.Id;
message.Headers["X-ForumScanner-TopicName"] = post.TopicName;
message.Headers["X-ForumScanner-PostId"] = post.Id;
message.Headers["X-ForumScanner-PostIndex"] = post.Index.ToString();
message.Headers["Auto-Submitted"] = "auto-generated";
message.Headers["Precedence"] = "bulk";
message.Date = post.Date;
message.From.Add(GetMailboxAddress(Configuration.GetSection("Email:From"), post.Author));
message.To.Add(GetMailboxAddress(Configuration.GetSection("Email:To")));
message.Subject = GetSubject(Configuration.GetSection("Email"), forum, topic, post);
message.Body = new TextPart("html")
{
Text = GetEmailBody(post)
};
var source = $"{post.ForumName} > {post.TopicName} > #{post.Index} ({post.Id}) at {post.Date.ToString("T")} on {post.Date.ToString("D")} by {post.Author}";
var errors = await Storage.ExecuteScalarAsync("SELECT COUNT(*) FROM Errors WHERE Source = @Param0", source) as long?;
if (errors.Value < MaxEmailErrors)
{
try
{
Console.WriteLine($" Email: {source}");
EmailsSent++;
if (!Debug)
{
await SendEmail(message);
}
updated = true;
}
catch (Exception error)
{
Console.WriteLine($" {error.ToString().Split("\n")[0].Replace("\r", "")}");
await Storage.ExecuteNonQueryAsync("INSERT INTO Errors (Source, Date, Error) VALUES (@Param0, @Param1, @Param2)", source, DateTimeOffset.Now, error.ToString());
}
}
return(await SetItemUpdated(post, updated));
}