public async Task <SearchResultPage> Search(
string terms, string author, string parentAuthor, string category, int page, bool oldestFirst = false)
{
if (string.IsNullOrEmpty(category))
{
category = "all";
}
var query = HttpUtility.ParseQueryString("");
query.Add("chatty", "1");
query.Add("type", "4");
query.Add("chatty_term", terms);
query.Add("chatty_user", author);
query.Add("chatty_author", parentAuthor);
query.Add("chatty_filter", category);
query.Add("page", $"{page}");
query.Add("result_sort", oldestFirst ? "postdate_asc" : "postdate_desc");
var html = await _downloadService.DownloadWithSharedLogin(
"http://www.shacknews.com/search?" + query.ToString());
var p = new Parser(html);
var searchResultPage =
new SearchResultPage
{
Results = new List <SearchResult>(),
CurrentPage = page
};
if (p.Peek(1, "<h2 class=\"search-num-found\"") != -1)
{
searchResultPage.TotalResults = int.Parse(p.Clip(
new[] { "<h2 class=\"search-num-found\"", ">" },
" ").Replace(",", ""));
}
while (p.Peek(1, "<li class=\"result") != -1 &&
p.Peek(1, "<span class=\"chatty-author\">") != -1)
{
var result = new SearchResult();
result.Author = p.Clip(
new[] { "<span class=\"chatty-author\">", "<a class=\"more\"", ">" },
":</a></span>");
result.Date = DateParser.Parse(p.Clip(
new[] { "<span class=\"postdate\"", ">", " " },
"</span>"));
result.Id = int.Parse(p.Clip(
new[] { "<a href=\"/chatty", "chatty/", "/" },
"\""));
result.Preview = p.Clip(
new[] { ">" },
"</a>");
searchResultPage.Results.Add(result);
}
return(searchResultPage);
}
public async Task <List <FrontPageArticle> > GetStories()
{
var html = await _downloadService.DownloadWithSharedLogin("https://www.shacknews.com/feed/rss",
verifyLoginStatus : false);
var list = new List <FrontPageArticle>();
var p = new Parser(html);
p.Seek(1, "<channel>");
while (p.Peek(1, "<item>") != -1)
{
p.Seek(1, "<item>");
var endPos = p.Peek(1, "</item>");
var title = p.Clip(
new[] { "<title><![CDATA[", "CDATA[", "[" },
"]]></title>");
var link = p.Clip(
new[] { "<link>", ">" },
"</link>");
var pubDateStr = p.Clip(
new[] { "<pubDate>", ">" },
"</pubDate>");
var time = DateParser.Parse(pubDateStr);
// the time zone offset in this time is a lie. this is really UTC.
time = new DateTimeOffset(time.DateTime, TimeSpan.Zero);
var description = p.Clip(
new[] { "<description><![CDATA[", "CDATA[", "[" },
"]]></description>");
var linkParts = link.Split('/');
var id = int.Parse(linkParts[linkParts.Length - 2]);
list.Add(
new FrontPageArticle
{
Body = description,
Date = time,
Id = id,
Name = title,
Preview = description,
Url = link
});
}
return(list);
}
// if the ID doesn't exist, the returned list will be empty
public async Task <List <ChattyPost> > GetThreadBodies(int threadId)
{
var url = $"https://www.shacknews.com/frame_laryn.x?root={threadId}";
var html = await _downloadService.DownloadWithSharedLogin(url, verifyLoginStatus : false);
if (!html.Contains("</html>", StringComparison.Ordinal))
{
throw new ParsingException("Shacknews thread tree HTML ended prematurely.");
}
var p = new Parser(html);
var list = new List <ChattyPost>();
while (p.Peek(1, "<div id=\"item_") != -1)
{
var reply = new ChattyPost();
reply.Id = int.Parse(p.Clip(
_threadBodiesReplyIdStart,
"\">"));
reply.Category = V2ModerationFlagConverter.Parse(p.Clip(
new[] { "<div class=\"fullpost", "fpmod_", "_" },
" "));
var authorSection = p.Clip(_threadBodiesAuthorSectionStart, "\"");
reply.AuthorId = int.Parse(authorSection.Replace("fpfrozen", ""));
reply.IsFrozen = authorSection.Contains("fpfrozen", StringComparison.Ordinal);
reply.Author = HtmlDecodeExceptLtGt(p.Clip(
_threadBodiesReplyAuthorStart,
"</a>")).Trim();
reply.AuthorFlair = ParseUserFlair(p.Clip(_threadBodiesAuthorFlairStart, "</span>"));
reply.Body = MakeSpoilersClickable(HtmlDecodeExceptLtGt(RemoveNewlines(p.Clip(
_threadBodiesReplyBodyStart,
"</div>"))));
reply.Date = DateParser.Parse(StripTags(p.Clip(
_threadBodiesReplyDateStart,
"T</div")).Replace("Flag", "") + "T");
list.Add(reply);
}
return(list);
}
public async Task <(string Json, ChattyLolCounts LolCounts)> DownloadChattyLolCounts(
string previousJson = null, ChattyLolCounts previousChattyLolCounts = null)
{
var json = await _downloadService.DownloadWithSharedLogin(
"https://www.shacknews.com/api2/api-index.php?action2=ext_get_counts",
verifyLoginStatus : false);
if (json == previousJson)
{
return(json, previousChattyLolCounts);
}
var response = JsonSerializer.Deserialize <Dictionary <string, Dictionary <string, Dictionary <string, string> > > >(
json);
return(json,
new ChattyLolCounts
{
CountsByThreadId = (
from threadPair in response
let threadId = int.Parse(threadPair.Key)
let threadTagsDict = (
from postPair in threadPair.Value
let postId = int.Parse(postPair.Key)
let postLols = (
from x in postPair.Value
select new LolModel {
Tag = x.Key, Count = int.Parse(x.Value)
}
).ToList()
select(PostId: postId, PostTags: postLols)
).ToDictionary(x => x.PostId, x => x.PostTags)
let threadLolCounts = new ThreadLolCounts {
CountsByPostId = threadTagsDict
}
select(ThreadId: threadId, ThreadLolCounts: threadLolCounts)
).ToDictionary(x => x.ThreadId, x => x.ThreadLolCounts)
});
}
public async Task <CortexUserData> GetCortexUserData(int userId)
{
var page = await _downloadService.DownloadWithSharedLogin($"https://www.shacknews.com/cortex/user/{userId}");
var parser = new Parser(page);
var userData = new CortexUserData();
userData.UserId = userId;
userData.Username = parser.Clip(new string[] { "<span class=\"user-name\"", ">" }, "</span>");
//Order matters, because we're lazy here.
userData.Points = int.Parse(parser.Clip(new string[] { "<span class=\"stat-number\"", ">" }, "</span>"), NumberStyles.Any);
userData.Comments = int.Parse(parser.Clip(new string[] { "<span class=\"stat-number\"", ">" }, "</span>"), NumberStyles.Any);
userData.CortexPosts = int.Parse(parser.Clip(new string[] { "<span class=\"stat-number\"", ">" }, "</span>"), NumberStyles.Any);
userData.Wins = int.Parse(parser.Clip(new string[] { "<span class=\"stat-number\"", ">" }, "</span>"), NumberStyles.Any);
userData.RegistrationDate = DateTime.Parse(parser.Clip(new string[] { "<span class=\"stat-number\"", ">" }, "</span>"));
return(userData);
}
public async Task <(string Html, ChattyPage Page)> GetChattyPage(StepStopwatch stopwatch,
int page, string previousHtml = null, ChattyPage previousChattyPage = null)
{
stopwatch?.Step($"Download_{page}");
var url = $"https://www.shacknews.com/chatty?page={page}";
var html = await _downloadService.DownloadWithSharedLogin(url);
stopwatch?.Step($"Parse_{page}");
// remove the progress meter which changes on every load, so that we'll get identical html when nothing
// has changed
html = _progressMeterRegex.Replace(html, "");
// remove the comment count too, it appears on all pages even though some pages haven't changed
html = _commentCountRegex.Replace(html, "");
return(html, html == previousHtml ? previousChattyPage : ParseChattyPage(html));
}