internal static async Task SavePage(string path, string link)
{
try
{
var req = StaticSettings.Request;
var resp = req.Get(link).ToString();
path += "\\" + Regex.Match(link, @"(?<=\d+\-).+(?=\-\d+|$)").Value.GetValidPath();
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
try
{
var lastPage = await StaticSettings.GetLastPage(resp);
for (var i = lastPage; i > 1; i--)
using (var sw = new StreamWriter($"{path}\\page{i}.html", false))
sw.Write(req.Get($"{link}/page{i}.html"));
throw new Exception();
}
catch (Exception)
{
using (var sw = new StreamWriter(path + "\\page1.html", false))
sw.Write(resp);
}
}
catch (Exception)
{
}
}
internal static async Task<List<string>> GetPageLinks(string link)
{
var lastPage = 1;
var doc = new HtmlDocument();
var req = StaticSettings.Request;
var linksList = new List<string>();
do
{
var resp = req.Get(link + $"/page{lastPage}").ToString();
doc.LoadHtml(resp);
linksList.AddRange(
doc.DocumentNode.Descendants("a")
.Where(x => x.Attributes.Contains("id") && x.Attributes["id"].Value.StartsWith("thread_title_"))
.Select(x => @"http://stackthatmoney.com/forum/" + x.Attributes["href"].Value));
if (lastPage-- == 1)
lastPage = await StaticSettings.GetLastPage(resp);
} while (lastPage > 1);
return linksList;
}
