private async Task <List <Post> > FetchNewPosts(FanBentoDatabase database)
{
var posts = new List <Post>();
var hasNextPage = false;
var idList = database.Post.Select(t => t.Id).ToHashSet();
do
{
List <Post> list;
(list, hasNextPage) = await FanboxApi.GetPostsList(hasNextPage);
var newPostsList = list.AsParallel().Where(t => !idList.Contains(t.Id)).ToList();
if (newPostsList.Count != list.Count && Configuration.Config["Fanbox:FetchToEnd"] != "true")
{
// some posts already exists, next page should all be old posts
hasNextPage = false;
}
await DownloadPostsImages(newPostsList);
await DownloadPostsFiles(newPostsList);
posts.AddRange(newPostsList);
} while (hasNextPage);
return(posts);
}
private async Task DownloadPostsFiles(IEnumerable <Post> posts)
{
var fileSavePath = Configuration.Config["Fanbox:FileSavePath"];
var fileUrlList = posts.SelectMany(
t => t.Body.Files ?? t.Body.FileMap?.Values.ToList() ?? new List <File>(),
(_, d) => d.Url)
.ToList();
await Task.WhenAll(fileUrlList.Select(async url =>
{
try
{
await FanboxApi.DownloadFile(url, fileSavePath);
}
catch (Exception e)
{
LogTo.Warning(e, $"Failed to download file {url}");
}
}));
}
