private async Task <List <Post> > GetUnfetchedPostsAsync(Fetcher fetcher,
string blog, string tag, FilePlan plan, Dictionary <Media, long> medias)
{
var posts = new ConcurrentBag <Post>();
var block = new ActionBlock <Media>(
async media =>
{
int offset = 0;
while (true)
{
var postSet = await fetcher.GetPostsAsync(
blog, media, tag, offset);
logger.LogTrace($"Got {postSet} (Offset: {offset}, TotalPosts: {postSet.TotalPosts:N0})");
if (postSet.Count == 0)
{
return;
}
foreach (var post in postSet)
{
posts.Add(post);
}
if (postSet.Last().PostId <= medias[media])
{
return;
}
offset += 20;
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
medias.Keys.ToList().ForEach(m => block.Post(m));
block.Complete();
await block.Completion;
return(posts.OrderByDescending(p => p.PostId).ToList());
}
private async Task FetchMediasAsync(string folder, FilePlan plan, List <Post> posts)
{
var client = new HttpClient();
var jobber = new TransformManyBlock <Post, FetchJob>(
post =>
{
var jobs = new List <FetchJob>();
if (post.Media != Media.Photo)
{
jobs.Add(GetJob(post, post.VideoUri, null));
}
else
{
if (post.PhotoUris.Count == 1)
{
jobs.Add(GetJob(post, post.PhotoUris[0], null));
}
else
{
for (int i = 0; i < post.PhotoUris.Count; i++)
{
jobs.Add(GetJob(post, post.PhotoUris[i], i));
}
}
}
return(jobs);
});
var fetcher = new ActionBlock <FetchJob>(
async job =>
{
if (plan == FilePlan.Add && job.Exists(folder))
{
logger.LogDebug($"Skipped \"{job.FileName}\"");
return;
}
var response = await client.GetAsync(job.Uri);
if (response.IsSuccessStatusCode)
{
var stream = await response.Content.ReadAsStreamAsync();
await job.SaveToFileAsync(stream, folder);
logger.LogInformation($"Fetched \"{job.FileName}\"");
}
else
{
var sb = new StringBuilder();
sb.Append(response.ReasonPhrase);
sb.Append(" (StatusCode: ");
sb.Append(response.StatusCode);
sb.Append(", FileName: ");
sb.Append(job.FileName);
sb.Append(", Uri: ");
sb.Append(job.Uri);
sb.Append(")");
logger.LogWarning(sb.ToString());
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
});
jobber.LinkTo(fetcher, new DataflowLinkOptions {
PropagateCompletion = true
});
posts.ForEach(post => jobber.Post(post));
jobber.Complete();
await fetcher.Completion;
}