private static async Task FetchWithTPL(PagedCharacters firstPage)
{
/* Because API fetch is network bound parallel execution is beneficial.
* Therefore, use TPL dataflow for fetching and collecting data.
*/
var apiFetcher = new ActionBlock <int>(
async pageNr =>
{
var result = await FetchCharacter(pageNr);
concurrentBag.Add(result.Characters);
}
, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxParallel, // Degree of parallelism is controlled by this property.
SingleProducerConstrained = false // Instructing about the SingleProducer enhances the performance.
});
// Start by posting page numbers to the fetcher
for (var pageNr = 2; pageNr <= firstPage.PageInfo.Pages; pageNr++)
{
await apiFetcher.SendAsync(pageNr);
}
apiFetcher.Complete();
await apiFetcher.Completion;
Console.WriteLine($"API fetch completed. Elapsed: {stopwatch.Elapsed}");
}
private static async Task FetchAndSaveWithTPL(PagedCharacters firstPage)
{
/* This is an experimental method and NOT in use.
* This method links two DataFlow blocks.
* 1. TransformBlock for fetching API data and pass to the next block
* 2. AcitonBlock to save the fetched data.
*
* The idea is that because API Fetch is much slower than the db insert,
* parallel execution of API fetch and db save can reduce some of db saving load.
*
* But the result seems to show the gain from this approach is less than the overhead of
* having two TPL blocks being maintained.
*/
var apiFetcher = new TransformBlock <int, PagedCharacters>(
async pageNr =>
{
var result = await FetchCharacter(pageNr);
return(result);
}
, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxParallel,
SingleProducerConstrained = false
});
var saver = new ActionBlock <PagedCharacters>(pc =>
{
using var db = new RickAndMortyContext(dbOptions);
db.BulkInsert(pc.Characters.ToArray());
db.SaveChanges();
Console.WriteLine($"Save finished. Page: {pc.CurrentPage}");
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 //Do not allow concurrent writes
});
apiFetcher.LinkTo(saver, new DataflowLinkOptions {
PropagateCompletion = true
});
// Start by posting page numbers to the fetcher
for (var pageNr = 2; pageNr <= firstPage.PageInfo.Pages; pageNr++)
{
await apiFetcher.SendAsync(pageNr);
}
apiFetcher.Complete();
await saver.Completion;
Console.WriteLine($"API fetch & save completed. Elapsed: {stopwatch.Elapsed}");
}
private static void SavePage(PagedCharacters firstPage)
{
using var db = new RickAndMortyContext(dbOptions);
db.BulkInsert(firstPage.Characters.ToArray());
db.SaveChanges();
}
