private static async Task FetchAndSaveWithTPL(PagedCharacters firstPage)
{
/* This is an experimental method and NOT in use.
* This method links two DataFlow blocks.
* 1. TransformBlock for fetching API data and pass to the next block
* 2. AcitonBlock to save the fetched data.
*
* The idea is that because API Fetch is much slower than the db insert,
* parallel execution of API fetch and db save can reduce some of db saving load.
*
* But the result seems to show the gain from this approach is less than the overhead of
* having two TPL blocks being maintained.
*/
var apiFetcher = new TransformBlock <int, PagedCharacters>(
async pageNr =>
{
var result = await FetchCharacter(pageNr);
return(result);
}
, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxParallel,
SingleProducerConstrained = false
});
var saver = new ActionBlock <PagedCharacters>(pc =>
{
using var db = new RickAndMortyContext(dbOptions);
db.BulkInsert(pc.Characters.ToArray());
db.SaveChanges();
Console.WriteLine($"Save finished. Page: {pc.CurrentPage}");
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 //Do not allow concurrent writes
});
apiFetcher.LinkTo(saver, new DataflowLinkOptions {
PropagateCompletion = true
});
// Start by posting page numbers to the fetcher
for (var pageNr = 2; pageNr <= firstPage.PageInfo.Pages; pageNr++)
{
await apiFetcher.SendAsync(pageNr);
}
apiFetcher.Complete();
await saver.Completion;
Console.WriteLine($"API fetch & save completed. Elapsed: {stopwatch.Elapsed}");
}
private static void SaveItemsInConcurrentBag()
{
/*
* SQLite is file-based simple db and it locks the whole db when writing.
* Therefore, there is no gain at parallel writing.
* So, we write all the data at once.
*/
var allCharacters = concurrentBag.SelectMany(n => n).ToArray();
using var db = new RickAndMortyContext(dbOptions);
// Use BulkInsert extension
/* With SQLite, BulkInsert has no preformance gain since it does not support BulkCopy. */
db.BulkInsert(allCharacters);
db.SaveChanges();
Console.WriteLine($"Data save finished. Elapsed: {stopwatch.Elapsed}");
}
private static void SavePage(PagedCharacters firstPage)
{
using var db = new RickAndMortyContext(dbOptions);
db.BulkInsert(firstPage.Characters.ToArray());
db.SaveChanges();
}