Example #1
0
        private static async Task FetchWithTPL(PagedCharacters firstPage)
        {
            /* Because API fetch is network bound parallel execution is beneficial.
             *    Therefore, use TPL dataflow for fetching and collecting data.
             */

            var apiFetcher = new ActionBlock <int>(
                async pageNr =>
            {
                var result = await FetchCharacter(pageNr);
                concurrentBag.Add(result.Characters);
            }
                , new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism    = MaxParallel, // Degree of parallelism is controlled by this property.
                SingleProducerConstrained = false        // Instructing about the SingleProducer enhances the performance.
            });


            // Start by posting page numbers to the fetcher
            for (var pageNr = 2; pageNr <= firstPage.PageInfo.Pages; pageNr++)
            {
                await apiFetcher.SendAsync(pageNr);
            }

            apiFetcher.Complete();
            await apiFetcher.Completion;

            Console.WriteLine($"API fetch completed. Elapsed: {stopwatch.Elapsed}");
        }
Example #2
0
        private static async Task FetchAndSaveWithTPL(PagedCharacters firstPage)
        {
            /* This is an experimental method and NOT in use.
             * This method links two DataFlow blocks.
             *  1. TransformBlock for fetching API data and pass to the next block
             *  2. AcitonBlock to save the fetched data.
             *
             *  The idea is that because API Fetch is much slower than the db insert,
             *  parallel execution of API fetch and db save can reduce some of db saving load.
             *
             *  But the result seems to show the gain from this approach is less than the overhead of
             *  having two TPL blocks being maintained.
             */

            var apiFetcher = new TransformBlock <int, PagedCharacters>(
                async pageNr =>
            {
                var result = await FetchCharacter(pageNr);
                return(result);
            }
                , new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism    = MaxParallel,
                SingleProducerConstrained = false
            });

            var saver = new ActionBlock <PagedCharacters>(pc =>
            {
                using var db = new RickAndMortyContext(dbOptions);
                db.BulkInsert(pc.Characters.ToArray());
                db.SaveChanges();
                Console.WriteLine($"Save finished. Page: {pc.CurrentPage}");
            }, new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = 1 //Do not allow concurrent writes
            });

            apiFetcher.LinkTo(saver, new DataflowLinkOptions {
                PropagateCompletion = true
            });

            // Start by posting page numbers to the fetcher
            for (var pageNr = 2; pageNr <= firstPage.PageInfo.Pages; pageNr++)
            {
                await apiFetcher.SendAsync(pageNr);
            }

            apiFetcher.Complete();
            await saver.Completion;

            Console.WriteLine($"API fetch & save completed. Elapsed: {stopwatch.Elapsed}");
        }
Example #3
0
 private static void SavePage(PagedCharacters firstPage)
 {
     using var db = new RickAndMortyContext(dbOptions);
     db.BulkInsert(firstPage.Characters.ToArray());
     db.SaveChanges();
 }