Example #1
0
        private static async Task Run(AppOptions options)
        {
            Console.WriteLine($"Use '{options.SelectedStrategy}' strategy");

            using var cts = new CancellationTokenSource();

            Console.CancelKeyPress += (sender, eventArgs) =>
            {
                Console.WriteLine("Cancel event triggered");
                cts.Cancel();
                eventArgs.Cancel = true;
            };

            Console.WriteLine("Press Ctrl+C to exit...");

            IResultFormatter resultFormatter = new SimpleResultFormatter();
            IResultSaver     resultSaver     = new TextFileResultSaver(options.OutputFile, resultFormatter);

            using var httpClientFactory = new HttpClientFactoryWithTimeoutHandle();

            WebCrawlerStrategy crawlerStrategy = options.SelectedStrategy switch
            {
                AppOptions.Strategy.TasksDataflow => new TasksDataflowWebCrawlerStrategy(httpClientFactory, options,
                                                                                         resultSaver),
                AppOptions.Strategy.ParallelLinq => new ParallelLinqWebCrawlerStrategy(httpClientFactory, options,
                                                                                       resultSaver),
                AppOptions.Strategy.ParallelForEach => new ParallelForEachWebCrawlerStrategy(httpClientFactory, options,
                                                                                             resultSaver),
                AppOptions.Strategy.Partitioner => new PartitionerWebCrawlerStrategy(httpClientFactory, options,
                                                                                     resultSaver),
                AppOptions.Strategy.AsyncEnumerable => new AsyncEnumerableStrategy(httpClientFactory, options,
                                                                                   resultSaver),
                _ => throw new NotImplementedException($"Not implemented strategy: {options.SelectedStrategy}")
            };

            await new App(crawlerStrategy)
            .Run(cts.Token)
            .ConfigureAwait(false);

            Console.WriteLine("Finished...");
        }
    }
Example #2
0
 public App(WebCrawlerStrategy webCrawlerStrategy) => _webCrawlerStrategy = webCrawlerStrategy;