Exemplo n.º 1
0
        static async Task MainAsync()
        {
            var configuration = new ConfigurationBuilder()
                                .SetBasePath(Directory.GetCurrentDirectory())
                                .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                                .AddEnvironmentVariables()
                                .Build();

            var sendgridMailSenderOptions = new SendgridMailSenderOptions
            {
                ApiKey      = configuration.GetSection("SendgridMailSenderOptions").GetValue <string>("ApiKey"),
                EmailSender = configuration.GetSection("SendgridMailSenderOptions").GetValue <string>("EmailSender"),
            };
            var njuskaloStoreOptions = new NjuskaloStoreOptions
            {
                StorageName  = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("StorageName"),
                StorageKey   = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("StorageKey"),
                TableName    = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("TableName"),
                PartitionKey = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("PartitionKey"),
            };
            var njuskaloNotifierOptions = new NjuskaloNotifierOptions
            {
                Emails = configuration.GetSection("NjuskaloNotifierOptions").GetValue <string>("Emails")
            };

            using (var client = new HttpClient())
            {
                var logger      = new DelegateLogger(WriteLine);
                var emailSender = new SendgridMailSender(Options.Create(sendgridMailSenderOptions));
                var notifier    = new NjuskaloNotifier(Options.Create(njuskaloNotifierOptions), emailSender);
                var store       = new NjuskaloStore(Options.Create(njuskaloStoreOptions));


                var t2 = ScraperFactory.CreateNjuskaloDvosobniScraper(client, logger).ScrapeAsync();
                var t3 = ScraperFactory.CreateNjuskaloTrosobniScraper(client, logger).ScrapeAsync();
                var t4 = ScraperFactory.CreateNjuskaloCetverosobniScraper(client, logger).ScrapeAsync();

                await Task.WhenAll(t2, t3, t4);

                var entities = new HashSet <string>(t2.Result.Union(t3.Result));

                await store.InitStorageAsync();

                await store.PersistAsync(entities);

                var toNotify = await store.GetUnnotifiedAsync();

                if (toNotify.Any())
                {
                    var success = await notifier.NotifyAboutEntitiesAsync(toNotify, store.PartitionKey);

                    if (success)
                    {
                        await store.MarkNotifiedAsync(toNotify);
                    }
                }
            }
        }
        private static async Task ProcessScrapeAndNotifyAsync(SendgridMailSenderOptions sendgridMailSenderOptions, NjuskaloNotifierOptions njuskaloNotifierOptions, NjuskaloStoreOptions njuskaloStoreOptions, ICollection <NjuskaloScraperOptions> scraperOptions, Microsoft.Extensions.Logging.ILogger log)
        {
            using (var client = new HttpClient())
            {
                client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 OPR/63.0.3368.71");

                var logger      = new DelegateLogger(log);
                var emailSender = new SendgridMailSender(Options.Create(sendgridMailSenderOptions));
                var notifier    = new NjuskaloNotifier(Options.Create(njuskaloNotifierOptions), emailSender);
                var store       = new NjuskaloStore(Options.Create(njuskaloStoreOptions));

                var scrapeTasks = new List <Task <ICollection <string> > >();
                foreach (var options in scraperOptions)
                {
                    var scraper = new NjuskaloScraper(Options.Create(options), client, logger);
                    scrapeTasks.Add(scraper.ScrapeAsync());
                }
                await Task.WhenAll(scrapeTasks);

                var entities = new HashSet <string>(scrapeTasks.SelectMany(x => x.Result));
                logger.WriteLine($"Found {entities.Count} entities.");

                await store.InitStorageAsync();

                await store.PersistAsync(entities);

                var toNotify = await store.GetUnnotifiedAsync();

                logger.WriteLine($"To notify: {toNotify.Count}.");
                if (toNotify.Any())
                {
                    var success = await notifier.NotifyAboutEntitiesAsync(toNotify, store.PartitionKey);

                    if (success)
                    {
                        await store.MarkNotifiedAsync(toNotify);

                        logger.WriteLine($"Notified about {toNotify.Count} entities.");
                    }
                }
            }
        }