static async Task MainAsync()
{
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
var sendgridMailSenderOptions = new SendgridMailSenderOptions
{
ApiKey = configuration.GetSection("SendgridMailSenderOptions").GetValue <string>("ApiKey"),
EmailSender = configuration.GetSection("SendgridMailSenderOptions").GetValue <string>("EmailSender"),
};
var njuskaloStoreOptions = new NjuskaloStoreOptions
{
StorageName = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("StorageName"),
StorageKey = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("StorageKey"),
TableName = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("TableName"),
PartitionKey = configuration.GetSection("NjuskaloStoreOptions").GetValue <string>("PartitionKey"),
};
var njuskaloNotifierOptions = new NjuskaloNotifierOptions
{
Emails = configuration.GetSection("NjuskaloNotifierOptions").GetValue <string>("Emails")
};
using (var client = new HttpClient())
{
var logger = new DelegateLogger(WriteLine);
var emailSender = new SendgridMailSender(Options.Create(sendgridMailSenderOptions));
var notifier = new NjuskaloNotifier(Options.Create(njuskaloNotifierOptions), emailSender);
var store = new NjuskaloStore(Options.Create(njuskaloStoreOptions));
var t2 = ScraperFactory.CreateNjuskaloDvosobniScraper(client, logger).ScrapeAsync();
var t3 = ScraperFactory.CreateNjuskaloTrosobniScraper(client, logger).ScrapeAsync();
var t4 = ScraperFactory.CreateNjuskaloCetverosobniScraper(client, logger).ScrapeAsync();
await Task.WhenAll(t2, t3, t4);
var entities = new HashSet <string>(t2.Result.Union(t3.Result));
await store.InitStorageAsync();
await store.PersistAsync(entities);
var toNotify = await store.GetUnnotifiedAsync();
if (toNotify.Any())
{
var success = await notifier.NotifyAboutEntitiesAsync(toNotify, store.PartitionKey);
if (success)
{
await store.MarkNotifiedAsync(toNotify);
}
}
}
}
private static async Task ProcessScrapeAndNotifyAsync(SendgridMailSenderOptions sendgridMailSenderOptions, NjuskaloNotifierOptions njuskaloNotifierOptions, NjuskaloStoreOptions njuskaloStoreOptions, ICollection <NjuskaloScraperOptions> scraperOptions, Microsoft.Extensions.Logging.ILogger log)
{
using (var client = new HttpClient())
{
client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 OPR/63.0.3368.71");
var logger = new DelegateLogger(log);
var emailSender = new SendgridMailSender(Options.Create(sendgridMailSenderOptions));
var notifier = new NjuskaloNotifier(Options.Create(njuskaloNotifierOptions), emailSender);
var store = new NjuskaloStore(Options.Create(njuskaloStoreOptions));
var scrapeTasks = new List <Task <ICollection <string> > >();
foreach (var options in scraperOptions)
{
var scraper = new NjuskaloScraper(Options.Create(options), client, logger);
scrapeTasks.Add(scraper.ScrapeAsync());
}
await Task.WhenAll(scrapeTasks);
var entities = new HashSet <string>(scrapeTasks.SelectMany(x => x.Result));
logger.WriteLine($"Found {entities.Count} entities.");
await store.InitStorageAsync();
await store.PersistAsync(entities);
var toNotify = await store.GetUnnotifiedAsync();
logger.WriteLine($"To notify: {toNotify.Count}.");
if (toNotify.Any())
{
var success = await notifier.NotifyAboutEntitiesAsync(toNotify, store.PartitionKey);
if (success)
{
await store.MarkNotifiedAsync(toNotify);
logger.WriteLine($"Notified about {toNotify.Count} entities.");
}
}
}
}