public async Task Extract(WorkMode workMode, string basicUrl)
{
if (!basicUrl.Contains("otomoto.pl/osobowe"))
{
_logger.Log($"Wrong url: {basicUrl}");
return;
}
await InitSender(workMode);
var numberOfPage = await GetNumberOfPages(basicUrl);
_logger.Log($"Number of pages: {numberOfPage}");
var articlesUrl = new List <string>();
for (int i = 1; i <= numberOfPage; i++)
{
articlesUrl.AddRange(await GetArticlesUrlFromPage(basicUrl + "?page=" + i));
}
_logger.Log($"Number of articles: {articlesUrl.Count}");
int beforeLoad = 0;
if (workMode == WorkMode.Continuous)
{
beforeLoad = await _loader.GetNumgerOfRecords();
}
foreach (var url in articlesUrl)
{
var articleContent = await GetArticleContent(url);
if (articleContent == null)
{
continue;
}
await _sender.Send(articleContent);
}
if (workMode == WorkMode.Continuous)
{
var addedRecords = await _loader.GetNumgerOfRecords() - beforeLoad;
_logger.Log($"Load {addedRecords} records to database");
}
await Task.CompletedTask;
}