private static async Task GetCardsUrls()
{
var amountProcessedLock = new object();
var amountProcessed = 0;
var progressTrackingTask = Task.Run(async() =>
{
while (!_cardsUrls.IsAddingCompleted)
{
await Task.Delay(100);
lock (ConsoleWriterLock)
{
Console.SetCursorPosition(0, 0);
Console.Write($"Fetching cards urls... {amountProcessed} of {_amountOfCards}".PadRight(Console.WindowWidth, ' '));
}
}
lock (ConsoleWriterLock)
{
Console.SetCursorPosition(0, 0);
Console.Write("Fetching cards urls completed!".PadRight(Console.WindowWidth, ' '));
}
});
Parallel.For(1, _amountOfCardListPages + 1, pageNumber =>
{
var pageDocument = new HtmlWeb().Load(GetCardListPageUrl(pageNumber));
var cardListItems = pageDocument.QuerySelectorAll("#cardResults li");
foreach (var cardListItem in cardListItems)
{
var cardUrl = cardListItem
.QuerySelector("a")
.Attributes["href"].Value;
cardUrl = GetCardPageUrl(cardUrl);
_cardsUrls.Add(cardUrl);
lock (amountProcessedLock) amountProcessed++;
}
});
_cardsUrls.CompleteAdding();
await progressTrackingTask;
}
private static void SetAmountOfCards(int pages = -1)
{
if (pages < 0)
{
var cardListPage = new HtmlWeb().Load(GetCardListPageUrl());
var amountOfPages = int.Parse(
cardListPage
.QuerySelector("div#cards-load-more > div > span")
.InnerText
.Substring(5)
);
_amountOfCardsInList = cardListPage.QuerySelectorAll("#cardResults li").Count;
_amountOfCardListPages = amountOfPages;
}
else
{
_amountOfCardListPages = pages;
}
_amountOfCards = _amountOfCardsInList * _amountOfCardListPages;
}