private static async Task GetCardsUrls() { var amountProcessedLock = new object(); var amountProcessed = 0; var progressTrackingTask = Task.Run(async() => { while (!_cardsUrls.IsAddingCompleted) { await Task.Delay(100); lock (ConsoleWriterLock) { Console.SetCursorPosition(0, 0); Console.Write($"Fetching cards urls... {amountProcessed} of {_amountOfCards}".PadRight(Console.WindowWidth, ' ')); } } lock (ConsoleWriterLock) { Console.SetCursorPosition(0, 0); Console.Write("Fetching cards urls completed!".PadRight(Console.WindowWidth, ' ')); } }); Parallel.For(1, _amountOfCardListPages + 1, pageNumber => { var pageDocument = new HtmlWeb().Load(GetCardListPageUrl(pageNumber)); var cardListItems = pageDocument.QuerySelectorAll("#cardResults li"); foreach (var cardListItem in cardListItems) { var cardUrl = cardListItem .QuerySelector("a") .Attributes["href"].Value; cardUrl = GetCardPageUrl(cardUrl); _cardsUrls.Add(cardUrl); lock (amountProcessedLock) amountProcessed++; } }); _cardsUrls.CompleteAdding(); await progressTrackingTask; }
private static void SetAmountOfCards(int pages = -1) { if (pages < 0) { var cardListPage = new HtmlWeb().Load(GetCardListPageUrl()); var amountOfPages = int.Parse( cardListPage .QuerySelector("div#cards-load-more > div > span") .InnerText .Substring(5) ); _amountOfCardsInList = cardListPage.QuerySelectorAll("#cardResults li").Count; _amountOfCardListPages = amountOfPages; } else { _amountOfCardListPages = pages; } _amountOfCards = _amountOfCardsInList * _amountOfCardListPages; }