private IEnumerable <BoosterPack> GetBoosterPacks(IDictionary <string, string> links, out IEnumerable <Error> errors) { var boosterPacks = new ConcurrentBag <BoosterPack>(); var total = links.Count; var current = 0; var tempErrors = new ConcurrentBag <Error>(); Parallel.ForEach(links, ScraperConstants.ParallelOptions, kv => { try { var boosterPack = new BoosterPackParser(kv.Key, $"{ScraperConstants.YuGiPediaUrl}{ScraperConstants.MediaWikiParseIdUrl}{kv.Value}").Parse(HttpClient); boosterPack.Url = $"{ScraperConstants.YuGiPediaUrl}?curid={kv.Value}"; #region OCG TCG boosterPack.TcgExists = TcgBoosters.ContainsKey(boosterPack.Name); boosterPack.OcgExists = OcgBoosters.ContainsKey(boosterPack.Name); #endregion OCG TCG boosterPacks.Add(boosterPack); } catch (Exception exception) { tempErrors.Add(new Error() { Name = kv.Key, Exception = $"{exception.Message}\t{exception.StackTrace}", InnerException = exception.InnerException is null ? null : $"{exception.InnerException}\t{exception.InnerException.StackTrace}", Url = kv.Value, Type = "Booster Pack" });
private async Task <BoosterProcessorResponse> ProcessBoosters(IDictionary <string, string> tcgLinks, IDictionary <string, string> ocgLinks) { var nameToLinks = tcgLinks.Union(ocgLinks).ToList(); var size = nameToLinks.Count; if (Options.MaxBoostersToParse <= size) { nameToLinks = nameToLinks.RandomSubset(Options.MaxBoostersToParse).ToList(); size = nameToLinks.Count; } //nameToLinks = nameToLinks.Take(Options.MaxBoostersToParse); var errors = new ConcurrentBag <Error>(); var repo = new YuGiOhRepository(this); var count = 0; await Parallel.ForEachAsync(nameToLinks, ParallelOptions, async (nameToLink, _) => { var retryCount = 0; Exception check; do { var(name, link) = nameToLink; try { var parser = new BoosterPackParser(name, link); var card = await parser.Parse(); card.TcgExists = tcgLinks.ContainsKey(name); card.OcgExists = ocgLinks.ContainsKey(name); await repo.InsertBoosterPack(card); check = null; } catch (Exception ex) { retryCount++; check = ex; if (retryCount == Options.Config.MaxRetry) { errors.Add(new Error { Name = name, Message = ex.Message, StackTrace = ex.StackTrace, Url = string.Format(ConstantString.YugipediaUrl + ConstantString.MediaWikiIdUrl, link), Type = Type.Booster, Timestamp = DateTime.UtcNow }); } await Task.Delay(Options.Config.RetryDelay, CancellationToken.None); } } while (check is not null && retryCount < Options.Config.MaxRetry); var __ = WriteProgress("Booster packs", ref count, size); }); ClearLine(); return(new BoosterProcessorResponse { Count = size, Errors = errors }); }