private IEnumerable <BoosterPack> GetBoosterPacks(IDictionary <string, string> links, out IEnumerable <Error> errors)
{
var boosterPacks = new ConcurrentBag <BoosterPack>();
var total = links.Count;
var current = 0;
var tempErrors = new ConcurrentBag <Error>();
Parallel.ForEach(links, ScraperConstants.ParallelOptions, kv =>
{
try
{
var boosterPack = new BoosterPackParser(kv.Key, $"{ScraperConstants.YuGiPediaUrl}{ScraperConstants.MediaWikiParseIdUrl}{kv.Value}").Parse(HttpClient);
boosterPack.Url = $"{ScraperConstants.YuGiPediaUrl}?curid={kv.Value}";
#region OCG TCG
boosterPack.TcgExists = TcgBoosters.ContainsKey(boosterPack.Name);
boosterPack.OcgExists = OcgBoosters.ContainsKey(boosterPack.Name);
#endregion OCG TCG
boosterPacks.Add(boosterPack);
}
catch (Exception exception)
{
tempErrors.Add(new Error()
{
Name = kv.Key,
Exception = $"{exception.Message}\t{exception.StackTrace}",
InnerException = exception.InnerException is null ? null : $"{exception.InnerException}\t{exception.InnerException.StackTrace}",
Url = kv.Value,
Type = "Booster Pack"
});
protected override async Task <IDictionary <string, string> > GetBoosterPackLinks()
{
Console.WriteLine("Retrieving TCG and OCG booster pack list...");
TcgBoosters = await AggregateInfo(ScraperConstants.MediaWikiTcgPacks);
OcgBoosters = await AggregateInfo(ScraperConstants.MediaWikiOcgPacks);
Console.WriteLine("Retrieved TCG and OCG booster pack list.");
Console.WriteLine("Parsing returned response...");
Console.WriteLine("Finished parsing returned response.");
return(TcgBoosters.Concat(OcgBoosters).DistinctBy(kv => kv.Key).DoIf(list => Settings.BoosterPackAmount != -1, list => list.Take(Settings.BoosterPackAmount)).ToDictionary(kv => kv.Key, kv => kv.Value));
}
