private IEnumerable <BoosterPack> GetBoosterPacks(IDictionary <string, string> links, out IEnumerable <Error> errors)
{
var boosterPacks = new ConcurrentBag <BoosterPack>();
var total = links.Count;
var current = 0;
var tempErrors = new ConcurrentBag <Error>();
Parallel.ForEach(links, ScraperConstants.ParallelOptions, kv =>
{
try
{
var boosterPack = new BoosterPackParser(kv.Key, $"{ScraperConstants.YuGiPediaUrl}{ScraperConstants.MediaWikiParseIdUrl}{kv.Value}").Parse(HttpClient);
boosterPack.Url = $"{ScraperConstants.YuGiPediaUrl}?curid={kv.Value}";
#region OCG TCG
boosterPack.TcgExists = TcgBoosters.ContainsKey(boosterPack.Name);
boosterPack.OcgExists = OcgBoosters.ContainsKey(boosterPack.Name);
#endregion OCG TCG
boosterPacks.Add(boosterPack);
}
catch (Exception exception)
{
tempErrors.Add(new Error()
{
Name = kv.Key,
Exception = $"{exception.Message}\t{exception.StackTrace}",
InnerException = exception.InnerException is null ? null : $"{exception.InnerException}\t{exception.InnerException.StackTrace}",
Url = kv.Value,
Type = "Booster Pack"
});
private async Task <BoosterProcessorResponse> ProcessBoosters(IDictionary <string, string> tcgLinks, IDictionary <string, string> ocgLinks)
{
var nameToLinks = tcgLinks.Union(ocgLinks).ToList();
var size = nameToLinks.Count;
if (Options.MaxBoostersToParse <= size)
{
nameToLinks = nameToLinks.RandomSubset(Options.MaxBoostersToParse).ToList();
size = nameToLinks.Count;
}
//nameToLinks = nameToLinks.Take(Options.MaxBoostersToParse);
var errors = new ConcurrentBag <Error>();
var repo = new YuGiOhRepository(this);
var count = 0;
await Parallel.ForEachAsync(nameToLinks, ParallelOptions, async (nameToLink, _) =>
{
var retryCount = 0;
Exception check;
do
{
var(name, link) = nameToLink;
try
{
var parser = new BoosterPackParser(name, link);
var card = await parser.Parse();
card.TcgExists = tcgLinks.ContainsKey(name);
card.OcgExists = ocgLinks.ContainsKey(name);
await repo.InsertBoosterPack(card);
check = null;
}
catch (Exception ex)
{
retryCount++;
check = ex;
if (retryCount == Options.Config.MaxRetry)
{
errors.Add(new Error
{
Name = name,
Message = ex.Message,
StackTrace = ex.StackTrace,
Url = string.Format(ConstantString.YugipediaUrl + ConstantString.MediaWikiIdUrl, link),
Type = Type.Booster,
Timestamp = DateTime.UtcNow
});
}
await Task.Delay(Options.Config.RetryDelay, CancellationToken.None);
}
} while (check is not null && retryCount < Options.Config.MaxRetry);
var __ = WriteProgress("Booster packs", ref count, size);
});
ClearLine();
return(new BoosterProcessorResponse
{
Count = size,
Errors = errors
});
}
