private async Task AddCachedImageAsync(WeissSchwarzCard card, Func <Flurl.Url, CookieSession> _cookieSession, CancellationToken ct = default)
{
try
{
var imgURL = card.Images.Last();
Log.Information("Caching: {imgURL}", imgURL);
var session = _cookieSession(imgURL);
using (System.IO.Stream netStream = await card.GetImageStreamAsync(session, ct))
using (Image img = Image.Load(netStream))
{
var imageDirectoryPath = Path.Get(_IMAGE_CACHE_PATH);
if (!imageDirectoryPath.Exists)
{
imageDirectoryPath.CreateDirectory();
}
if (img.Height < img.Width)
{
Log.Debug("Image is probably incorrectly oriented, rotating it 90 degs. clockwise to compensate.");
img.Mutate(ipc => ipc.Rotate(90));
}
img.Metadata.ExifProfile ??= new ExifProfile();
img.Metadata.ExifProfile.SetValue(SixLabors.ImageSharp.Metadata.Profiles.Exif.ExifTag.Copyright, card.Images.Last().Authority);
var savePath = Path.Get(_IMAGE_CACHE_PATH).Combine($"{card.Serial.Replace('-', '_').AsFileNameFriendly()}.jpg");
await img.SaveAsPngAsync(savePath.FullPath, ct);
}
} catch (InvalidOperationException e) when(e.Message == "Sequence contains no elements")
{
Log.Warning("Cannot be cached as no image URLs were found: {serial}", card.Serial);
}
}
private async Task AddCachedImageAsync(WeissSchwarzCard card)
{
try
{
var imgURL = card.Images.Last();
Log.Information("Caching: {imgURL}", imgURL);
using (System.IO.Stream netStream = await card.GetImageStreamAsync())
using (Image img = Image.Load(netStream))
{
var imageDirectoryPath = Path.Get(_IMAGE_CACHE_PATH);
if (!imageDirectoryPath.Exists)
{
imageDirectoryPath.CreateDirectory();
}
img.Metadata.ExifProfile ??= new ExifProfile();
img.Metadata.ExifProfile.SetValue(SixLabors.ImageSharp.Metadata.Profiles.Exif.ExifTag.Copyright, card.Images.Last().Authority);
var savePath = Path.Get(_IMAGE_CACHE_PATH).Combine($"{card.Serial.Replace('-', '_').AsFileNameFriendly()}.jpg");
savePath.Open(img.SaveAsJpeg);
}
} catch (InvalidOperationException e) when(e.Message == "Sequence contains no elements")
{
Log.Warning("Cannot be cached as no image URLs were found: {serial}", card.Serial);
}
}
private async Task <WeissSchwarzDeck> Parse(Uri uri)
{
var encoreDecksDeckAPIURL = "https://www.encoredecks.com/api/deck";
var localPath = uri.LocalPath;
var deckID = localPath.Substring(localPath.LastIndexOf('/') + 1);
Log.Information("Deck ID: {deckID}", deckID);
dynamic deckJSON = await GetDeckJSON(encoreDecksDeckAPIURL, deckID);
WeissSchwarzDeck res = new WeissSchwarzDeck();
res.Name = deckJSON.name;
using (var db = _database())
{
await db.Database.MigrateAsync();
foreach (dynamic card in deckJSON.cards)
{
string serial = WeissSchwarzCard.GetSerial(card.set.ToString(), card.side.ToString(), card.lang.ToString(), card.release.ToString(), card.sid.ToString());
WeissSchwarzCard wscard = await db.WeissSchwarzCards.FindAsync(serial);
if (wscard == null)
{
string setID = card.series;
await _parse($"https://www.encoredecks.com/api/series/{setID}/cards");
wscard = await db.WeissSchwarzCards.FindAsync(serial);
}
if (res.Ratios.TryGetValue(wscard, out int quantity))
{
res.Ratios[wscard]++;
}
else
{
res.Ratios[wscard] = 1;
}
//Log.Information("Parsed: {@wscard}", wscard);
}
}
var simpleRatios = res.AsSimpleDictionary();
Log.Information("Deck Parsed: {@simpleRatios}", simpleRatios);
Log.Information("Cards in Deck: {@count}", simpleRatios.Values.Sum());
return(res);
}
private async Task <WeissSchwarzCard> AddImageFromConsole(WeissSchwarzCard card, InspectionOptions options)
{
var modifiedCard = card.Clone();
Log.Information("Please enter a new image URL: ");
var newURIString = Console.ReadLine();
try
{
Log.Information("Validating URL...");
var newURI = new Uri(newURIString);
Log.Information("Looks good; validating image... (will not check if the image itself is correct!)");
using (Image img = Image.Load(await newURI.WithImageHeaders().GetStreamAsync()))
{
Log.Information("Image can be loaded. Is the ratio reasonable?");
var aspectRatio = (img.Width * 1.0d) / img.Height;
var flooredAspectRatio = Math.Floor(aspectRatio * 100);
if (flooredAspectRatio < 70 || flooredAspectRatio > 72)
{
Log.Information("Image Ratio ({aspectRatio}) isn't correct (it must be approx. 0.71428571428); Failed inspection.", aspectRatio);
return(null);
}
else
{
if (img.Width < 400)
{
Log.Warning("The image is of low quality; this may not be good for exporting purposes. Continue? [Y/N] (Default is N)");
if (!ConsoleUtils.Prompted(options.IsNonInteractive, options.NoWarning))
{
return(null);
}
}
modifiedCard.Images.Add(newURI);
Log.Information("All preliminary tests passed. Modified {card}.", card.Serial);
}
}
}
catch (UnknownImageFormatException)
{
Log.Error("The URL does not point to a valid image. Inspection failed.");
return(null);
}
catch (Exception e)
{
Log.Error("{e}", e);
return(null);
}
return(modifiedCard);
}
public async IAsyncEnumerable <WeissSchwarzCard> Parse(string urlOrFile)
{
if (encoreDecksSiteSetMatcher.IsMatch(urlOrFile))
{
urlOrFile = TransformIntoAPIFormat(urlOrFile);
}
IList <dynamic> setCards = null;
do
{
try
{
setCards = await urlOrFile.WithRESTHeaders().GetJsonListAsync();
}
catch (FlurlHttpException)
{
// Do nothing
}
} while (setCards == null);
foreach (var setCard in setCards)
{
WeissSchwarzCard result = new WeissSchwarzCard();
result.Name = new MultiLanguageString();
var enOptional = DynamicExtensions.AsOptional(setCard.locale.EN);
var jpOptional = DynamicExtensions.AsOptional(setCard.locale.NP);
result.Name.EN = enOptional.name;
result.Name.JP = jpOptional.name;
(List <object>, List <object>)attributes = (enOptional.attributes, jpOptional.attributes);
result.Traits = TranslateTraits(attributes).ToList();
result.Effect = ((List <object>)enOptional.ability)?.Cast <string>().ToArray();
result.Rarity = setCard.rarity;
result.Side = TranslateSide(setCard.side);
result.Level = (int?)setCard.level;
result.Cost = (int?)setCard.cost;
result.Power = (int?)setCard.power;
result.Soul = (int?)setCard.soul;
result.Triggers = TranslateTriggers(setCard.trigger);
result.Serial = WeissSchwarzCard.GetSerial(setCard.set.ToString(), setCard.side.ToString(), setCard.lang.ToString(), setCard.release.ToString(), setCard.sid.ToString());
result.Type = TranslateType(setCard.cardtype);
result.Color = TranslateColor(setCard.colour);
result.Remarks = $"Parsed: {this.GetType().Name}";
yield return(result);
}
// Get
yield break;
}
public async Task Run(IContainer ioc)
{
Log.Information("Starting.");
var language = InterpretLanguage(Language);
IAsyncEnumerable <WeissSchwarzCard> list = null;
using (var db = ioc.GetInstance <CardDatabaseContext>())
{
await db.Database.MigrateAsync();
if (language == null)
{
try
{
var tuple = WeissSchwarzCard.ParseSerial(ReleaseIDorFullSerialID);
}
catch (Exception)
{
Log.Error("Serial cannot be parsed properly. Did you mean to cache a release set? If so, please indicate the language (EN/JP) as well.");
return;
}
var query = from card in db.WeissSchwarzCards.AsQueryable()
where card.Serial.ToLower() == ReleaseIDorFullSerialID.ToLower()
select card;
list = query.ToAsyncEnumerable().Take(1);
}
else
{
var releaseID = ReleaseIDorFullSerialID.ToLower().Replace("%", "");
var query = from card in db.WeissSchwarzCards.AsQueryable()
where EF.Functions.Like(card.Serial.ToLower(), $"%/{releaseID}%")
select card;
list = query.ToAsyncEnumerable().Where(c => c.Language == language.Value);
}
await foreach (var card in list)
{
await AddCachedImageAsync(card);
}
Log.Information("Done.");
Log.Information("PS: Please refrain from executing this command continuously as this may cause your IP address to get tagged as a DDoS bot.");
Log.Information(" Only cache the images you may need.");
Log.Information(" -ronelm2000");
}
}
private async IAsyncEnumerable <WeissSchwarzCard> Process(WeissSchwarzCard firstCard, IAsyncEnumerable <WeissSchwarzCard> originalCards)
{
Log.Information("Starting...");
var menu = await "http://jktcg.com/MenuLeftEN.html"
.WithHTMLHeaders()
.GetHTMLAsync();
var pair = CardListURLFrom(menu, firstCard);
var cardList = await pair.url
.WithHTMLHeaders()
.GetHTMLAsync();
var releaseID = firstCard.ReleaseID;
var cardImages = cardList.QuerySelectorAll <IHtmlImageElement>("a > img")
.Select(ele => (
Serial: GetSerial(ele),
Source: ele.Source.Replace("\t", "")
))
.ToDictionary(p => p.Serial, p => p.Source);//(setID + "-" + str.AsSpan().Slice(c => c.LastIndexOf('_') + 1, c => c.LastIndexOf(".")).ToString()).ToLower());
// .ToLookup(ele => ele
await foreach (var card in originalCards)
{
var res = card.Clone();
try
{
var imgURL = cardImages[res.Serial.ToLower()];
res.Images.Add(new Uri(imgURL));
Log.Information("Attached image to {serial}: {imgURL}", res.Serial, imgURL);
}
catch (KeyNotFoundException)
{
Log.Warning("Tried to post-process {serial} when the URL for {releaseID} was loaded. Skipping.", res.Serial, releaseID);
}
yield return(res);
}
Log.Information("Finished.");
yield break;
}
private WeissSchwarzCard ParseHOTCText(string hotcText)
{
var cursor = hotcText.AsSpanCursor();
var res = new WeissSchwarzCard();
var cardNoText = "Card No.: ";
var rarityText = "Rarity:";
var colorText = "Color: ";
var sideText = "Side: ";
var levelText = "Level: ";
var costText = "Cost: ";
var powerText = "Power: ";
var soulText = "Soul: ";
var traitsText = "Traits: ";
var trait1Text = "Trait 1: ";
var triggersText = "Triggers: ";
var flavorText = "Flavor:";
var rulesTextText = "TEXT:";
while (!cursor.CurrentLine.StartsWith(cardNoText))
{
cursor.Next();
}
var linesToCardNoText = cursor.LineNumber;
// ReadOnlySpan<char> cardNoLine = cursor.CurrentLine;
res.Serial = cursor.CurrentLine.Slice(
c => c.IndexOf(cardNoText) + cardNoText.Length,
c => c.IndexOf(rarityText)
)
.Trim()
.ToString();
try
{
res.Rarity = cursor.CurrentLine.Slice(c => c.IndexOf(rarityText) + rarityText.Length).Trim().ToString();
} catch (Exception e)
{
res.Rarity = HandleRarityCorrections(res.Serial, e);
}
cursor.MoveUp();
// Log.Information("+1 above Card No: {line}", cursor.CurrentLine.ToString());
res.Name = new MultiLanguageString();
res.Name["jp"] = cursor.CurrentLine.ToString();
if (cursor.LineNumber > 1)
{
cursor.MoveUp();
res.Name["en"] = cursor.CurrentLine.ToString();
}
//cursor.Next(3);
cursor.Next(linesToCardNoText - cursor.LineNumber + 1);
try
{
res.Color = cursor.CurrentLine.Slice(
c => c.IndexOf(colorText) + colorText.Length,
c => c.IndexOf(sideText)
)
.Trim()
.ToEnum <CardColor>()
.Value;
} catch (Exception e)
{
res.Color = HandleColorCorrections(res.Serial, e);
}
try
{
res.Side = cursor.CurrentLine.Slice(
c => c.IndexOf(sideText) + sideText.Length,
c => c.IndexOf(sideText) + sideText.Length + c.Slice(c.IndexOf(sideText) + sideText.Length).IndexOf(' ')
)
.Trim()
.ToEnum <CardSide>()
.Value;
var sideString = res.Side.ToString();
res.Type = cursor.CurrentLine.Slice(
c => c.IndexOf(sideString, StringComparison.CurrentCultureIgnoreCase) + sideString.Length
)
.Trim()
.ToEnum <CardType>()
.Value;
} catch (Exception)
{
(res.Side, res.Type) = HandleCorrections(res.Serial);
}
cursor.Next();
switch (res.Type)
{
case CardType.Character:
res.Power = cursor.CurrentLine.Slice(
c => c.IndexOf(powerText) + powerText.Length,
c => c.IndexOf(soulText)
)
.Trim()
.AsParsed <int>(int.TryParse);
res.Soul = cursor.CurrentLine.Slice(
c => c.IndexOf(soulText) + soulText.Length
)
.Trim()
.AsParsed <int>(int.TryParse);
goto case CardType.Event;
case CardType.Event:
res.Level = cursor.CurrentLine.Slice(
c => c.IndexOf(levelText) + levelText.Length,
c => c.IndexOf(costText)
)
.Trim()
.AsParsed <int>(int.TryParse);
res.Cost = cursor.CurrentLine.Slice(
c => c.IndexOf(costText) + costText.Length,
c => c.IndexOf(powerText)
)
.Trim()
.AsParsed <int>(int.TryParse);
break;
default:
break;
}
cursor.Next();
if (cursor.CurrentLine.ToString().Contains(traitsText))
{
res.Traits = cursor.CurrentLine
.Slice(c => c.IndexOf(traitsText) + traitsText.Length)
.Trim()
.ToString()
.SplitWithRegex(@"([^(]+)\(([^\)]+)\),{0,1}")
.Select(this.ParseTrait)
.Where(o => o != null)
.ToList();
}
else if (cursor.CurrentLine.ToString().Contains(trait1Text))
{
res.Traits = cursor.CurrentLine
.Slice(c => c.IndexOf(trait1Text) + trait1Text.Length)
.Trim()
.ToString()
.Split("Trait 2: ")
.SelectMany(s => s.Trim().SplitWithRegex(@"([^(]+)\(([^\)]+)\),{0,1}")) //TODO: Duplicate Regex, may be identical with traitMatcher.
.Select(this.ParseTrait)
.Where(o => o != null)
.ToList();
}
cursor.Next();
var stringTriggers = cursor.CurrentLine
.Slice(c => c.IndexOf(triggersText) + triggersText.Length)
.ToString();
res.Triggers = TranslateTriggers(stringTriggers.Trim());
cursor.Next();
res.Flavor = cursor.CurrentLine.Slice(c => c.IndexOf(flavorText) + flavorText.Length).ToString();
while (cursor.Next() && !cursor.CurrentLine.StartsWith(rulesTextText))
{
res.Flavor += " " + cursor.CurrentLine.ToString();
}
var stringEffect = cursor.LinesUntilEOS
.Slice(c => c.IndexOf(rulesTextText) + rulesTextText.Length)
.Trim()
.ToString();
// Divide the string into separate lines of actual effects.
var effectSplit = stringEffect
.Replace("[A]", "[AUTO]")
.Replace("[C]", "[CONT]")
.Replace("[S]", "[ACT]")
.Replace("\n[AUTO]", "\n[A][AUTO]")
.Replace("\n[CONT]", "\n[A][CONT]")
.Replace("\n[ACT]", "\n[A][ACT]")
.Split("\n[A]", StringSplitOptions.RemoveEmptyEntries);
res.Effect = effectSplit.Select(s => Clean(s)).ToArray();
res.Remarks = $"Extractor: {this.GetType().Name}";
// foreach (var effect in effectSplit)
// Log.Information("Effect {serial}: {effect}", res.Serial, effect);
Log.Information("Extracted: {serial}", res.Serial);
return(res);
}
private (string setLinkWithUnderscores, string url) CardListURLFrom(IDocument menu, WeissSchwarzCard firstCard)
{
var ogReleaseID = firstCard.ReleaseID;
var releaseIDs = new List <string>();
releaseIDs.Add(firstCard.ReleaseID);
if (ogReleaseID.StartsWith("EN-"))
{
releaseIDs.Add(ogReleaseID.Substring(3));
}
var setLink = menu.Links.Cast <IHtmlAnchorElement>()
.Where(ele => LinkMatcher.IsMatch(ele.Href))
.Where(ele => releaseIDs.Any(s => ele.Href.Contains(s)))
.FirstOrDefault();
if (setLink == null)
{
Log.Error("Cannot find a link that matches {SID} using this list of links: {@items}", ogReleaseID, menu.Links.Cast <IHtmlAnchorElement>().Select(ele => ele.Href).ToList());
throw new Exception();
}
var enPreString = "EN-";
var setLinkWithUnderscores = setLink.Href.AsSpan()
.Slice(x => x.IndexOf(enPreString))
.ToString()
.Replace("-", "_");
return(setLinkWithUnderscores, $"http://jktcg.com/WS_EN/{setLinkWithUnderscores}/{setLinkWithUnderscores}.html");
}
public async Task Run(IContainer ioc, IProgress <CommandProgressReport> progress, CancellationToken cancellationToken = default)
{
Log.Information("Starting.");
var report = CommandProgressReport.Starting(CommandProgressReportVerbType.Caching);
progress.Report(report);
var language = InterpretLanguage(Language);
IAsyncEnumerable <WeissSchwarzCard> list = null;
Func <Flurl.Url, CookieSession> _cookieSession = (url) => ioc.GetInstance <GlobalCookieJar>()[url.Root];
using (var db = ioc.GetInstance <CardDatabaseContext>())
{
await db.Database.MigrateAsync(cancellationToken);
if (language == null)
{
try
{
var tuple = WeissSchwarzCard.ParseSerial(ReleaseIDorFullSerialID);
}
catch (Exception)
{
Log.Error("Serial cannot be parsed properly. Did you mean to cache a release set? If so, please indicate the language (EN/JP) as well.");
return;
}
var query = from card in db.WeissSchwarzCards.AsQueryable()
where card.Serial.ToLower() == ReleaseIDorFullSerialID.ToLower()
select card;
list = query.ToAsyncEnumerable().Take(1);
}
else
{
var releaseID = ReleaseIDorFullSerialID.ToLower().Replace("%", "");
var query = from card in db.WeissSchwarzCards.AsQueryable()
where EF.Functions.Like(card.Serial.ToLower(), $"%/{releaseID}%")
select card;
list = query.ToAsyncEnumerable().Where(c => c.Language == language.Value);
}
await foreach (var card in list.WithCancellation(cancellationToken))
{
report = report with
{
MessageType = MessageType.InProgress,
ReportMessage = new Card.API.Entities.Impls.MultiLanguageString
{
EN = $"Caching [${card.Serial}]..."
},
Percentage = 50
};
progress.Report(report);
await AddCachedImageAsync(card, _cookieSession, cancellationToken);
}
Log.Information("Done.");
Log.Information("PS: Please refrain from executing this command continuously as this may cause your IP address to get tagged as a DDoS bot.");
Log.Information(" Only cache the images you may need.");
Log.Information(" -ronelm2000");
}
report = report.AsDone();
progress.Report(report);
}
private async IAsyncEnumerable <WeissSchwarzCard> Process(WeissSchwarzCard firstCard, IAsyncEnumerable <WeissSchwarzCard> originalCards)
{
Log.Information("Starting...");
var menu = await "http://jktcg.com/MenuLeftEN.html"
.WithHTMLHeaders()
.GetHTMLAsync();
var pair = CardListURLFrom(menu, firstCard);
var cardList = await pair.url
.WithHTMLHeaders()
.GetHTMLAsync();
var releaseID = firstCard.ReleaseID;
var cardImages = cardList.QuerySelectorAll <IHtmlImageElement>("a > img")
.Select(ele => (
Serial: GetSerial(ele),
Source: ele.Source.Replace("\t", "")
))
.ToDictionary(p => p.Serial, p => p.Source);//(setID + "-" + str.AsSpan().Slice(c => c.LastIndexOf('_') + 1, c => c.LastIndexOf(".")).ToString()).ToLower());
/* Commented for future use
* Log.Information("Getting all PRs on card database without a YYT image link...");
* using (var db = _database())
* {
* var prCards = db.WeissSchwarzCards.AsAsyncEnumerable()
* .Where(c => c.ReleaseID == releaseID
* && c.Language == CardLanguage.English
* && c.Rarity == "PR"
* && !c.Images.Any(u => u.Authority == "jktcg.com")
* );
* await foreach (var prCard in prCards)
* {
* if (cardImages.TryGetValue(prCard.Serial, out var urlLink))
* {
* var imgUrl = new Uri(urlLink);
* prCard.Images.Add(imgUrl);
* db.Update(prCard);
* Log.Information("Attached to {serial}: {imgUrl}", prCard.Serial, urlLink);
* }
* }
* await db.SaveChangesAsync();
* }
*/
await foreach (var card in originalCards)
{
var res = card.Clone();
try
{
var imgURL = cardImages[res.Serial.ToLower()];
res.Images.Add(new Uri(imgURL));
Log.Information("Attached image to {serial}: {imgURL}", res.Serial, imgURL);
}
catch (KeyNotFoundException)
{
Log.Warning("Tried to post-process {serial} when the URL for {releaseID} was loaded. Skipping.", res.Serial, releaseID);
}
yield return(res);
}
Log.Information("Finished.");
yield break;
}