private void findCards(FsPath decksLocation, RegexDeckFormatter formatter)
{
var matches = decksLocation.EnumerateFiles(formatter.FileNamePattern, SearchOption.AllDirectories)
.SelectMany(file => file.ReadAllLines().Select(line => new { line, file }))
.GroupBy(_ => _.line)
.Select(gr =>
{
string line = gr.Key;
var match = formatter.LineRegex.Match(line);
var card = match.Success
? formatter.GetCard(match)
: null;
return(new { match, card, files = gr.Select(_ => _.file).ToArray() });
})
.Where(_ => _.match.Success)
.ToArray();
if (matches.Length == 0)
{
Assert.Fail("Decks not found");
}
foreach (var match in matches)
{
if (match.card == null)
{
Log.Debug("NOT FOUND {0} in\r\n{1}", match.match.Value, string.Join("\r\n", match.files.Select(_ => '\t' + _.Value)));
}
}
}
private static IEnumerable <FsPath> getDirectoryFiles(Dictionary <FsPath, IList <FsPath> > filesByDirCache, FsPath path)
{
if (filesByDirCache.TryGetValue(path, out var cache))
{
foreach (FsPath file in cache)
{
yield return(file);
}
}
else
{
cache = new List <FsPath>();
foreach (FsPath file in path.EnumerateFiles(option: SearchOption.AllDirectories))
{
if (!_extensions.Contains(file.Extension()))
{
continue;
}
cache.Add(file);
yield return(file);
}
filesByDirCache.Add(path, cache);
}
}
public Dictionary <string, Dictionary <string, TcgCard> > ParseCards()
{
var result = new Dictionary <string, Dictionary <string, TcgCard> >();
FsPath dir = _resourcesDir.Join("pages");
var files = dir.EnumerateFiles("*.html");
foreach (var file in files)
{
var content = file.ReadAllText();
string setCode = file.Basename(extension: false);
var tableBeginPosition = content.IndexOf("<table class=\"priceGuideTable", 0, Str.Comparison);
string closingTag = "</table>";
var tableEndPosition = content.IndexOf(closingTag, tableBeginPosition, Str.Comparison);
var tableElement = XElement.Parse(content
.Substring(tableBeginPosition, tableEndPosition + closingTag.Length - tableBeginPosition)
.Replace("—", string.Empty));
var trs = tableElement.Element("tbody").Elements("tr").ToArray();
var cards = new Dictionary <string, TcgCard>();
result.Add(setCode, cards);
foreach (var tr in trs)
{
var tds = tr.Elements("td").ToArray();
var cellWrapperDivs = tds[0].Element("div").Elements("div").ToArray();
var thumbnailUrl = cellWrapperDivs[0].Element("a").Element("img").Attribute("data-original").Value;
var sidBegin = thumbnailUrl.LastIndexOf("/", Str.Comparison);
var sidEnd = thumbnailUrl.IndexOf("_", sidBegin, Str.Comparison);
var a = cellWrapperDivs[1].Element("a");
var url = a.Attribute("href").Value;
var codeBegin = url.LastIndexOf("/", Str.Comparison);
cards.Add(url.Substring(codeBegin + 1),
new TcgCard
{
Id = thumbnailUrl.Substring(sidBegin + 1, sidEnd - sidBegin - 1),
Name = a.Value,
Number = tds[2].Element("div").Value.Trim().NullIfEmpty(),
MarketPrice = parsePrice(tds[3].Element("div")),
BuylistMarketPrice = parsePrice(tds[5].Element("div")),
ListedMedianPrice = parsePrice(tds[6].Element("div")),
Rarity = tds[1].Element("div").Value.Trim()
});
}
}
return(result);
}
public void EnsureNoCompressedDuplicates()
{
foreach (FsPath qualityDir in getQualities(small: true, zoom: true))
{
foreach ((_, _, FsPath tokenSuffix) in getIsToken(nonToken: true, token: true))
{
FsPath compressedRoot = TargetDir.Join(qualityDir).Concat(tokenSuffix).Concat(ZipDirSuffix);
var caseInsensitiveMap = new Dictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
using var scope = new AssertionScope($"Duplicates in {compressedRoot}");
foreach (var name in compressedRoot.EnumerateFiles("*.7z", SearchOption.TopDirectoryOnly))
{
string nameRelative = name.Basename();
caseInsensitiveMap.Should().NotContainKey(nameRelative);
caseInsensitiveMap[nameRelative] = true;
}
}
}
}
public async Task DownloadGathererImages(string setCodesStr, bool nonToken, bool token)
{
var clients = new List <ImageDownloaderBase>(2)
{
// new GathererClient(),
new ScryfallClient(),
};
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
var repo = new CardRepository(new CardFormatter(), () => null)
{
FilterSetCode = setCode => setCodes?.Contains(setCode) != false,
};
repo.LoadFile();
repo.Load();
foreach (Set set in repo.SetsByCode.Values)
{
if (setCodes?.Contains(set.Code) == false)
{
continue;
}
foreach ((bool isToken, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
var cards = set.List(isToken);
if (cards.Count == 0)
{
continue;
}
var missingCardsByClient = clients.ToDictionary(_ => _, _ => 0);
FsPath setSubdir = new FsPath(set.Code + (Str.Equals(set.Code, "con") ? " escape" : string.Empty));
FsPath downloadRootDir = DevPaths.MtgContentDir.Join(_createZoom ? OriginalSubdir : PreProcessedSubdir);
FsPath rootDirZoom = DevPaths.MtgContentDir.Join(PreProcessedSubdir);
FsPath setDirectory = downloadRootDir.Join(typeSubdir, setSubdir);
FsPath setDirectoryZoom = rootDirZoom.Join(typeSubdir, setSubdir);
FsPath setDirectoryPng = setDirectory.Concat(".png");
bool dirExisted = setDirectoryPng.IsDirectory();
if (!dirExisted)
{
setDirectoryPng.CreateDirectory();
}
foreach (var card in cards)
{
FsPath targetFile = setDirectoryPng.Join(card.ImageName + ".png");
FsPath processedFile = setDirectory.Join(card.ImageName + ".jpg");
FsPath processedFileZoom = setDirectoryZoom.Join(card.ImageName + ".jpg");
if (targetFile.IsFile() || processedFile.IsFile() && processedFileZoom.IsFile())
{
continue;
}
if (targetFile.Basename(extension: false).EndsWith("1"))
{
var unnumbered = targetFile.WithName(_ => _.Replace("1.png", ".png"));
if (unnumbered.IsFile())
{
unnumbered.MoveFileTo(targetFile);
continue;
}
}
foreach (ImageDownloaderBase client in clients)
{
int attempts = 5;
for (int i = 0; i < attempts; i++)
{
var cancellation = new CancellationTokenSource();
var time = DateTime.UtcNow;
var downloadTask = client.DownloadCardImage(card, targetFile, cancellation.Token);
var waitTask = Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token);
await Task.WhenAny(downloadTask, waitTask);
cancellation.Cancel();
var elapsed = DateTime.UtcNow - time;
var delta = TimeSpan.FromSeconds(0.5) - elapsed;
if (delta.TotalSeconds > 0)
{
await Task.Delay(delta);
}
if (targetFile.IsFile())
{
break;
}
}
if (targetFile.IsFile())
{
break;
}
missingCardsByClient[client]++;
}
}
if (!dirExisted && !setDirectoryPng.EnumerateFiles().Any())
{
setDirectoryPng.DeleteDirectory();
}
}
}
}
public void PreProcessImages(string setCodesStr, bool nonToken, bool token)
{
setupImageConversion();
FsPath smallDir = DevPaths.GathererOriginalDir;
FsPath zoomDir = DevPaths.GathererPreprocessedDir;
FsPath smallDirBak = BakDir.Join(smallDir.Basename());
FsPath zoomDirBak = BakDir.Join(zoomDir.Basename());
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
IEnumerable <FsPath> getSetSubdirs(FsPath typeSubdir)
{
FsPath typeDir = smallDir.Join(typeSubdir);
return(typeDir
.EnumerateDirectories(_fromPng ? "*.png" : "*", SearchOption.TopDirectoryOnly)
.Select(_ => new FsPath(
Regex.Replace(
_.Basename(),
@"\.png$",
string.Empty))));
}
foreach ((_, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
foreach (FsPath setSubdir in getSetSubdirs(typeSubdir))
{
if (setCodes?.Contains(setSubdir.Value) == false)
{
continue;
}
FsPath smallJpgDir = smallDir.Join(typeSubdir, setSubdir);
FsPath zoomJpgDir = zoomDir.Join(typeSubdir, setSubdir);
if (!_fromPng)
{
if (!_createZoom)
{
return;
}
zoomJpgDir.CreateDirectory();
foreach (FsPath smallImg in smallJpgDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
if (_keepExisting && zoomImg.IsFile() && isZoomed(zoomImg))
{
continue;
}
scale(smallImg, zoomImg);
}
}
else
{
FsPath setPngSubdir = setSubdir.Concat(".png");
FsPath smallPngDir = smallDir.Join(typeSubdir, setPngSubdir);
FsPath smallPngDirBak = smallDirBak.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDir = zoomDir.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDirBak = zoomDirBak.Join(typeSubdir, setPngSubdir);
var dirs = new List <(FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom)>();
if (_createZoom)
{
zoomPngDir.CreateDirectory();
foreach (FsPath smallImg in smallPngDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
FsPath convertedImg = zoomImg.ChangeDirectory(zoomPngDir, zoomJpgDir)
.WithName(_ => _.Replace(".png", ".jpg"));
if (_keepExisting && (
zoomImg.IsFile() && isZoomed(zoomImg) ||
convertedImg.IsFile() && isZoomed(convertedImg)))
{
continue;
}
scale(smallImg, zoomImg);
}
dirs.Add((pngDir: zoomPngDir, pngDirBak: zoomPngDirBak, jpgDir: zoomJpgDir, isZoom: true));
}
dirs.Add((pngDir: smallPngDir, pngDirBak: smallPngDirBak, jpgDir: smallJpgDir, isZoom: false));
foreach ((FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom) in dirs)
{
jpgDir.CreateDirectory();
var pngImages = pngDir.EnumerateFiles();
foreach (FsPath sourceImage in pngImages)
{
convertToJpg(sourceImage, jpgDir, isZoom);
}
moveDirectoryToBackup(pngDir, pngDirBak);
}
}
}
}
}
private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
{
FsPath targetSubdirectory = progress.TargetSubdirectory;
targetSubdirectory.CreateDirectory();
if (progress.FilesOnline == null)
{
return(false);
}
bool alreadyDownloaded = true;
var existingFiles = new HashSet <FsPath>(
targetSubdirectory.EnumerateFiles("*", SearchOption.AllDirectories));
var existingSignatures = new Dictionary <FsPath, FileSignature>();
foreach (var fileOnline in progress.FilesOnline.Values)
{
FsPath filePath = targetSubdirectory.Join(fileOnline.Path);
if (!existingFiles.Contains(filePath))
{
alreadyDownloaded = false;
continue;
}
FileSignature tempQualifier = Signer.CreateSignature(filePath, useAbsolutePath: true);
var existingSignature =
progress.FilesCorrupted.TryGet(fileOnline.Path) ??
progress.FilesDownloaded.TryGet(fileOnline.Path) ??
new FileSignature
{
Path = tempQualifier.Path.RelativeTo(targetSubdirectory).Intern(true),
Md5Hash = tempQualifier.Md5Hash
};
if (existingSignature.Md5Hash != fileOnline.Md5Hash)
{
alreadyDownloaded = false;
Console.WriteLine("Deleting modified or corrupted file {0}", filePath);
lock (ImageLoader.SyncIo)
{
try
{
filePath.DeleteFile();
}
catch (IOException ex)
{
Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
}
}
}
else
{
existingSignatures.Add(existingSignature.Path, existingSignature);
}
}
foreach (FsPath file in existingFiles)
{
var relativePath = file.RelativeTo(targetSubdirectory);
if (!progress.FilesOnline.ContainsKey(relativePath) && relativePath != Signer.SignaturesFile)
{
Console.WriteLine("Deleting {0}", file);
file.DeleteFile();
}
}
if (alreadyDownloaded)
{
ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
}
return(alreadyDownloaded);
}
