Ejemplo n.º 1
0
        private void findCards(FsPath decksLocation, RegexDeckFormatter formatter)
        {
            var matches = decksLocation.EnumerateFiles(formatter.FileNamePattern, SearchOption.AllDirectories)
                          .SelectMany(file => file.ReadAllLines().Select(line => new { line, file }))
                          .GroupBy(_ => _.line)
                          .Select(gr =>
            {
                string line = gr.Key;
                var match   = formatter.LineRegex.Match(line);

                var card = match.Success
                                                ? formatter.GetCard(match)
                                                : null;

                return(new { match, card, files = gr.Select(_ => _.file).ToArray() });
            })
                          .Where(_ => _.match.Success)
                          .ToArray();

            if (matches.Length == 0)
            {
                Assert.Fail("Decks not found");
            }

            foreach (var match in matches)
            {
                if (match.card == null)
                {
                    Log.Debug("NOT FOUND {0} in\r\n{1}", match.match.Value, string.Join("\r\n", match.files.Select(_ => '\t' + _.Value)));
                }
            }
        }
Ejemplo n.º 2
0
        private static IEnumerable <FsPath> getDirectoryFiles(Dictionary <FsPath, IList <FsPath> > filesByDirCache, FsPath path)
        {
            if (filesByDirCache.TryGetValue(path, out var cache))
            {
                foreach (FsPath file in cache)
                {
                    yield return(file);
                }
            }
            else
            {
                cache = new List <FsPath>();

                foreach (FsPath file in path.EnumerateFiles(option: SearchOption.AllDirectories))
                {
                    if (!_extensions.Contains(file.Extension()))
                    {
                        continue;
                    }

                    cache.Add(file);
                    yield return(file);
                }

                filesByDirCache.Add(path, cache);
            }
        }
Ejemplo n.º 3
0
        public Dictionary <string, Dictionary <string, TcgCard> > ParseCards()
        {
            var result = new Dictionary <string, Dictionary <string, TcgCard> >();

            FsPath dir = _resourcesDir.Join("pages");

            var files = dir.EnumerateFiles("*.html");

            foreach (var file in files)
            {
                var    content = file.ReadAllText();
                string setCode = file.Basename(extension: false);

                var    tableBeginPosition = content.IndexOf("<table class=\"priceGuideTable", 0, Str.Comparison);
                string closingTag         = "</table>";
                var    tableEndPosition   = content.IndexOf(closingTag, tableBeginPosition, Str.Comparison);
                var    tableElement       = XElement.Parse(content
                                                           .Substring(tableBeginPosition, tableEndPosition + closingTag.Length - tableBeginPosition)
                                                           .Replace("&mdash;", string.Empty));

                var trs = tableElement.Element("tbody").Elements("tr").ToArray();

                var cards = new Dictionary <string, TcgCard>();
                result.Add(setCode, cards);

                foreach (var tr in trs)
                {
                    var tds = tr.Elements("td").ToArray();

                    var cellWrapperDivs = tds[0].Element("div").Elements("div").ToArray();

                    var thumbnailUrl = cellWrapperDivs[0].Element("a").Element("img").Attribute("data-original").Value;
                    var sidBegin     = thumbnailUrl.LastIndexOf("/", Str.Comparison);
                    var sidEnd       = thumbnailUrl.IndexOf("_", sidBegin, Str.Comparison);

                    var a         = cellWrapperDivs[1].Element("a");
                    var url       = a.Attribute("href").Value;
                    var codeBegin = url.LastIndexOf("/", Str.Comparison);

                    cards.Add(url.Substring(codeBegin + 1),
                              new TcgCard
                    {
                        Id                 = thumbnailUrl.Substring(sidBegin + 1, sidEnd - sidBegin - 1),
                        Name               = a.Value,
                        Number             = tds[2].Element("div").Value.Trim().NullIfEmpty(),
                        MarketPrice        = parsePrice(tds[3].Element("div")),
                        BuylistMarketPrice = parsePrice(tds[5].Element("div")),
                        ListedMedianPrice  = parsePrice(tds[6].Element("div")),
                        Rarity             = tds[1].Element("div").Value.Trim()
                    });
                }
            }

            return(result);
        }
Ejemplo n.º 4
0
 public void EnsureNoCompressedDuplicates()
 {
     foreach (FsPath qualityDir in getQualities(small: true, zoom: true))
     {
         foreach ((_, _, FsPath tokenSuffix) in getIsToken(nonToken: true, token: true))
         {
             FsPath compressedRoot     = TargetDir.Join(qualityDir).Concat(tokenSuffix).Concat(ZipDirSuffix);
             var    caseInsensitiveMap = new Dictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
             using var scope = new AssertionScope($"Duplicates in {compressedRoot}");
             foreach (var name in compressedRoot.EnumerateFiles("*.7z", SearchOption.TopDirectoryOnly))
             {
                 string nameRelative = name.Basename();
                 caseInsensitiveMap.Should().NotContainKey(nameRelative);
                 caseInsensitiveMap[nameRelative] = true;
             }
         }
     }
 }
Ejemplo n.º 5
0
        public async Task DownloadGathererImages(string setCodesStr, bool nonToken, bool token)
        {
            var clients = new List <ImageDownloaderBase>(2)
            {
                // new GathererClient(),
                new ScryfallClient(),
            };

            var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
            var repo     = new CardRepository(new CardFormatter(), () => null)
            {
                FilterSetCode = setCode => setCodes?.Contains(setCode) != false,
            };

            repo.LoadFile();
            repo.Load();

            foreach (Set set in repo.SetsByCode.Values)
            {
                if (setCodes?.Contains(set.Code) == false)
                {
                    continue;
                }

                foreach ((bool isToken, FsPath typeSubdir, _) in getIsToken(nonToken, token))
                {
                    var cards = set.List(isToken);
                    if (cards.Count == 0)
                    {
                        continue;
                    }

                    var missingCardsByClient = clients.ToDictionary(_ => _, _ => 0);

                    FsPath setSubdir        = new FsPath(set.Code + (Str.Equals(set.Code, "con") ? " escape" : string.Empty));
                    FsPath downloadRootDir  = DevPaths.MtgContentDir.Join(_createZoom ? OriginalSubdir : PreProcessedSubdir);
                    FsPath rootDirZoom      = DevPaths.MtgContentDir.Join(PreProcessedSubdir);
                    FsPath setDirectory     = downloadRootDir.Join(typeSubdir, setSubdir);
                    FsPath setDirectoryZoom = rootDirZoom.Join(typeSubdir, setSubdir);
                    FsPath setDirectoryPng  = setDirectory.Concat(".png");

                    bool dirExisted = setDirectoryPng.IsDirectory();
                    if (!dirExisted)
                    {
                        setDirectoryPng.CreateDirectory();
                    }

                    foreach (var card in cards)
                    {
                        FsPath targetFile        = setDirectoryPng.Join(card.ImageName + ".png");
                        FsPath processedFile     = setDirectory.Join(card.ImageName + ".jpg");
                        FsPath processedFileZoom = setDirectoryZoom.Join(card.ImageName + ".jpg");
                        if (targetFile.IsFile() || processedFile.IsFile() && processedFileZoom.IsFile())
                        {
                            continue;
                        }

                        if (targetFile.Basename(extension: false).EndsWith("1"))
                        {
                            var unnumbered = targetFile.WithName(_ => _.Replace("1.png", ".png"));
                            if (unnumbered.IsFile())
                            {
                                unnumbered.MoveFileTo(targetFile);
                                continue;
                            }
                        }

                        foreach (ImageDownloaderBase client in clients)
                        {
                            int attempts = 5;

                            for (int i = 0; i < attempts; i++)
                            {
                                var cancellation = new CancellationTokenSource();
                                var time         = DateTime.UtcNow;

                                var downloadTask = client.DownloadCardImage(card, targetFile, cancellation.Token);
                                var waitTask     = Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token);

                                await Task.WhenAny(downloadTask, waitTask);

                                cancellation.Cancel();

                                var elapsed = DateTime.UtcNow - time;
                                var delta   = TimeSpan.FromSeconds(0.5) - elapsed;
                                if (delta.TotalSeconds > 0)
                                {
                                    await Task.Delay(delta);
                                }

                                if (targetFile.IsFile())
                                {
                                    break;
                                }
                            }

                            if (targetFile.IsFile())
                            {
                                break;
                            }

                            missingCardsByClient[client]++;
                        }
                    }

                    if (!dirExisted && !setDirectoryPng.EnumerateFiles().Any())
                    {
                        setDirectoryPng.DeleteDirectory();
                    }
                }
            }
        }
Ejemplo n.º 6
0
        public void PreProcessImages(string setCodesStr, bool nonToken, bool token)
        {
            setupImageConversion();

            FsPath smallDir    = DevPaths.GathererOriginalDir;
            FsPath zoomDir     = DevPaths.GathererPreprocessedDir;
            FsPath smallDirBak = BakDir.Join(smallDir.Basename());
            FsPath zoomDirBak  = BakDir.Join(zoomDir.Basename());

            var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);

            IEnumerable <FsPath> getSetSubdirs(FsPath typeSubdir)
            {
                FsPath typeDir = smallDir.Join(typeSubdir);

                return(typeDir
                       .EnumerateDirectories(_fromPng ? "*.png" : "*", SearchOption.TopDirectoryOnly)
                       .Select(_ => new FsPath(
                                   Regex.Replace(
                                       _.Basename(),
                                       @"\.png$",
                                       string.Empty))));
            }

            foreach ((_, FsPath typeSubdir, _) in getIsToken(nonToken, token))
            {
                foreach (FsPath setSubdir in getSetSubdirs(typeSubdir))
                {
                    if (setCodes?.Contains(setSubdir.Value) == false)
                    {
                        continue;
                    }

                    FsPath smallJpgDir = smallDir.Join(typeSubdir, setSubdir);
                    FsPath zoomJpgDir  = zoomDir.Join(typeSubdir, setSubdir);

                    if (!_fromPng)
                    {
                        if (!_createZoom)
                        {
                            return;
                        }

                        zoomJpgDir.CreateDirectory();
                        foreach (FsPath smallImg in smallJpgDir.EnumerateFiles())
                        {
                            FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
                            if (_keepExisting && zoomImg.IsFile() && isZoomed(zoomImg))
                            {
                                continue;
                            }

                            scale(smallImg, zoomImg);
                        }
                    }
                    else
                    {
                        FsPath setPngSubdir   = setSubdir.Concat(".png");
                        FsPath smallPngDir    = smallDir.Join(typeSubdir, setPngSubdir);
                        FsPath smallPngDirBak = smallDirBak.Join(typeSubdir, setPngSubdir);
                        FsPath zoomPngDir     = zoomDir.Join(typeSubdir, setPngSubdir);
                        FsPath zoomPngDirBak  = zoomDirBak.Join(typeSubdir, setPngSubdir);

                        var dirs = new List <(FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom)>();
                        if (_createZoom)
                        {
                            zoomPngDir.CreateDirectory();
                            foreach (FsPath smallImg in smallPngDir.EnumerateFiles())
                            {
                                FsPath zoomImg      = smallImg.ChangeDirectory(smallDir, zoomDir);
                                FsPath convertedImg = zoomImg.ChangeDirectory(zoomPngDir, zoomJpgDir)
                                                      .WithName(_ => _.Replace(".png", ".jpg"));

                                if (_keepExisting && (
                                        zoomImg.IsFile() && isZoomed(zoomImg) ||
                                        convertedImg.IsFile() && isZoomed(convertedImg)))
                                {
                                    continue;
                                }

                                scale(smallImg, zoomImg);
                            }

                            dirs.Add((pngDir: zoomPngDir, pngDirBak: zoomPngDirBak, jpgDir: zoomJpgDir, isZoom: true));
                        }

                        dirs.Add((pngDir: smallPngDir, pngDirBak: smallPngDirBak, jpgDir: smallJpgDir, isZoom: false));

                        foreach ((FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom) in dirs)
                        {
                            jpgDir.CreateDirectory();

                            var pngImages = pngDir.EnumerateFiles();
                            foreach (FsPath sourceImage in pngImages)
                            {
                                convertToJpg(sourceImage, jpgDir, isZoom);
                            }

                            moveDirectoryToBackup(pngDir, pngDirBak);
                        }
                    }
                }
            }
        }
Ejemplo n.º 7
0
        private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
        {
            FsPath targetSubdirectory = progress.TargetSubdirectory;

            targetSubdirectory.CreateDirectory();

            if (progress.FilesOnline == null)
            {
                return(false);
            }

            bool alreadyDownloaded = true;

            var existingFiles = new HashSet <FsPath>(
                targetSubdirectory.EnumerateFiles("*", SearchOption.AllDirectories));

            var existingSignatures = new Dictionary <FsPath, FileSignature>();

            foreach (var fileOnline in progress.FilesOnline.Values)
            {
                FsPath filePath = targetSubdirectory.Join(fileOnline.Path);
                if (!existingFiles.Contains(filePath))
                {
                    alreadyDownloaded = false;
                    continue;
                }

                FileSignature tempQualifier     = Signer.CreateSignature(filePath, useAbsolutePath: true);
                var           existingSignature =
                    progress.FilesCorrupted.TryGet(fileOnline.Path) ??
                    progress.FilesDownloaded.TryGet(fileOnline.Path) ??
                    new FileSignature
                {
                    Path    = tempQualifier.Path.RelativeTo(targetSubdirectory).Intern(true),
                    Md5Hash = tempQualifier.Md5Hash
                };

                if (existingSignature.Md5Hash != fileOnline.Md5Hash)
                {
                    alreadyDownloaded = false;
                    Console.WriteLine("Deleting modified or corrupted file {0}", filePath);

                    lock (ImageLoader.SyncIo)
                    {
                        try
                        {
                            filePath.DeleteFile();
                        }
                        catch (IOException ex)
                        {
                            Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
                        }
                    }
                }
                else
                {
                    existingSignatures.Add(existingSignature.Path, existingSignature);
                }
            }

            foreach (FsPath file in existingFiles)
            {
                var relativePath = file.RelativeTo(targetSubdirectory);
                if (!progress.FilesOnline.ContainsKey(relativePath) && relativePath != Signer.SignaturesFile)
                {
                    Console.WriteLine("Deleting {0}", file);
                    file.DeleteFile();
                }
            }

            if (alreadyDownloaded)
            {
                ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
            }

            return(alreadyDownloaded);
        }