Beispiel #1
0
        public void ZipImages(string setCodesStr, bool small, bool zoom, bool nonToken, bool token)
        {
            var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);

            foreach (FsPath qualityDir in getQualities(small, zoom))
            {
                foreach ((_, _, FsPath tokenSuffix) in getIsToken(nonToken, token))
                {
                    FsPath compressedRoot = TargetDir.Join(qualityDir).Concat(tokenSuffix).Concat(ZipDirSuffix);
                    compressedRoot.CreateDirectory();

                    FsPath sourceRoot = TargetDir.Join(qualityDir).Concat(tokenSuffix);
                    foreach (var subdir in sourceRoot.EnumerateDirectories())
                    {
                        string subdirRelative = subdir.Basename();
                        if (setCodes?.Contains(subdirRelative) == false)
                        {
                            continue;
                        }

                        var targetFile = compressedRoot.Join(subdirRelative).Concat(SevenZipExtension);
                        if (targetFile.IsFile())
                        {
                            targetFile.DeleteFile();
                        }

                        new SevenZip(false).Compress(subdir, targetFile)
                        .Should().BeTrue();
                    }
                }
            }
        }
Beispiel #2
0
        public void SignFiles(FsPath packagePath, FsPath output, string setCodes)
        {
            FsPath parentDir = output.Parent();

            parentDir.CreateDirectory();

            if (packagePath.IsDirectory())
            {
                var sets = setCodes?.Split(';', ',', '|')
                           // to remove duplicates with different set name casing
                           .ToHashSet(StringComparer.OrdinalIgnoreCase);

                var prevSignatureByPath = sets != null && output.IsFile()
                                        ? Signer.ReadFromFile(output, internPath: false)
                                          .Where(_ => !sets.Contains(_.Path.Parent().Value))
                                          .ToDictionary(_ => _.Path)
                                        : new Dictionary <FsPath, FileSignature>();

                var signatures = Signer.CreateSignatures(packagePath, precalculated: prevSignatureByPath);
                Signer.WriteToFile(output, signatures);
            }
            else if (packagePath.IsFile())
            {
                var metadata = Signer.CreateSignature(packagePath);
                Signer.WriteToFile(output, Sequence.Array(metadata));
            }
            else
            {
                Console.WriteLine("Specified path {0} does not exist", packagePath);
            }
        }
Beispiel #3
0
        public void RenameWizardsWebpageImages(string htmlFile, string targetSubdir)
        {
            FsPath htmlPath  = HtmlDir.Join(htmlFile);
            FsPath targetDir = DevPaths.GathererOriginalDir.Join(targetSubdir);

            string htmlFileName  = htmlPath.Basename(extension: false);
            FsPath directoryName = htmlPath.Parent();

            if (!directoryName.HasValue())
            {
                throw new ArgumentException(htmlPath.Value, nameof(htmlPath));
            }

            FsPath filesDirectory = directoryName.Join(htmlFileName + "_files");

            string content = htmlPath.ReadAllText();
            var    matches = _imgTagPattern.Matches(content);

            targetDir.CreateDirectory();
            foreach (Match match in matches)
            {
                string originalFileName = match.Groups["file"].Value;
                string ext = Path.GetExtension(originalFileName);

                FsPath filePath = filesDirectory.Join(originalFileName);

                string name = HttpUtility.HtmlDecode(match.Groups["name"].Value)
                              .Replace(" // ", "");

                FsPath defaultTargetPath = targetDir.Join(name + ext);

                bool defaultTargetExists = defaultTargetPath.IsFile();

                if (defaultTargetExists || getTargetPath(1).IsFile())
                {
                    if (defaultTargetExists)
                    {
                        defaultTargetPath.MoveFileTo(getTargetPath(1));
                    }

                    for (int i = 2; i < 12; i++)
                    {
                        FsPath targetPath = getTargetPath(i);
                        if (!targetPath.IsFile())
                        {
                            filePath.CopyFileTo(targetPath, overwrite: false);
                            break;
                        }
                    }
                }
                else
                {
                    filePath.CopyFileTo(defaultTargetPath, overwrite: false);
                }

                FsPath getTargetPath(int num) =>
                targetDir.Join(name + num + ext);
            }
        }
Beispiel #4
0
        private static FsPath ensureSetSubdirectory(FsPath smallSet)
        {
            Console.WriteLine($"	Creating {smallSet} ...");

            try
            {
                smallSet.CreateDirectory();
            }
            catch (DirectoryNotFoundException)
            {
                // CON is banned as file / folder name in Windows
                smallSet = smallSet.Concat(" escape");
                smallSet.CreateDirectory();
            }

            return(smallSet);
        }
Beispiel #5
0
        public void Save(FsPath file)
        {
            FsPath directory = file.Parent();

            directory.CreateDirectory();
            var state = getState();

            WriteHistory(file, state);
        }
Beispiel #6
0
        public void MigrateHistoryFiles()
        {
            FsPath firstFormDirectory = AppDir.History.Join(0.ToString());

            if (firstFormDirectory.IsDirectory())
            {
                return;
            }

            firstFormDirectory.CreateDirectory();

            foreach (FsPath file in AppDir.History.EnumerateFiles())
            {
                file.CopyFileTo(firstFormDirectory.Join(file.Basename()));
            }
        }
Beispiel #7
0
        public void LoadHistory(FsPath file)
        {
            FsPath directory = file.Parent();

            directory.CreateDirectory();

            if (TryReadHistory(file, out var state))
            {
                _settingsHistory = state.SettingsHistory;
                _settingsIndex   = state.SettingsIndex;
            }
            else
            {
                _settingsHistory = new List <GuiSettings>();
                var defaultSettings = new GuiSettings();
                Add(defaultSettings);
            }

            IsLoaded = true;
            Loaded?.Invoke();
        }
Beispiel #8
0
        public async Task DownloadGathererImages(string setCodesStr, bool nonToken, bool token)
        {
            var clients = new List <ImageDownloaderBase>(2)
            {
                // new GathererClient(),
                new ScryfallClient(),
            };

            var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
            var repo     = new CardRepository(new CardFormatter(), () => null)
            {
                FilterSetCode = setCode => setCodes?.Contains(setCode) != false,
            };

            repo.LoadFile();
            repo.Load();

            foreach (Set set in repo.SetsByCode.Values)
            {
                if (setCodes?.Contains(set.Code) == false)
                {
                    continue;
                }

                foreach ((bool isToken, FsPath typeSubdir, _) in getIsToken(nonToken, token))
                {
                    var cards = set.List(isToken);
                    if (cards.Count == 0)
                    {
                        continue;
                    }

                    var missingCardsByClient = clients.ToDictionary(_ => _, _ => 0);

                    FsPath setSubdir        = new FsPath(set.Code + (Str.Equals(set.Code, "con") ? " escape" : string.Empty));
                    FsPath downloadRootDir  = DevPaths.MtgContentDir.Join(_createZoom ? OriginalSubdir : PreProcessedSubdir);
                    FsPath rootDirZoom      = DevPaths.MtgContentDir.Join(PreProcessedSubdir);
                    FsPath setDirectory     = downloadRootDir.Join(typeSubdir, setSubdir);
                    FsPath setDirectoryZoom = rootDirZoom.Join(typeSubdir, setSubdir);
                    FsPath setDirectoryPng  = setDirectory.Concat(".png");

                    bool dirExisted = setDirectoryPng.IsDirectory();
                    if (!dirExisted)
                    {
                        setDirectoryPng.CreateDirectory();
                    }

                    foreach (var card in cards)
                    {
                        FsPath targetFile        = setDirectoryPng.Join(card.ImageName + ".png");
                        FsPath processedFile     = setDirectory.Join(card.ImageName + ".jpg");
                        FsPath processedFileZoom = setDirectoryZoom.Join(card.ImageName + ".jpg");
                        if (targetFile.IsFile() || processedFile.IsFile() && processedFileZoom.IsFile())
                        {
                            continue;
                        }

                        if (targetFile.Basename(extension: false).EndsWith("1"))
                        {
                            var unnumbered = targetFile.WithName(_ => _.Replace("1.png", ".png"));
                            if (unnumbered.IsFile())
                            {
                                unnumbered.MoveFileTo(targetFile);
                                continue;
                            }
                        }

                        foreach (ImageDownloaderBase client in clients)
                        {
                            int attempts = 5;

                            for (int i = 0; i < attempts; i++)
                            {
                                var cancellation = new CancellationTokenSource();
                                var time         = DateTime.UtcNow;

                                var downloadTask = client.DownloadCardImage(card, targetFile, cancellation.Token);
                                var waitTask     = Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token);

                                await Task.WhenAny(downloadTask, waitTask);

                                cancellation.Cancel();

                                var elapsed = DateTime.UtcNow - time;
                                var delta   = TimeSpan.FromSeconds(0.5) - elapsed;
                                if (delta.TotalSeconds > 0)
                                {
                                    await Task.Delay(delta);
                                }

                                if (targetFile.IsFile())
                                {
                                    break;
                                }
                            }

                            if (targetFile.IsFile())
                            {
                                break;
                            }

                            missingCardsByClient[client]++;
                        }
                    }

                    if (!dirExisted && !setDirectoryPng.EnumerateFiles().Any())
                    {
                        setDirectoryPng.DeleteDirectory();
                    }
                }
            }
        }
Beispiel #9
0
        public void PreProcessImages(string setCodesStr, bool nonToken, bool token)
        {
            setupImageConversion();

            FsPath smallDir    = DevPaths.GathererOriginalDir;
            FsPath zoomDir     = DevPaths.GathererPreprocessedDir;
            FsPath smallDirBak = BakDir.Join(smallDir.Basename());
            FsPath zoomDirBak  = BakDir.Join(zoomDir.Basename());

            var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);

            IEnumerable <FsPath> getSetSubdirs(FsPath typeSubdir)
            {
                FsPath typeDir = smallDir.Join(typeSubdir);

                return(typeDir
                       .EnumerateDirectories(_fromPng ? "*.png" : "*", SearchOption.TopDirectoryOnly)
                       .Select(_ => new FsPath(
                                   Regex.Replace(
                                       _.Basename(),
                                       @"\.png$",
                                       string.Empty))));
            }

            foreach ((_, FsPath typeSubdir, _) in getIsToken(nonToken, token))
            {
                foreach (FsPath setSubdir in getSetSubdirs(typeSubdir))
                {
                    if (setCodes?.Contains(setSubdir.Value) == false)
                    {
                        continue;
                    }

                    FsPath smallJpgDir = smallDir.Join(typeSubdir, setSubdir);
                    FsPath zoomJpgDir  = zoomDir.Join(typeSubdir, setSubdir);

                    if (!_fromPng)
                    {
                        if (!_createZoom)
                        {
                            return;
                        }

                        zoomJpgDir.CreateDirectory();
                        foreach (FsPath smallImg in smallJpgDir.EnumerateFiles())
                        {
                            FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
                            if (_keepExisting && zoomImg.IsFile() && isZoomed(zoomImg))
                            {
                                continue;
                            }

                            scale(smallImg, zoomImg);
                        }
                    }
                    else
                    {
                        FsPath setPngSubdir   = setSubdir.Concat(".png");
                        FsPath smallPngDir    = smallDir.Join(typeSubdir, setPngSubdir);
                        FsPath smallPngDirBak = smallDirBak.Join(typeSubdir, setPngSubdir);
                        FsPath zoomPngDir     = zoomDir.Join(typeSubdir, setPngSubdir);
                        FsPath zoomPngDirBak  = zoomDirBak.Join(typeSubdir, setPngSubdir);

                        var dirs = new List <(FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom)>();
                        if (_createZoom)
                        {
                            zoomPngDir.CreateDirectory();
                            foreach (FsPath smallImg in smallPngDir.EnumerateFiles())
                            {
                                FsPath zoomImg      = smallImg.ChangeDirectory(smallDir, zoomDir);
                                FsPath convertedImg = zoomImg.ChangeDirectory(zoomPngDir, zoomJpgDir)
                                                      .WithName(_ => _.Replace(".png", ".jpg"));

                                if (_keepExisting && (
                                        zoomImg.IsFile() && isZoomed(zoomImg) ||
                                        convertedImg.IsFile() && isZoomed(convertedImg)))
                                {
                                    continue;
                                }

                                scale(smallImg, zoomImg);
                            }

                            dirs.Add((pngDir: zoomPngDir, pngDirBak: zoomPngDirBak, jpgDir: zoomJpgDir, isZoom: true));
                        }

                        dirs.Add((pngDir: smallPngDir, pngDirBak: smallPngDirBak, jpgDir: smallJpgDir, isZoom: false));

                        foreach ((FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom) in dirs)
                        {
                            jpgDir.CreateDirectory();

                            var pngImages = pngDir.EnumerateFiles();
                            foreach (FsPath sourceImage in pngImages)
                            {
                                convertToJpg(sourceImage, jpgDir, isZoom);
                            }

                            moveDirectoryToBackup(pngDir, pngDirBak);
                        }
                    }
                }
            }
        }
Beispiel #10
0
 public void Setup()
 {
     _tempDirectoryPath = new FsPath(Path.GetTempPath()).Join(Path.GetRandomFileName());
     _tempDirectoryPath.CreateDirectory();
 }
Beispiel #11
0
        public async Task Download(string storageUrl, FsPath targetDirectory, string name, CancellationToken token, bool silent = false, int?timeoutSec = null)
        {
            if (_process != null)
            {
                throw new InvalidOperationException("Download is already running. Use another instance to start new download.");
            }

            _silent = silent;

            await _syncSelfDownload.WaitAsync(token);

            try
            {
                if (!MegadlExePath.IsFile())
                {
                    var webClient = new YandexDiskClientWrapper(new YandexDiskClient(), _yandexKey);
                    // alternatively download from https://yadi.sk/d/f1HuKUg7xW2FUQ/tools?w=1
                    await webClient.DownloadAndExtract(_megatoolsUrl, AppDir.Update, new FsPath("megatools.7z"), token);
                }
            }
            finally
            {
                _syncSelfDownload.Release();
            }

            DownloadedCount = 0;

            string arguments;

            targetDirectory.CreateDirectory();
            if (targetDirectory.Value.Contains(' '))
            {
                arguments = $@"--path=""{targetDirectory}"" --print-names {storageUrl}";
            }
            else
            {
                arguments = $@"--path={targetDirectory} --print-names {storageUrl}";
            }

            _process = new Process
            {
                StartInfo = new ProcessStartInfo(MegadlExePath.Value, arguments)
                {
                    RedirectStandardOutput = true,
                    RedirectStandardError  = true,
                    UseShellExecute        = false,
                    CreateNoWindow         = true
                },

                EnableRaisingEvents = true
            };

            _process.OutputDataReceived += downloadOutputReceived;
            _process.ErrorDataReceived  += downloadErrorReceived;

            if (!_silent)
            {
                Console.WriteLine("Downloading {0} from {1} to {2}", name, storageUrl, targetDirectory);
            }

            AppDomain.CurrentDomain.ProcessExit += processExit;
            _process.Start();
            _process.BeginOutputReadLine();
            _process.BeginErrorReadLine();

            if (timeoutSec.HasValue)
            {
                _process.WaitForExit(timeoutSec.Value * 1000);
            }
            else
            {
                _process.WaitForExit();
            }

            Abort();
        }
Beispiel #12
0
        private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
        {
            FsPath targetSubdirectory = progress.TargetSubdirectory;

            targetSubdirectory.CreateDirectory();

            if (progress.FilesOnline == null)
            {
                return(false);
            }

            bool alreadyDownloaded = true;

            var existingFiles = new HashSet <FsPath>(
                targetSubdirectory.EnumerateFiles("*", SearchOption.AllDirectories));

            var existingSignatures = new Dictionary <FsPath, FileSignature>();

            foreach (var fileOnline in progress.FilesOnline.Values)
            {
                FsPath filePath = targetSubdirectory.Join(fileOnline.Path);
                if (!existingFiles.Contains(filePath))
                {
                    alreadyDownloaded = false;
                    continue;
                }

                FileSignature tempQualifier     = Signer.CreateSignature(filePath, useAbsolutePath: true);
                var           existingSignature =
                    progress.FilesCorrupted.TryGet(fileOnline.Path) ??
                    progress.FilesDownloaded.TryGet(fileOnline.Path) ??
                    new FileSignature
                {
                    Path    = tempQualifier.Path.RelativeTo(targetSubdirectory).Intern(true),
                    Md5Hash = tempQualifier.Md5Hash
                };

                if (existingSignature.Md5Hash != fileOnline.Md5Hash)
                {
                    alreadyDownloaded = false;
                    Console.WriteLine("Deleting modified or corrupted file {0}", filePath);

                    lock (ImageLoader.SyncIo)
                    {
                        try
                        {
                            filePath.DeleteFile();
                        }
                        catch (IOException ex)
                        {
                            Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
                        }
                    }
                }
                else
                {
                    existingSignatures.Add(existingSignature.Path, existingSignature);
                }
            }

            foreach (FsPath file in existingFiles)
            {
                var relativePath = file.RelativeTo(targetSubdirectory);
                if (!progress.FilesOnline.ContainsKey(relativePath) && relativePath != Signer.SignaturesFile)
                {
                    Console.WriteLine("Deleting {0}", file);
                    file.DeleteFile();
                }
            }

            if (alreadyDownloaded)
            {
                ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
            }

            return(alreadyDownloaded);
        }