public void ZipImages(string setCodesStr, bool small, bool zoom, bool nonToken, bool token)
{
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (FsPath qualityDir in getQualities(small, zoom))
{
foreach ((_, _, FsPath tokenSuffix) in getIsToken(nonToken, token))
{
FsPath compressedRoot = TargetDir.Join(qualityDir).Concat(tokenSuffix).Concat(ZipDirSuffix);
compressedRoot.CreateDirectory();
FsPath sourceRoot = TargetDir.Join(qualityDir).Concat(tokenSuffix);
foreach (var subdir in sourceRoot.EnumerateDirectories())
{
string subdirRelative = subdir.Basename();
if (setCodes?.Contains(subdirRelative) == false)
{
continue;
}
var targetFile = compressedRoot.Join(subdirRelative).Concat(SevenZipExtension);
if (targetFile.IsFile())
{
targetFile.DeleteFile();
}
new SevenZip(false).Compress(subdir, targetFile)
.Should().BeTrue();
}
}
}
}
public void SignFiles(FsPath packagePath, FsPath output, string setCodes)
{
FsPath parentDir = output.Parent();
parentDir.CreateDirectory();
if (packagePath.IsDirectory())
{
var sets = setCodes?.Split(';', ',', '|')
// to remove duplicates with different set name casing
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var prevSignatureByPath = sets != null && output.IsFile()
? Signer.ReadFromFile(output, internPath: false)
.Where(_ => !sets.Contains(_.Path.Parent().Value))
.ToDictionary(_ => _.Path)
: new Dictionary <FsPath, FileSignature>();
var signatures = Signer.CreateSignatures(packagePath, precalculated: prevSignatureByPath);
Signer.WriteToFile(output, signatures);
}
else if (packagePath.IsFile())
{
var metadata = Signer.CreateSignature(packagePath);
Signer.WriteToFile(output, Sequence.Array(metadata));
}
else
{
Console.WriteLine("Specified path {0} does not exist", packagePath);
}
}
public void RenameWizardsWebpageImages(string htmlFile, string targetSubdir)
{
FsPath htmlPath = HtmlDir.Join(htmlFile);
FsPath targetDir = DevPaths.GathererOriginalDir.Join(targetSubdir);
string htmlFileName = htmlPath.Basename(extension: false);
FsPath directoryName = htmlPath.Parent();
if (!directoryName.HasValue())
{
throw new ArgumentException(htmlPath.Value, nameof(htmlPath));
}
FsPath filesDirectory = directoryName.Join(htmlFileName + "_files");
string content = htmlPath.ReadAllText();
var matches = _imgTagPattern.Matches(content);
targetDir.CreateDirectory();
foreach (Match match in matches)
{
string originalFileName = match.Groups["file"].Value;
string ext = Path.GetExtension(originalFileName);
FsPath filePath = filesDirectory.Join(originalFileName);
string name = HttpUtility.HtmlDecode(match.Groups["name"].Value)
.Replace(" // ", "");
FsPath defaultTargetPath = targetDir.Join(name + ext);
bool defaultTargetExists = defaultTargetPath.IsFile();
if (defaultTargetExists || getTargetPath(1).IsFile())
{
if (defaultTargetExists)
{
defaultTargetPath.MoveFileTo(getTargetPath(1));
}
for (int i = 2; i < 12; i++)
{
FsPath targetPath = getTargetPath(i);
if (!targetPath.IsFile())
{
filePath.CopyFileTo(targetPath, overwrite: false);
break;
}
}
}
else
{
filePath.CopyFileTo(defaultTargetPath, overwrite: false);
}
FsPath getTargetPath(int num) =>
targetDir.Join(name + num + ext);
}
}
private static FsPath ensureSetSubdirectory(FsPath smallSet)
{
Console.WriteLine($" Creating {smallSet} ...");
try
{
smallSet.CreateDirectory();
}
catch (DirectoryNotFoundException)
{
// CON is banned as file / folder name in Windows
smallSet = smallSet.Concat(" escape");
smallSet.CreateDirectory();
}
return(smallSet);
}
public void Save(FsPath file)
{
FsPath directory = file.Parent();
directory.CreateDirectory();
var state = getState();
WriteHistory(file, state);
}
public void MigrateHistoryFiles()
{
FsPath firstFormDirectory = AppDir.History.Join(0.ToString());
if (firstFormDirectory.IsDirectory())
{
return;
}
firstFormDirectory.CreateDirectory();
foreach (FsPath file in AppDir.History.EnumerateFiles())
{
file.CopyFileTo(firstFormDirectory.Join(file.Basename()));
}
}
public void LoadHistory(FsPath file)
{
FsPath directory = file.Parent();
directory.CreateDirectory();
if (TryReadHistory(file, out var state))
{
_settingsHistory = state.SettingsHistory;
_settingsIndex = state.SettingsIndex;
}
else
{
_settingsHistory = new List <GuiSettings>();
var defaultSettings = new GuiSettings();
Add(defaultSettings);
}
IsLoaded = true;
Loaded?.Invoke();
}
public async Task DownloadGathererImages(string setCodesStr, bool nonToken, bool token)
{
var clients = new List <ImageDownloaderBase>(2)
{
// new GathererClient(),
new ScryfallClient(),
};
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
var repo = new CardRepository(new CardFormatter(), () => null)
{
FilterSetCode = setCode => setCodes?.Contains(setCode) != false,
};
repo.LoadFile();
repo.Load();
foreach (Set set in repo.SetsByCode.Values)
{
if (setCodes?.Contains(set.Code) == false)
{
continue;
}
foreach ((bool isToken, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
var cards = set.List(isToken);
if (cards.Count == 0)
{
continue;
}
var missingCardsByClient = clients.ToDictionary(_ => _, _ => 0);
FsPath setSubdir = new FsPath(set.Code + (Str.Equals(set.Code, "con") ? " escape" : string.Empty));
FsPath downloadRootDir = DevPaths.MtgContentDir.Join(_createZoom ? OriginalSubdir : PreProcessedSubdir);
FsPath rootDirZoom = DevPaths.MtgContentDir.Join(PreProcessedSubdir);
FsPath setDirectory = downloadRootDir.Join(typeSubdir, setSubdir);
FsPath setDirectoryZoom = rootDirZoom.Join(typeSubdir, setSubdir);
FsPath setDirectoryPng = setDirectory.Concat(".png");
bool dirExisted = setDirectoryPng.IsDirectory();
if (!dirExisted)
{
setDirectoryPng.CreateDirectory();
}
foreach (var card in cards)
{
FsPath targetFile = setDirectoryPng.Join(card.ImageName + ".png");
FsPath processedFile = setDirectory.Join(card.ImageName + ".jpg");
FsPath processedFileZoom = setDirectoryZoom.Join(card.ImageName + ".jpg");
if (targetFile.IsFile() || processedFile.IsFile() && processedFileZoom.IsFile())
{
continue;
}
if (targetFile.Basename(extension: false).EndsWith("1"))
{
var unnumbered = targetFile.WithName(_ => _.Replace("1.png", ".png"));
if (unnumbered.IsFile())
{
unnumbered.MoveFileTo(targetFile);
continue;
}
}
foreach (ImageDownloaderBase client in clients)
{
int attempts = 5;
for (int i = 0; i < attempts; i++)
{
var cancellation = new CancellationTokenSource();
var time = DateTime.UtcNow;
var downloadTask = client.DownloadCardImage(card, targetFile, cancellation.Token);
var waitTask = Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token);
await Task.WhenAny(downloadTask, waitTask);
cancellation.Cancel();
var elapsed = DateTime.UtcNow - time;
var delta = TimeSpan.FromSeconds(0.5) - elapsed;
if (delta.TotalSeconds > 0)
{
await Task.Delay(delta);
}
if (targetFile.IsFile())
{
break;
}
}
if (targetFile.IsFile())
{
break;
}
missingCardsByClient[client]++;
}
}
if (!dirExisted && !setDirectoryPng.EnumerateFiles().Any())
{
setDirectoryPng.DeleteDirectory();
}
}
}
}
public void PreProcessImages(string setCodesStr, bool nonToken, bool token)
{
setupImageConversion();
FsPath smallDir = DevPaths.GathererOriginalDir;
FsPath zoomDir = DevPaths.GathererPreprocessedDir;
FsPath smallDirBak = BakDir.Join(smallDir.Basename());
FsPath zoomDirBak = BakDir.Join(zoomDir.Basename());
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
IEnumerable <FsPath> getSetSubdirs(FsPath typeSubdir)
{
FsPath typeDir = smallDir.Join(typeSubdir);
return(typeDir
.EnumerateDirectories(_fromPng ? "*.png" : "*", SearchOption.TopDirectoryOnly)
.Select(_ => new FsPath(
Regex.Replace(
_.Basename(),
@"\.png$",
string.Empty))));
}
foreach ((_, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
foreach (FsPath setSubdir in getSetSubdirs(typeSubdir))
{
if (setCodes?.Contains(setSubdir.Value) == false)
{
continue;
}
FsPath smallJpgDir = smallDir.Join(typeSubdir, setSubdir);
FsPath zoomJpgDir = zoomDir.Join(typeSubdir, setSubdir);
if (!_fromPng)
{
if (!_createZoom)
{
return;
}
zoomJpgDir.CreateDirectory();
foreach (FsPath smallImg in smallJpgDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
if (_keepExisting && zoomImg.IsFile() && isZoomed(zoomImg))
{
continue;
}
scale(smallImg, zoomImg);
}
}
else
{
FsPath setPngSubdir = setSubdir.Concat(".png");
FsPath smallPngDir = smallDir.Join(typeSubdir, setPngSubdir);
FsPath smallPngDirBak = smallDirBak.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDir = zoomDir.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDirBak = zoomDirBak.Join(typeSubdir, setPngSubdir);
var dirs = new List <(FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom)>();
if (_createZoom)
{
zoomPngDir.CreateDirectory();
foreach (FsPath smallImg in smallPngDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
FsPath convertedImg = zoomImg.ChangeDirectory(zoomPngDir, zoomJpgDir)
.WithName(_ => _.Replace(".png", ".jpg"));
if (_keepExisting && (
zoomImg.IsFile() && isZoomed(zoomImg) ||
convertedImg.IsFile() && isZoomed(convertedImg)))
{
continue;
}
scale(smallImg, zoomImg);
}
dirs.Add((pngDir: zoomPngDir, pngDirBak: zoomPngDirBak, jpgDir: zoomJpgDir, isZoom: true));
}
dirs.Add((pngDir: smallPngDir, pngDirBak: smallPngDirBak, jpgDir: smallJpgDir, isZoom: false));
foreach ((FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom) in dirs)
{
jpgDir.CreateDirectory();
var pngImages = pngDir.EnumerateFiles();
foreach (FsPath sourceImage in pngImages)
{
convertToJpg(sourceImage, jpgDir, isZoom);
}
moveDirectoryToBackup(pngDir, pngDirBak);
}
}
}
}
}
public void Setup()
{
_tempDirectoryPath = new FsPath(Path.GetTempPath()).Join(Path.GetRandomFileName());
_tempDirectoryPath.CreateDirectory();
}
public async Task Download(string storageUrl, FsPath targetDirectory, string name, CancellationToken token, bool silent = false, int?timeoutSec = null)
{
if (_process != null)
{
throw new InvalidOperationException("Download is already running. Use another instance to start new download.");
}
_silent = silent;
await _syncSelfDownload.WaitAsync(token);
try
{
if (!MegadlExePath.IsFile())
{
var webClient = new YandexDiskClientWrapper(new YandexDiskClient(), _yandexKey);
// alternatively download from https://yadi.sk/d/f1HuKUg7xW2FUQ/tools?w=1
await webClient.DownloadAndExtract(_megatoolsUrl, AppDir.Update, new FsPath("megatools.7z"), token);
}
}
finally
{
_syncSelfDownload.Release();
}
DownloadedCount = 0;
string arguments;
targetDirectory.CreateDirectory();
if (targetDirectory.Value.Contains(' '))
{
arguments = $@"--path=""{targetDirectory}"" --print-names {storageUrl}";
}
else
{
arguments = $@"--path={targetDirectory} --print-names {storageUrl}";
}
_process = new Process
{
StartInfo = new ProcessStartInfo(MegadlExePath.Value, arguments)
{
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
},
EnableRaisingEvents = true
};
_process.OutputDataReceived += downloadOutputReceived;
_process.ErrorDataReceived += downloadErrorReceived;
if (!_silent)
{
Console.WriteLine("Downloading {0} from {1} to {2}", name, storageUrl, targetDirectory);
}
AppDomain.CurrentDomain.ProcessExit += processExit;
_process.Start();
_process.BeginOutputReadLine();
_process.BeginErrorReadLine();
if (timeoutSec.HasValue)
{
_process.WaitForExit(timeoutSec.Value * 1000);
}
else
{
_process.WaitForExit();
}
Abort();
}
private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
{
FsPath targetSubdirectory = progress.TargetSubdirectory;
targetSubdirectory.CreateDirectory();
if (progress.FilesOnline == null)
{
return(false);
}
bool alreadyDownloaded = true;
var existingFiles = new HashSet <FsPath>(
targetSubdirectory.EnumerateFiles("*", SearchOption.AllDirectories));
var existingSignatures = new Dictionary <FsPath, FileSignature>();
foreach (var fileOnline in progress.FilesOnline.Values)
{
FsPath filePath = targetSubdirectory.Join(fileOnline.Path);
if (!existingFiles.Contains(filePath))
{
alreadyDownloaded = false;
continue;
}
FileSignature tempQualifier = Signer.CreateSignature(filePath, useAbsolutePath: true);
var existingSignature =
progress.FilesCorrupted.TryGet(fileOnline.Path) ??
progress.FilesDownloaded.TryGet(fileOnline.Path) ??
new FileSignature
{
Path = tempQualifier.Path.RelativeTo(targetSubdirectory).Intern(true),
Md5Hash = tempQualifier.Md5Hash
};
if (existingSignature.Md5Hash != fileOnline.Md5Hash)
{
alreadyDownloaded = false;
Console.WriteLine("Deleting modified or corrupted file {0}", filePath);
lock (ImageLoader.SyncIo)
{
try
{
filePath.DeleteFile();
}
catch (IOException ex)
{
Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
}
}
}
else
{
existingSignatures.Add(existingSignature.Path, existingSignature);
}
}
foreach (FsPath file in existingFiles)
{
var relativePath = file.RelativeTo(targetSubdirectory);
if (!progress.FilesOnline.ContainsKey(relativePath) && relativePath != Signer.SignaturesFile)
{
Console.WriteLine("Deleting {0}", file);
file.DeleteFile();
}
}
if (alreadyDownloaded)
{
ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
}
return(alreadyDownloaded);
}