public async Task <bool> DownloadAndExtract(string remotePath, FsPath targetDirectory, FsPath fileName, CancellationToken token)
{
if (!Str.Equals(".7z", fileName.Extension()))
{
throw new ArgumentException();
}
FsPath archiveFileName = targetDirectory.Join(fileName);
if (archiveFileName.IsFile())
{
try
{
archiveFileName.DeleteFile();
}
catch (Exception ex)
{
lock (_syncOutput)
Console.WriteLine($"Failed to remove {archiveFileName}: {ex.Message}");
return(false);
}
}
bool downloaded = await TryDownloadFile(remotePath, archiveFileName, token);
if (!downloaded)
{
return(false);
}
if (!archiveFileName.IsFile())
{
lock (_syncOutput)
Console.WriteLine($"Failed to download {archiveFileName} from {remotePath}");
return(false);
}
var sevenZip = new SevenZip(silent: true);
sevenZip.Extract(archiveFileName, targetDirectory, Enumerable.Empty <FsPath>());
try
{
archiveFileName.DeleteFile();
}
catch (Exception ex)
{
lock (_syncOutput)
Console.WriteLine($"Failed to remove {archiveFileName}: {ex.Message}");
}
return(true);
}
/// <summary>
/// Sets or removes read-only attribute on files.
/// </summary>
/// <param name="set">True to set the read-only attribute. False - to remove the read-only attribute.</param>
public async Task <bool> SetLockedByAnotherUserAsync(bool set)
{
bool resultSet = false;
// Changing read-only attribute on folders triggers folders listing. Changing it on files only.
if (FsPath.IsFile(userFileSystemPath))
{
FileInfo file = new FileInfo(userFileSystemPath);
if (set != file.IsReadOnly)
{
// Set/Remove read-only attribute.
if (set)
{
new FileInfo(userFileSystemPath).Attributes |= System.IO.FileAttributes.ReadOnly;
}
else
{
new FileInfo(userFileSystemPath).Attributes &= ~System.IO.FileAttributes.ReadOnly;
}
resultSet = true;
}
}
return(resultSet);
}
public void DisplayNews()
{
if (_unreadNews == null)
{
return;
}
foreach (var file in _unreadNews)
{
FsPath readAnnounceFile = getReadNewsFile(file);
string text = file.ReadAllText().Trim();
bool isLocked = file.Value.IndexOf("[locked]", Str.Comparison) >= 0;
if (!isLocked && !readAnnounceFile.IsFile())
{
file.MoveFileTo(readAnnounceFile);
}
if (string.IsNullOrEmpty(text))
{
continue;
}
Console.WriteLine();
Console.WriteLine(text);
Console.WriteLine();
}
_unreadNews.Clear();
NewsDisplayed?.Invoke();
}
public bool CreateShortcut(FsPath exePath, FsPath iconPath, FsPath shortcutPath)
{
bool useBackup = false;
FsPath backupPath = FsPath.None;
if (shortcutPath.IsFile())
{
backupPath = shortcutPath.WithName(_ => _ + ".bak");
useBackup = !backupPath.IsFile();
if (useBackup)
{
shortcutPath.MoveFileTo(backupPath);
}
else
{
shortcutPath.DeleteFile();
}
}
bool success = run($"\"{_script}\" \"{shortcutPath}\" \"{exePath}\" \"{iconPath}\"");
if (!success && useBackup)
{
backupPath.MoveFileTo(shortcutPath);
}
return(success);
}
public void SignFiles(FsPath packagePath, FsPath output, string setCodes)
{
FsPath parentDir = output.Parent();
parentDir.CreateDirectory();
if (packagePath.IsDirectory())
{
var sets = setCodes?.Split(';', ',', '|')
// to remove duplicates with different set name casing
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var prevSignatureByPath = sets != null && output.IsFile()
? Signer.ReadFromFile(output, internPath: false)
.Where(_ => !sets.Contains(_.Path.Parent().Value))
.ToDictionary(_ => _.Path)
: new Dictionary <FsPath, FileSignature>();
var signatures = Signer.CreateSignatures(packagePath, precalculated: prevSignatureByPath);
Signer.WriteToFile(output, signatures);
}
else if (packagePath.IsFile())
{
var metadata = Signer.CreateSignature(packagePath);
Signer.WriteToFile(output, Sequence.Array(metadata));
}
else
{
Console.WriteLine("Specified path {0} does not exist", packagePath);
}
}
public override async Task DownloadCardImage(Card card, FsPath targetPath, CancellationToken token)
{
if (string.IsNullOrEmpty(card.Identifiers.ScryfallId))
{
_log.Info("Emtpy scryfall id: {0}", card);
return;
}
string format = Jpg ? "normal" : "png";
if (card.MultiverseId.HasValue)
{
string urlMultiverseId = "https://api.scryfall.com/cards/multiverse/" + card.MultiverseId.Value +
"?format=image&version=" + format;
await downloadImage(urlMultiverseId, targetPath, token);
if (targetPath.IsFile())
{
return;
}
}
string urlScryfallId = "https://api.scryfall.com/cards/" +
card.Identifiers.ScryfallId + "?format=image&version=" + format;
if (Str.Equals(card.Side, CardSides.B))
{
urlScryfallId += "&face=back";
}
await downloadImage(urlScryfallId, targetPath, token);
}
public void Invalidate()
{
if (_completionLabelFile.IsFile())
{
_completionLabelFile.DeleteFile();
}
}
/// <summary>
/// Sets or removes "Locked another another user" icon and read-only attribute on files.
/// </summary>
/// <param name="set">True to display the icon and read-only attribute. False - to remove the icon and read-only attribute.</param>
internal async Task SetLockedByAnotherUserAsync(bool set)
{
// Can not set icon on read-only files.
// Changing read-only attribute on folders triggers folders listing. Changing it on files only.
if (FsPath.IsFile(userFileSystemPath))
{
FileInfo file = new FileInfo(userFileSystemPath);
if (set != file.IsReadOnly)
{
// Remove read-only attribute.
new FileInfo(userFileSystemPath).Attributes &= ~System.IO.FileAttributes.ReadOnly;
// Update the lock icon, to indicate that the item is locked by another user.
await SetLockedByAnotherUserIconAsync(set);
// Set read-only attribute.
if (set)
{
new FileInfo(userFileSystemPath).Attributes |= System.IO.FileAttributes.ReadOnly;
}
}
}
else
{
// Update the lock icon, to indicate that the item is locked by another user.
await SetLockedByAnotherUserIconAsync(set);
}
}
private void openInExplorerClick(object sender, EventArgs e)
{
FsPath fullPath = _models[_imageIndex].ImageFile.FullPath;
if (!fullPath.IsFile())
{
return;
}
if (Runtime.IsLinux)
{
var explorerApp = detectFileExplorerApp();
if (!string.IsNullOrEmpty(explorerApp))
{
Process.Start(explorerApp, $"--select \"{fullPath}\"");
}
else
{
Process.Start("xdg-open", $"\"{fullPath.Parent()}\"");
}
}
else
{
Process.Start("explorer.exe", $"/select, \"{fullPath}\"");
}
}
private static void ensureFileDeleted(FsPath file)
{
if (file.IsFile())
{
file.DeleteFile();
}
}
/// <summary>
/// Deletes a file or folder placeholder in user file system.
/// </summary>
/// <remarks>This method failes if the file or folder in user file system is modified (not in sync with the remote storage).</remarks>
public async Task DeleteAsync()
{
// Windows does not provide a function to delete a placeholder file only if it is not modified.
// Here we check that the file is not modified in user file system, using GetInSync() call.
// To avoid the file modification between GetInSync() call and Delete() call in this method we
// open it without FileShare.Write flag.
using (WindowsFileSystemItem userFileSystemWinItem = WindowsFileSystemItem.Open(userFileSystemPath, (FileAccess)0, FileMode.Open, FileShare.Read | FileShare.Delete))
{
if (PlaceholderItem.GetInSync(userFileSystemWinItem.SafeHandle))
{
if (FsPath.IsFile(userFileSystemPath))
{
File.Delete(userFileSystemPath);
}
else
{
Directory.Delete(userFileSystemPath, true);
}
}
else
{
throw new IOException("File not in-sync.");
}
}
}
public void SignImages(string setCodesStr, bool small, bool zoom, bool nonToken, bool token)
{
foreach (FsPath qualityDir in getQualities(small, zoom))
{
foreach ((_, _, FsPath tokenSuffix) in getIsToken(nonToken, token))
{
FsPath outputFile = getSignatureFile(qualityDir, tokenSuffix);
FsPath packagePath = TargetDir.Join(qualityDir).Concat(tokenSuffix);
new ImageDirectorySigner().SignFiles(packagePath, outputFile, setCodesStr);
FsPath signatureFile = getSignatureFile(qualityDir, tokenSuffix);
FsPath compressedSignatureFile = signatureFile
.Parent()
.Join(signatureFile.Basename(extension: false))
.Concat(SevenZipExtension);
if (compressedSignatureFile.IsFile())
{
compressedSignatureFile.DeleteFile();
}
new SevenZip(false).Compress(signatureFile, compressedSignatureFile)
.Should().BeTrue();
}
}
}
public void RenameWizardsWebpageImages(string htmlFile, string targetSubdir)
{
FsPath htmlPath = HtmlDir.Join(htmlFile);
FsPath targetDir = DevPaths.GathererOriginalDir.Join(targetSubdir);
string htmlFileName = htmlPath.Basename(extension: false);
FsPath directoryName = htmlPath.Parent();
if (!directoryName.HasValue())
{
throw new ArgumentException(htmlPath.Value, nameof(htmlPath));
}
FsPath filesDirectory = directoryName.Join(htmlFileName + "_files");
string content = htmlPath.ReadAllText();
var matches = _imgTagPattern.Matches(content);
targetDir.CreateDirectory();
foreach (Match match in matches)
{
string originalFileName = match.Groups["file"].Value;
string ext = Path.GetExtension(originalFileName);
FsPath filePath = filesDirectory.Join(originalFileName);
string name = HttpUtility.HtmlDecode(match.Groups["name"].Value)
.Replace(" // ", "");
FsPath defaultTargetPath = targetDir.Join(name + ext);
bool defaultTargetExists = defaultTargetPath.IsFile();
if (defaultTargetExists || getTargetPath(1).IsFile())
{
if (defaultTargetExists)
{
defaultTargetPath.MoveFileTo(getTargetPath(1));
}
for (int i = 2; i < 12; i++)
{
FsPath targetPath = getTargetPath(i);
if (!targetPath.IsFile())
{
filePath.CopyFileTo(targetPath, overwrite: false);
break;
}
}
}
else
{
filePath.CopyFileTo(defaultTargetPath, overwrite: false);
}
FsPath getTargetPath(int num) =>
targetDir.Join(name + num + ext);
}
}
private void updateApplicationShortcut(FsPath shortcutLocation)
{
FsPath shortcutPath = shortcutLocation.Join(ShortcutFileName);
if (shortcutPath.IsFile())
{
CreateApplicationShortcut(shortcutLocation);
}
}
public void CreateApplicationShortcut(FsPath shortcutLocation)
{
string appVersionInstalled = GetAppVersionInstalled();
// Mtgdb.Gui.v1.3.5.10.zip
var prefix = "Mtgdb.Gui.";
var postfix = ".zip";
var versionDir = appVersionInstalled.Substring(prefix.Length, appVersionInstalled.Length - prefix.Length - postfix.Length);
FsPath currentBin = AppDir.BinVersion.Parent().Join(versionDir);
// may be different from currently running executable because of just installed upgrade
FsPath execPath = currentBin.Join(ExecutableFileName);
FsPath iconPath = currentBin.Join("mtg64.ico");
FsPath shortcutPath = shortcutLocation.Join(ShortcutFileName);
if (createApplicationShortcut(shortcutPath, execPath, iconPath))
{
return;
}
// workaround a problem with WshShell unable to create the link within desktop directory
// due to a mismatch between physical and localized directory names
var tempLocation = new FsPath(Path.GetTempPath());
FsPath tempPath = tempLocation.Join(ShortcutFileName);
if (createApplicationShortcut(tempPath, execPath, iconPath))
{
try
{
if (shortcutPath.IsFile())
{
shortcutPath.DeleteFile();
}
tempPath.MoveFileTo(shortcutPath);
Console.WriteLine("Moved application shortcut from {0} to {1}",
tempLocation, shortcutLocation);
}
catch (Exception ex)
{
Console.WriteLine("Failed to move application shortcut from {0} to {1}: {2}",
tempLocation, shortcutLocation, ex);
try
{
tempPath.DeleteFile();
}
catch (Exception cleanupEx)
{
Console.WriteLine("Failed to remove application shortcut from {0}: {1}",
tempLocation, cleanupEx);
}
}
}
}
public async Task UpdateAsync(string userFileSystemPath)
{
if (FsPath.IsFile(userFileSystemPath))
{
await CreateOrUpdateFileAsync(userFileSystemPath, false);
}
else
{
await CreateOrUpdateFolderAsync(userFileSystemPath, false);
}
}
public static async Task CreateAsync(string remoteStoragePath, string userFileSystemPath)
{
if (FsPath.IsFile(userFileSystemPath))
{
await new RemoteStorageItem(remoteStoragePath).CreateOrUpdateFileAsync(userFileSystemPath, true);
}
else
{
await new RemoteStorageItem(remoteStoragePath).CreateOrUpdateFolderAsync(userFileSystemPath, true);
}
}
public static void CreateApplicationShortcut(FsPath exePath, FsPath iconPath, FsPath shortcutPath)
{
if (shortcutPath.IsFile())
{
shortcutPath.DeleteFile();
}
var wsh = new WshShell();
IWshShortcut shortcut;
try
{
shortcut = wsh.CreateShortcut(shortcutPath.Value) as IWshShortcut;
}
catch (Exception ex)
{
Console.Error.WriteLine(
"Failed to create shortcut object {0} at {1}: {2}",
exePath, shortcutPath, ex);
return;
}
FsPath bin = exePath.Parent();
if (shortcut == null)
{
Console.Error.WriteLine("Failed to create shortcut {0} at {1}: {2}.{3} returned null",
exePath, shortcutPath, nameof(WshShell), nameof(wsh.CreateShortcut));
return;
}
shortcut.Arguments = "";
shortcut.TargetPath = exePath.Value;
shortcut.WindowStyle = 1;
shortcut.Description = "Application to search MTG cards and build decks";
shortcut.WorkingDirectory = bin.Value;
if (iconPath.HasValue())
{
shortcut.IconLocation = iconPath.Value;
}
try
{
shortcut.Save();
}
catch (Exception ex)
{
Console.Error.WriteLine("Failed to create shortcut {0} at {1}: {2}", exePath, shortcutPath, ex);
}
}
private void convertToJpg(FsPath sourceImage, FsPath targetDir, bool isZoomDir)
{
FsPath targetImage = targetDir.Join(sourceImage.Basename(extension: false)).Concat(".jpg");
if (_keepExisting && targetImage.IsFile() && (isZoomed(targetImage) || !isZoomDir))
{
return;
}
using var original = new Bitmap(sourceImage.Value);
new BmpAlphaToBackgroundColorTransformation(original, Color.White)
.Execute();
original.Save(targetImage.Value, _jpegCodec, _jpegEncoderParams);
}
/// <summary>
/// Returns true if hydration is required. False - otherwise.
/// </summary>
internal bool HydrationRequired()
{
if (FsPath.IsFile(userFileSystemPath))
{
int attributes = (int)new FileInfo(userFileSystemPath).Attributes;
// Download content (hydrate) if file is pinned and no file content is present locally (offline).
if (((attributes & (int)FileAttributesExt.Pinned) != 0) &&
((attributes & (int)FileAttributesExt.Offline) != 0))
{
return(true);
}
}
return(false);
}
/// <summary>
/// Returns true if dehydration is required. False - otherwise.
/// </summary>
internal bool DehydrationRequired()
{
if (FsPath.IsFile(userFileSystemPath))
{
int attributes = (int)new FileInfo(userFileSystemPath).Attributes;
// Remove content (dehydrate) if file is unpinned and file content is present locally (not offline).
if (((attributes & (int)FileAttributesExt.Unpinned) != 0) &&
((attributes & (int)FileAttributesExt.Offline) == 0))
{
return(true);
}
}
return(false);
}
static string detect()
{
var queryProcess = Process.Start(
new ProcessStartInfo("xdg-mime", "query default inode/directory")
{
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
});
if (queryProcess == null)
{
return(string.Empty);
}
queryProcess.WaitForExit();
string output = queryProcess.StandardOutput.ReadToEnd();
var appDesktopName = output.TrimEnd();
var appDesktopFile = new FsPath($"/usr/share/applications/{appDesktopName}");
if (!appDesktopFile.IsFile())
{
return(string.Empty);
}
string[] lines;
try
{
lines = appDesktopFile.ReadAllLines();
}
catch
{
return(string.Empty);
}
const string prefix = "Exec=";
string line = lines.FirstOrDefault(_ => _.StartsWith(prefix));
if (line == null)
{
return(string.Empty);
}
string command = line.Substring(prefix.Length);
string name = new Regex(@"( %\w)+$").Replace(command, "");
return(name);
}
//$>
/// <summary>
/// Deletes a file or folder placeholder in user file system.
/// </summary>
/// <remarks>
/// This method throws <see cref="ConflictException"/> if the file or folder or any file or folder
/// in the folder hierarchy being deleted in user file system is modified (not in sync with the remote storage).
/// </remarks>
/// <returns>True if the file was deleted. False - otherwise.</returns>
public async Task <bool> DeleteAsync()
{
// Cloud Filter API does not provide a function to delete a placeholder file only if it is not modified.
// Here we check that the file is not modified in user file system, using GetInSync() call.
// To avoid the file modification between GetInSync() call and Delete() call we
// open it without FileShare.Write flag.
try
{
// Because of the on-demand population the file or folder placeholder may not exist in the user file system.
if (FsPath.Exists(userFileSystemPath))
{
using (WindowsFileSystemItem userFileSystemWinItem = WindowsFileSystemItem.Open(userFileSystemPath, (FileAccess)0, FileMode.Open, FileShare.Read | FileShare.Delete))
{
if (PlaceholderItem.GetInSync(userFileSystemWinItem.SafeHandle))
{
if (FsPath.IsFile(userFileSystemPath))
{
File.Delete(userFileSystemPath);
}
else
{
Directory.Delete(userFileSystemPath, true);
}
// Delete ETag
ETag.DeleteETag(userFileSystemPath);
return(true);
}
else
{
throw new ConflictException(Modified.Client, "The item is not in-sync with the cloud.");
}
}
}
}
catch (Exception ex)
{
await SetDownloadErrorStateAsync(ex);
// Rethrow the exception preserving stack trace of the original exception.
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex).Throw();
}
return(false);
}
internal bool TryReadHistory(FsPath file, out HistoryState state)
{
state = null;
if (!file.IsFile())
{
return(false);
}
try
{
using (var fileReader = file.OpenText())
{
using var jsonReader = new JsonTextReader(fileReader);
state = _serializer.Deserialize <HistoryState>(jsonReader);
}
return(true);
}
catch (Exception ex)
{
_log.Error(ex, "Failed to read history");
return(false);
}
}
/// <summary>
/// Hydrates or dehydrates the file depending on pinned/unpinned attributes and offline attribute setting.
/// </summary>
/// <returns>
/// Returns true if hydration is competed, false if dehydration is completed
/// or null if no hydration/dehydrations was done.
/// </returns>
internal async Task <bool?> UpdateHydrationAsync()
{
if (FsPath.IsFile(userFileSystemPath))
{
int attributes = (int)new FileInfo(userFileSystemPath).Attributes;
// Download content (hydrate) if file is pinned and no file content is present locally (offline).
if (((attributes & (int)FileAttributesExt.Pinned) != 0) &&
((attributes & (int)FileAttributesExt.Offline) != 0))
{
new PlaceholderFile(userFileSystemPath).Hydrate(0, -1);
return(true);
}
// Remove content (dehydrate) if file is unpinned and file content is present locally (not offline).
else if (((attributes & (int)FileAttributesExt.Unpinned) != 0) &&
((attributes & (int)FileAttributesExt.Offline) == 0))
{
new PlaceholderFile(userFileSystemPath).Dehydrate(0, -1, false);
return(false);
}
}
return(null);
}
/// <summary>
/// Creates or updates the item in the remote storage.
/// </summary>
/// <param name="mode">
/// Indicates if the file should created or updated.
/// Supported modes are <see cref="FileMode.CreateNew"/> and <see cref="FileMode.Open"/>
/// </param>
/// <param name="lockInfo">Information about the lock. Pass null if the item is not locked.</param>
private async Task CreateOrUpdateAsync(FileMode mode, ServerLockInfo lockInfo = null)
{
if ((mode != FileMode.CreateNew) && (mode != FileMode.Open))
{
throw new ArgumentOutOfRangeException("mode", $"Must be {FileMode.CreateNew} or {FileMode.Open}");
}
FileSystemInfo userFileSystemItem = FsPath.GetFileSystemItem(userFileSystemPath);
using (WindowsFileSystemItem userFileSystemWinItem = WindowsFileSystemItem.OpenReadAttributes(userFileSystemPath, FileMode.Open, FileShare.Read))
//await using (FileStream userFileSystemStream = userFileSystemFile.Open(FileMode.Open, FileAccess.Read, FileShare.Read))
{
// Create the new file/folder in the remote storage only if the file/folder in the user file system was not moved.
// If the file is moved in user file system, move must first be syched to remote storage.
if ((mode == FileMode.CreateNew) && PlaceholderItem.GetItem(userFileSystemPath).IsMoved())
{
string originalPath = PlaceholderItem.GetItem(userFileSystemPath).GetOriginalPath();
throw new ConflictException(Modified.Client, $"The item was moved. Original path: {originalPath}");
}
// Ensures LastWriteTimeUtc is in sync with file content after Open() was called.
userFileSystemItem.Refresh();
IFileSystemItemBasicInfo info = GetBasicInfo(userFileSystemItem);
// Update remote storage file.
FileStream userFileSystemStream = null;
try
{
string eTag = null;
if (FsPath.IsFile(userFileSystemPath))
{
// File is marked as not in-sync when updated OR moved.
// Opening a file for reading triggers hydration, make sure to open only if content is modified.
if (PlaceholderFile.GetFileDataSizeInfo(userFileSystemWinItem.SafeHandle).ModifiedDataSize > 0)
{
//userFileSystemStream = new FileStream(userFileSystemWinItem.SafeHandle, FileAccess.Read);
userFileSystemStream = ((FileInfo)userFileSystemItem).Open(FileMode.Open, FileAccess.Read, FileShare.Read);
}
if (mode == FileMode.CreateNew)
{
string userFileSystemParentPath = Path.GetDirectoryName(userFileSystemPath);
IUserFolder userFolder = await virtualDrive.GetItemAsync <IUserFolder>(userFileSystemParentPath);
eTag = await userFolder.CreateFileAsync((IFileBasicInfo)info, userFileSystemStream);
}
else
{
IUserFile userFile = await virtualDrive.GetItemAsync <IUserFile>(userFileSystemPath);
eTag = await userFile.UpdateAsync((IFileBasicInfo)info, userFileSystemStream, lockInfo);
}
}
else
{
if (mode == FileMode.CreateNew)
{
string userFileSystemParentPath = Path.GetDirectoryName(userFileSystemPath);
IUserFolder userFolder = await virtualDrive.GetItemAsync <IUserFolder>(userFileSystemParentPath);
eTag = await userFolder.CreateFolderAsync((IFolderBasicInfo)info);
}
else
{
IUserFolder userFolder = await virtualDrive.GetItemAsync <IUserFolder>(userFileSystemPath);
eTag = await userFolder.UpdateAsync((IFolderBasicInfo)info, lockInfo);
}
}
await ETag.SetETagAsync(userFileSystemPath, eTag);
if (mode == FileMode.CreateNew)
{
PlaceholderItem.GetItem(userFileSystemPath).SetOriginalPath(userFileSystemPath);
}
}
finally
{
if (userFileSystemStream != null)
{
userFileSystemStream.Close();
}
}
PlaceholderItem.SetInSync(userFileSystemWinItem.SafeHandle, true);
}
}
private void export(
FsPath directory,
string setCodesStr,
ISet <FsPath> exportedSmall,
ISet <FsPath> exportedZoomed,
bool small,
bool zoomed,
FsPath smallSubdir,
FsPath zoomedSubdir,
bool matchingSet,
bool forceRemoveCorner,
bool token)
{
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach ((string setCode, Set set) in _cardRepo.SetsByCode)
{
Console.WriteLine(setCode);
if (setCodes?.Contains(setCode) == false)
{
continue;
}
FsPath smallSetSubdir = FsPath.None;
FsPath zoomedSetSubdir = FsPath.None;
if (small)
{
if (smallSubdir.HasValue())
{
smallSetSubdir = directory.Join(smallSubdir).Join(setCode);
}
else
{
smallSetSubdir = directory.Join(setCode);
}
smallSetSubdir = ensureSetSubdirectory(smallSetSubdir);
}
if (zoomed)
{
if (zoomedSubdir.HasValue())
{
zoomedSetSubdir = directory.Join(zoomedSubdir).Join(setCode);
}
else
{
zoomedSetSubdir = directory.Join(setCode);
}
zoomedSetSubdir = ensureSetSubdirectory(zoomedSetSubdir);
}
foreach (var card in set.Cards)
{
if (card.IsSingleSide() && card.Faces.Main != card)
{
continue;
}
if (card.IsToken != token)
{
continue;
}
Bitmap original = null;
ImageModel modelSmall = null;
if (small)
{
modelSmall = _imageRepo.GetSmallImage(card, _cardRepo.GetReleaseDateSimilarity);
if (modelSmall != null &&
Str.Equals(card.SetCode, modelSmall.ImageFile.SetCode) == matchingSet &&
exportedSmall.Add(modelSmall.ImageFile.FullPath))
{
FsPath smallPath = getTargetPath(modelSmall.ImageFile, smallSetSubdir);
if (!smallPath.IsFile() || card.Faces.Count > 1)
{
original = ImageLoader.Open(modelSmall);
addFile(original, modelSmall.ImageFile, smallPath, small: true, forceRemoveCorner);
}
}
}
if (zoomed)
{
var modelZoom = _imageRepo.GetImagePrint(card, _cardRepo.GetReleaseDateSimilarity);
if (modelZoom != null &&
Str.Equals(card.SetCode, modelZoom.ImageFile.SetCode) == matchingSet &&
exportedZoomed.Add(modelZoom.ImageFile.FullPath))
{
FsPath zoomedPath = getTargetPath(modelZoom.ImageFile, zoomedSetSubdir);
if (!zoomedPath.IsFile() || card.Faces.Count > 1)
{
if (original == null || modelSmall.ImageFile.FullPath != modelZoom.ImageFile.FullPath)
{
original?.Dispose();
original = ImageLoader.Open(modelZoom);
}
addFile(original, modelZoom.ImageFile, zoomedPath, small: false, forceRemoveCorner);
}
}
}
original?.Dispose();
}
smallSetSubdir.DeleteEmptyDirectory();
zoomedSetSubdir.DeleteEmptyDirectory();
}
}
private async Task downloadSignatures(QualityGroupConfig qualityGroup, CancellationToken token)
{
if (qualityGroup.FileListMegaId == null && qualityGroup.YandexName == null)
{
return;
}
(FsPath signaturesDir, FsPath signaturesFile) = getSignaturesFile(qualityGroup);
FsPath signaturesFileBak = signaturesFile.Concat(".bak");
if (signaturesFile.IsFile())
{
if (signaturesFileBak.IsFile())
{
signaturesFileBak.DeleteFile();
}
signaturesFile.MoveFileTo(signaturesFileBak);
}
signaturesDir.CreateDirectory();
if (qualityGroup.FileListMegaId != null)
{
string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId;
await _megatools.Download(megaUrl, signaturesDir,
name : $"Signatures for {qualityGroup.Quality} images", silent : true, token : token);
}
else if (qualityGroup.YandexName != null)
{
FsPath fileListArchive = signaturesDir.Join("filelist.7z");
var client = new YandexDiskClient();
Console.Write("{0} filelist.7z: get YandexDisk download url ... ", qualityGroup.Name);
var url = await client.GetFilelistDownloadUrl(_config, qualityGroup, token);
Console.Write("downloading ... ");
bool success;
try
{
await client.DownloadFile(url, fileListArchive, token);
Console.WriteLine($"done");
success = true;
}
catch (Exception ex)
{
Console.WriteLine($"failed: {ex.Message}");
_log.Warn(ex, $"Failed download {fileListArchive} from {url}");
success = false;
}
if (success)
{
if (fileListArchive.IsFile())
{
new SevenZip(silent: true).Extract(fileListArchive, signaturesDir);
}
}
}
else
{
throw new ArgumentException($"No downloader can get filelist for quality {qualityGroup.Quality}");
}
if (!signaturesFile.IsFile())
{
if (signaturesFileBak.IsFile())
{
Console.WriteLine("Failed to unzip signatures");
Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile);
signaturesFileBak.MoveFileTo(signaturesFile);
}
}
else
{
signaturesFileBak.DeleteFile();
}
}
public void PreProcessImages(string setCodesStr, bool nonToken, bool token)
{
setupImageConversion();
FsPath smallDir = DevPaths.GathererOriginalDir;
FsPath zoomDir = DevPaths.GathererPreprocessedDir;
FsPath smallDirBak = BakDir.Join(smallDir.Basename());
FsPath zoomDirBak = BakDir.Join(zoomDir.Basename());
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
IEnumerable <FsPath> getSetSubdirs(FsPath typeSubdir)
{
FsPath typeDir = smallDir.Join(typeSubdir);
return(typeDir
.EnumerateDirectories(_fromPng ? "*.png" : "*", SearchOption.TopDirectoryOnly)
.Select(_ => new FsPath(
Regex.Replace(
_.Basename(),
@"\.png$",
string.Empty))));
}
foreach ((_, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
foreach (FsPath setSubdir in getSetSubdirs(typeSubdir))
{
if (setCodes?.Contains(setSubdir.Value) == false)
{
continue;
}
FsPath smallJpgDir = smallDir.Join(typeSubdir, setSubdir);
FsPath zoomJpgDir = zoomDir.Join(typeSubdir, setSubdir);
if (!_fromPng)
{
if (!_createZoom)
{
return;
}
zoomJpgDir.CreateDirectory();
foreach (FsPath smallImg in smallJpgDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
if (_keepExisting && zoomImg.IsFile() && isZoomed(zoomImg))
{
continue;
}
scale(smallImg, zoomImg);
}
}
else
{
FsPath setPngSubdir = setSubdir.Concat(".png");
FsPath smallPngDir = smallDir.Join(typeSubdir, setPngSubdir);
FsPath smallPngDirBak = smallDirBak.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDir = zoomDir.Join(typeSubdir, setPngSubdir);
FsPath zoomPngDirBak = zoomDirBak.Join(typeSubdir, setPngSubdir);
var dirs = new List <(FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom)>();
if (_createZoom)
{
zoomPngDir.CreateDirectory();
foreach (FsPath smallImg in smallPngDir.EnumerateFiles())
{
FsPath zoomImg = smallImg.ChangeDirectory(smallDir, zoomDir);
FsPath convertedImg = zoomImg.ChangeDirectory(zoomPngDir, zoomJpgDir)
.WithName(_ => _.Replace(".png", ".jpg"));
if (_keepExisting && (
zoomImg.IsFile() && isZoomed(zoomImg) ||
convertedImg.IsFile() && isZoomed(convertedImg)))
{
continue;
}
scale(smallImg, zoomImg);
}
dirs.Add((pngDir: zoomPngDir, pngDirBak: zoomPngDirBak, jpgDir: zoomJpgDir, isZoom: true));
}
dirs.Add((pngDir: smallPngDir, pngDirBak: smallPngDirBak, jpgDir: smallJpgDir, isZoom: false));
foreach ((FsPath pngDir, FsPath pngDirBak, FsPath jpgDir, bool isZoom) in dirs)
{
jpgDir.CreateDirectory();
var pngImages = pngDir.EnumerateFiles();
foreach (FsPath sourceImage in pngImages)
{
convertToJpg(sourceImage, jpgDir, isZoom);
}
moveDirectoryToBackup(pngDir, pngDirBak);
}
}
}
}
}
public async Task DownloadGathererImages(string setCodesStr, bool nonToken, bool token)
{
var clients = new List <ImageDownloaderBase>(2)
{
// new GathererClient(),
new ScryfallClient(),
};
var setCodes = setCodesStr?.Split(',').ToHashSet(StringComparer.OrdinalIgnoreCase);
var repo = new CardRepository(new CardFormatter(), () => null)
{
FilterSetCode = setCode => setCodes?.Contains(setCode) != false,
};
repo.LoadFile();
repo.Load();
foreach (Set set in repo.SetsByCode.Values)
{
if (setCodes?.Contains(set.Code) == false)
{
continue;
}
foreach ((bool isToken, FsPath typeSubdir, _) in getIsToken(nonToken, token))
{
var cards = set.List(isToken);
if (cards.Count == 0)
{
continue;
}
var missingCardsByClient = clients.ToDictionary(_ => _, _ => 0);
FsPath setSubdir = new FsPath(set.Code + (Str.Equals(set.Code, "con") ? " escape" : string.Empty));
FsPath downloadRootDir = DevPaths.MtgContentDir.Join(_createZoom ? OriginalSubdir : PreProcessedSubdir);
FsPath rootDirZoom = DevPaths.MtgContentDir.Join(PreProcessedSubdir);
FsPath setDirectory = downloadRootDir.Join(typeSubdir, setSubdir);
FsPath setDirectoryZoom = rootDirZoom.Join(typeSubdir, setSubdir);
FsPath setDirectoryPng = setDirectory.Concat(".png");
bool dirExisted = setDirectoryPng.IsDirectory();
if (!dirExisted)
{
setDirectoryPng.CreateDirectory();
}
foreach (var card in cards)
{
FsPath targetFile = setDirectoryPng.Join(card.ImageName + ".png");
FsPath processedFile = setDirectory.Join(card.ImageName + ".jpg");
FsPath processedFileZoom = setDirectoryZoom.Join(card.ImageName + ".jpg");
if (targetFile.IsFile() || processedFile.IsFile() && processedFileZoom.IsFile())
{
continue;
}
if (targetFile.Basename(extension: false).EndsWith("1"))
{
var unnumbered = targetFile.WithName(_ => _.Replace("1.png", ".png"));
if (unnumbered.IsFile())
{
unnumbered.MoveFileTo(targetFile);
continue;
}
}
foreach (ImageDownloaderBase client in clients)
{
int attempts = 5;
for (int i = 0; i < attempts; i++)
{
var cancellation = new CancellationTokenSource();
var time = DateTime.UtcNow;
var downloadTask = client.DownloadCardImage(card, targetFile, cancellation.Token);
var waitTask = Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token);
await Task.WhenAny(downloadTask, waitTask);
cancellation.Cancel();
var elapsed = DateTime.UtcNow - time;
var delta = TimeSpan.FromSeconds(0.5) - elapsed;
if (delta.TotalSeconds > 0)
{
await Task.Delay(delta);
}
if (targetFile.IsFile())
{
break;
}
}
if (targetFile.IsFile())
{
break;
}
missingCardsByClient[client]++;
}
}
if (!dirExisted && !setDirectoryPng.EnumerateFiles().Any())
{
setDirectoryPng.DeleteDirectory();
}
}
}
}
