internal FileSignature[] GetOnlineImages(QualityGroupConfig qualityGroup) { (_, FsPath signaturesFile) = getSignaturesFile(qualityGroup); // do not intern image path here because it contains parent directory var imagesOnline = Signer.ReadFromFile(signaturesFile, internPath: false); return(imagesOnline); }
public ImageDownloadProgress( ImageSourcesConfig imageSource, QualityGroupConfig qualityGroup, ImageDirConfig dir, FileSignature[] imagesOnline) { ImageSource = imageSource; QualityGroup = qualityGroup; Dir = dir; TargetDirectory = QualityGroup.TargetDirectory.ToAppRootedPath(); TargetSubdirectory = Dir.Subdir.Value == null ? TargetDirectory : TargetDirectory.Join(Dir.Subdir); MegaUrl = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId; var existingSignatures = readExistingSignatures() ?.ToDictionary(_ => _.Path); if (imagesOnline == null) { FilesDownloaded = existingSignatures; return; } FilesOnline = Dir.Subdir.Value == null ? imagesOnline.ToDictionary(_ => _.Path) : imagesOnline.Where(_ => Dir.Subdir.IsParentOf(_.Path)) .Select(_ => new FileSignature { Path = _.Path.RelativeTo(Dir.Subdir).Intern(true), Md5Hash = _.Md5Hash }) .ToDictionary(_ => _.Path); FilesDownloaded = new Dictionary <FsPath, FileSignature>(); FilesCorrupted = new Dictionary <FsPath, FileSignature>(); foreach (var onlineImage in FilesOnline.Values) { var existingSignature = existingSignatures?.TryGet(onlineImage.Path); if (existingSignature == null) { continue; } if (existingSignature.Md5Hash == onlineImage.Md5Hash) { FilesDownloaded.Add(existingSignature.Path, existingSignature); } else { FilesCorrupted.Add(existingSignature.Path, existingSignature); } } }
public ImageDownloadProgress( ImageSourcesConfig imageSource, QualityGroupConfig qualityGroup, ImageDirConfig dir, FileSignature[] imagesOnline) { ImageSource = imageSource; QualityGroup = qualityGroup; Dir = dir; TargetDirectory = QualityGroup.TargetDirectory.ToAppRootedPath(); TargetSubdirectory = TargetDirectory.AddPath(Dir.Subdirectory); MegaUrl = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId; GdriveUrl = string.IsNullOrEmpty(Dir.GdriveId) ? null: ImageSource.GdrivePrefix + Dir.GdriveId; var existingSignatures = readExistingSignatures() ?.ToDictionary(_ => _.Path, Str.Comparer); if (imagesOnline == null) { FilesDownloaded = existingSignatures; return; } FilesOnline = imagesOnline .Where(_ => _.IsRelativeTo(Dir.Subdirectory)) .Select(_ => _.AsRelativeTo(Dir.Subdirectory)) .ToDictionary(_ => _.Path, Str.Comparer); FilesDownloaded = new Dictionary <string, FileSignature>(Str.Comparer); FilesCorrupted = new Dictionary <string, FileSignature>(Str.Comparer); foreach (var onlineImage in FilesOnline.Values) { var existingSignature = existingSignatures?.TryGet(onlineImage.Path); if (existingSignature == null) { continue; } if (existingSignature.Md5Hash == onlineImage.Md5Hash) { FilesDownloaded.Add(existingSignature.Path, existingSignature); } else { FilesCorrupted.Add(existingSignature.Path, existingSignature); } } }
private void downloadSignatures(QualityGroupConfig qualityGroup) { if (qualityGroup.FileListMegaId == null && qualityGroup.FileListGdriveId == null) { return; } string signaturesDir = Path.Combine(_updateImgDir, qualityGroup.Quality); string signaturesFile = Path.Combine(signaturesDir, Signer.SignaturesFile); string signaturesFileBak = signaturesFile + ".bak"; if (File.Exists(signaturesFile)) { if (File.Exists(signaturesFileBak)) { File.Delete(signaturesFileBak); } File.Move(signaturesFile, signaturesFileBak); } if (qualityGroup.FileListMegaId != null) { string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId; _megatools.Download(megaUrl, signaturesDir, $"Signatures for {qualityGroup.Quality} images", silent: true); } else if (qualityGroup.FileListGdriveId != null) { var webClient = new WebClientBase(); string gdriveUrl = _config.GdrivePrefix + qualityGroup.FileListGdriveId; string fileListArchive = signaturesDir.AddPath("filelist.7z"); Console.WriteLine($"Downloading {fileListArchive} from {gdriveUrl}"); webClient.DownloadFile(gdriveUrl, fileListArchive); new SevenZip(silent: true).Extract(fileListArchive, signaturesDir); } if (!File.Exists(signaturesFile)) { if (File.Exists(signaturesFileBak)) { Console.WriteLine("Failed to download signatures"); Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile); File.Move(signaturesFileBak, signaturesFile); } } else { File.Delete(signaturesFileBak); } }
public string GetYandexDiskPath(QualityGroupConfig quality, FsPath subdirectory) => string.Format(YandexDirPath, quality.YandexName, subdirectory);
private async Task downloadSignatures(QualityGroupConfig qualityGroup, CancellationToken token) { if (qualityGroup.FileListMegaId == null && qualityGroup.YandexName == null) { return; } (FsPath signaturesDir, FsPath signaturesFile) = getSignaturesFile(qualityGroup); FsPath signaturesFileBak = signaturesFile.Concat(".bak"); if (signaturesFile.IsFile()) { if (signaturesFileBak.IsFile()) { signaturesFileBak.DeleteFile(); } signaturesFile.MoveFileTo(signaturesFileBak); } signaturesDir.CreateDirectory(); if (qualityGroup.FileListMegaId != null) { string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId; await _megatools.Download(megaUrl, signaturesDir, name : $"Signatures for {qualityGroup.Quality} images", silent : true, token : token); } else if (qualityGroup.YandexName != null) { FsPath fileListArchive = signaturesDir.Join("filelist.7z"); var client = new YandexDiskClient(); Console.Write("{0} filelist.7z: get YandexDisk download url ... ", qualityGroup.Name); var url = await client.GetFilelistDownloadUrl(_config, qualityGroup, token); Console.Write("downloading ... "); bool success; try { await client.DownloadFile(url, fileListArchive, token); Console.WriteLine($"done"); success = true; } catch (Exception ex) { Console.WriteLine($"failed: {ex.Message}"); _log.Warn(ex, $"Failed download {fileListArchive} from {url}"); success = false; } if (success) { if (fileListArchive.IsFile()) { new SevenZip(silent: true).Extract(fileListArchive, signaturesDir); } } } else { throw new ArgumentException($"No downloader can get filelist for quality {qualityGroup.Quality}"); } if (!signaturesFile.IsFile()) { if (signaturesFileBak.IsFile()) { Console.WriteLine("Failed to unzip signatures"); Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile); signaturesFileBak.MoveFileTo(signaturesFile); } } else { signaturesFileBak.DeleteFile(); } }
public bool SignaturesFileExist(QualityGroupConfig qualityGroup) => getSignaturesFile(qualityGroup).File.IsFile();
private static FsPath getSignaturesDir(QualityGroupConfig qualityGroup) => _updateImgDir.Join(qualityGroup.Name);
private static (FsPath Directory, FsPath File) getSignaturesFile(QualityGroupConfig qualityGroup) { FsPath dir = getSignaturesDir(qualityGroup); return(Directory : dir, File : dir.Join(Signer.SignaturesFile)); }
public Task <string> GetFilelistDownloadUrl(ImageSourcesConfig source, QualityGroupConfig quality, CancellationToken token) => GetDownloadLink(source.YandexKey, string.Format(source.YandexListPath, quality.YandexName), token);