Esempio n. 1
0
        internal FileSignature[] GetOnlineImages(QualityGroupConfig qualityGroup)
        {
            (_, FsPath signaturesFile) = getSignaturesFile(qualityGroup);
            // do not intern image path here because it contains parent directory
            var imagesOnline = Signer.ReadFromFile(signaturesFile, internPath: false);

            return(imagesOnline);
        }
Esempio n. 2
0
        public ImageDownloadProgress(
            ImageSourcesConfig imageSource,
            QualityGroupConfig qualityGroup,
            ImageDirConfig dir,
            FileSignature[] imagesOnline)
        {
            ImageSource  = imageSource;
            QualityGroup = qualityGroup;
            Dir          = dir;

            TargetDirectory    = QualityGroup.TargetDirectory.ToAppRootedPath();
            TargetSubdirectory = Dir.Subdir.Value == null
                                ? TargetDirectory
                                : TargetDirectory.Join(Dir.Subdir);

            MegaUrl = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId;

            var existingSignatures = readExistingSignatures()
                                     ?.ToDictionary(_ => _.Path);

            if (imagesOnline == null)
            {
                FilesDownloaded = existingSignatures;
                return;
            }

            FilesOnline = Dir.Subdir.Value == null
                                ? imagesOnline.ToDictionary(_ => _.Path)
                                : imagesOnline.Where(_ => Dir.Subdir.IsParentOf(_.Path))
                          .Select(_ => new FileSignature
            {
                Path    = _.Path.RelativeTo(Dir.Subdir).Intern(true),
                Md5Hash = _.Md5Hash
            })
                          .ToDictionary(_ => _.Path);

            FilesDownloaded = new Dictionary <FsPath, FileSignature>();
            FilesCorrupted  = new Dictionary <FsPath, FileSignature>();

            foreach (var onlineImage in FilesOnline.Values)
            {
                var existingSignature = existingSignatures?.TryGet(onlineImage.Path);

                if (existingSignature == null)
                {
                    continue;
                }

                if (existingSignature.Md5Hash == onlineImage.Md5Hash)
                {
                    FilesDownloaded.Add(existingSignature.Path, existingSignature);
                }
                else
                {
                    FilesCorrupted.Add(existingSignature.Path, existingSignature);
                }
            }
        }
Esempio n. 3
0
        public ImageDownloadProgress(
            ImageSourcesConfig imageSource,
            QualityGroupConfig qualityGroup,
            ImageDirConfig dir,
            FileSignature[] imagesOnline)
        {
            ImageSource  = imageSource;
            QualityGroup = qualityGroup;
            Dir          = dir;

            TargetDirectory    = QualityGroup.TargetDirectory.ToAppRootedPath();
            TargetSubdirectory = TargetDirectory.AddPath(Dir.Subdirectory);

            MegaUrl   = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId;
            GdriveUrl = string.IsNullOrEmpty(Dir.GdriveId) ? null: ImageSource.GdrivePrefix + Dir.GdriveId;

            var existingSignatures = readExistingSignatures()
                                     ?.ToDictionary(_ => _.Path, Str.Comparer);

            if (imagesOnline == null)
            {
                FilesDownloaded = existingSignatures;
                return;
            }

            FilesOnline = imagesOnline
                          .Where(_ => _.IsRelativeTo(Dir.Subdirectory))
                          .Select(_ => _.AsRelativeTo(Dir.Subdirectory))
                          .ToDictionary(_ => _.Path, Str.Comparer);

            FilesDownloaded = new Dictionary <string, FileSignature>(Str.Comparer);
            FilesCorrupted  = new Dictionary <string, FileSignature>(Str.Comparer);

            foreach (var onlineImage in FilesOnline.Values)
            {
                var existingSignature = existingSignatures?.TryGet(onlineImage.Path);

                if (existingSignature == null)
                {
                    continue;
                }

                if (existingSignature.Md5Hash == onlineImage.Md5Hash)
                {
                    FilesDownloaded.Add(existingSignature.Path, existingSignature);
                }
                else
                {
                    FilesCorrupted.Add(existingSignature.Path, existingSignature);
                }
            }
        }
Esempio n. 4
0
        private void downloadSignatures(QualityGroupConfig qualityGroup)
        {
            if (qualityGroup.FileListMegaId == null && qualityGroup.FileListGdriveId == null)
            {
                return;
            }

            string signaturesDir     = Path.Combine(_updateImgDir, qualityGroup.Quality);
            string signaturesFile    = Path.Combine(signaturesDir, Signer.SignaturesFile);
            string signaturesFileBak = signaturesFile + ".bak";

            if (File.Exists(signaturesFile))
            {
                if (File.Exists(signaturesFileBak))
                {
                    File.Delete(signaturesFileBak);
                }

                File.Move(signaturesFile, signaturesFileBak);
            }

            if (qualityGroup.FileListMegaId != null)
            {
                string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId;
                _megatools.Download(megaUrl, signaturesDir, $"Signatures for {qualityGroup.Quality} images", silent: true);
            }
            else if (qualityGroup.FileListGdriveId != null)
            {
                var    webClient       = new WebClientBase();
                string gdriveUrl       = _config.GdrivePrefix + qualityGroup.FileListGdriveId;
                string fileListArchive = signaturesDir.AddPath("filelist.7z");
                Console.WriteLine($"Downloading {fileListArchive} from {gdriveUrl}");
                webClient.DownloadFile(gdriveUrl, fileListArchive);
                new SevenZip(silent: true).Extract(fileListArchive, signaturesDir);
            }

            if (!File.Exists(signaturesFile))
            {
                if (File.Exists(signaturesFileBak))
                {
                    Console.WriteLine("Failed to download signatures");

                    Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile);
                    File.Move(signaturesFileBak, signaturesFile);
                }
            }
            else
            {
                File.Delete(signaturesFileBak);
            }
        }
Esempio n. 5
0
 public string GetYandexDiskPath(QualityGroupConfig quality, FsPath subdirectory) =>
 string.Format(YandexDirPath, quality.YandexName, subdirectory);
Esempio n. 6
0
        private async Task downloadSignatures(QualityGroupConfig qualityGroup, CancellationToken token)
        {
            if (qualityGroup.FileListMegaId == null && qualityGroup.YandexName == null)
            {
                return;
            }

            (FsPath signaturesDir, FsPath signaturesFile) = getSignaturesFile(qualityGroup);
            FsPath signaturesFileBak = signaturesFile.Concat(".bak");

            if (signaturesFile.IsFile())
            {
                if (signaturesFileBak.IsFile())
                {
                    signaturesFileBak.DeleteFile();
                }

                signaturesFile.MoveFileTo(signaturesFileBak);
            }

            signaturesDir.CreateDirectory();

            if (qualityGroup.FileListMegaId != null)
            {
                string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId;
                await _megatools.Download(megaUrl, signaturesDir,
                                          name : $"Signatures for {qualityGroup.Quality} images", silent : true, token : token);
            }
            else if (qualityGroup.YandexName != null)
            {
                FsPath fileListArchive = signaturesDir.Join("filelist.7z");
                var    client          = new YandexDiskClient();
                Console.Write("{0} filelist.7z: get YandexDisk download url ... ", qualityGroup.Name);
                var url = await client.GetFilelistDownloadUrl(_config, qualityGroup, token);

                Console.Write("downloading ... ");

                bool success;
                try
                {
                    await client.DownloadFile(url, fileListArchive, token);

                    Console.WriteLine($"done");
                    success = true;
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"failed: {ex.Message}");
                    _log.Warn(ex, $"Failed download {fileListArchive} from {url}");
                    success = false;
                }

                if (success)
                {
                    if (fileListArchive.IsFile())
                    {
                        new SevenZip(silent: true).Extract(fileListArchive, signaturesDir);
                    }
                }
            }
            else
            {
                throw new ArgumentException($"No downloader can get filelist for quality {qualityGroup.Quality}");
            }


            if (!signaturesFile.IsFile())
            {
                if (signaturesFileBak.IsFile())
                {
                    Console.WriteLine("Failed to unzip signatures");

                    Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile);
                    signaturesFileBak.MoveFileTo(signaturesFile);
                }
            }
            else
            {
                signaturesFileBak.DeleteFile();
            }
        }
Esempio n. 7
0
 public bool SignaturesFileExist(QualityGroupConfig qualityGroup) =>
 getSignaturesFile(qualityGroup).File.IsFile();
Esempio n. 8
0
 private static FsPath getSignaturesDir(QualityGroupConfig qualityGroup) =>
 _updateImgDir.Join(qualityGroup.Name);
Esempio n. 9
0
        private static (FsPath Directory, FsPath File) getSignaturesFile(QualityGroupConfig qualityGroup)
        {
            FsPath dir = getSignaturesDir(qualityGroup);

            return(Directory : dir, File : dir.Join(Signer.SignaturesFile));
        }
Esempio n. 10
0
 public Task <string> GetFilelistDownloadUrl(ImageSourcesConfig source, QualityGroupConfig quality, CancellationToken token) =>
 GetDownloadLink(source.YandexKey, string.Format(source.YandexListPath, quality.YandexName), token);