internal FileSignature[] GetOnlineImages(QualityGroupConfig qualityGroup)
{
(_, FsPath signaturesFile) = getSignaturesFile(qualityGroup);
// do not intern image path here because it contains parent directory
var imagesOnline = Signer.ReadFromFile(signaturesFile, internPath: false);
return(imagesOnline);
}
public ImageDownloadProgress(
ImageSourcesConfig imageSource,
QualityGroupConfig qualityGroup,
ImageDirConfig dir,
FileSignature[] imagesOnline)
{
ImageSource = imageSource;
QualityGroup = qualityGroup;
Dir = dir;
TargetDirectory = QualityGroup.TargetDirectory.ToAppRootedPath();
TargetSubdirectory = Dir.Subdir.Value == null
? TargetDirectory
: TargetDirectory.Join(Dir.Subdir);
MegaUrl = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId;
var existingSignatures = readExistingSignatures()
?.ToDictionary(_ => _.Path);
if (imagesOnline == null)
{
FilesDownloaded = existingSignatures;
return;
}
FilesOnline = Dir.Subdir.Value == null
? imagesOnline.ToDictionary(_ => _.Path)
: imagesOnline.Where(_ => Dir.Subdir.IsParentOf(_.Path))
.Select(_ => new FileSignature
{
Path = _.Path.RelativeTo(Dir.Subdir).Intern(true),
Md5Hash = _.Md5Hash
})
.ToDictionary(_ => _.Path);
FilesDownloaded = new Dictionary <FsPath, FileSignature>();
FilesCorrupted = new Dictionary <FsPath, FileSignature>();
foreach (var onlineImage in FilesOnline.Values)
{
var existingSignature = existingSignatures?.TryGet(onlineImage.Path);
if (existingSignature == null)
{
continue;
}
if (existingSignature.Md5Hash == onlineImage.Md5Hash)
{
FilesDownloaded.Add(existingSignature.Path, existingSignature);
}
else
{
FilesCorrupted.Add(existingSignature.Path, existingSignature);
}
}
}
public ImageDownloadProgress(
ImageSourcesConfig imageSource,
QualityGroupConfig qualityGroup,
ImageDirConfig dir,
FileSignature[] imagesOnline)
{
ImageSource = imageSource;
QualityGroup = qualityGroup;
Dir = dir;
TargetDirectory = QualityGroup.TargetDirectory.ToAppRootedPath();
TargetSubdirectory = TargetDirectory.AddPath(Dir.Subdirectory);
MegaUrl = string.IsNullOrEmpty(Dir.MegaId) ? null : ImageSource.MegaPrefix + Dir.MegaId;
GdriveUrl = string.IsNullOrEmpty(Dir.GdriveId) ? null: ImageSource.GdrivePrefix + Dir.GdriveId;
var existingSignatures = readExistingSignatures()
?.ToDictionary(_ => _.Path, Str.Comparer);
if (imagesOnline == null)
{
FilesDownloaded = existingSignatures;
return;
}
FilesOnline = imagesOnline
.Where(_ => _.IsRelativeTo(Dir.Subdirectory))
.Select(_ => _.AsRelativeTo(Dir.Subdirectory))
.ToDictionary(_ => _.Path, Str.Comparer);
FilesDownloaded = new Dictionary <string, FileSignature>(Str.Comparer);
FilesCorrupted = new Dictionary <string, FileSignature>(Str.Comparer);
foreach (var onlineImage in FilesOnline.Values)
{
var existingSignature = existingSignatures?.TryGet(onlineImage.Path);
if (existingSignature == null)
{
continue;
}
if (existingSignature.Md5Hash == onlineImage.Md5Hash)
{
FilesDownloaded.Add(existingSignature.Path, existingSignature);
}
else
{
FilesCorrupted.Add(existingSignature.Path, existingSignature);
}
}
}
private void downloadSignatures(QualityGroupConfig qualityGroup)
{
if (qualityGroup.FileListMegaId == null && qualityGroup.FileListGdriveId == null)
{
return;
}
string signaturesDir = Path.Combine(_updateImgDir, qualityGroup.Quality);
string signaturesFile = Path.Combine(signaturesDir, Signer.SignaturesFile);
string signaturesFileBak = signaturesFile + ".bak";
if (File.Exists(signaturesFile))
{
if (File.Exists(signaturesFileBak))
{
File.Delete(signaturesFileBak);
}
File.Move(signaturesFile, signaturesFileBak);
}
if (qualityGroup.FileListMegaId != null)
{
string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId;
_megatools.Download(megaUrl, signaturesDir, $"Signatures for {qualityGroup.Quality} images", silent: true);
}
else if (qualityGroup.FileListGdriveId != null)
{
var webClient = new WebClientBase();
string gdriveUrl = _config.GdrivePrefix + qualityGroup.FileListGdriveId;
string fileListArchive = signaturesDir.AddPath("filelist.7z");
Console.WriteLine($"Downloading {fileListArchive} from {gdriveUrl}");
webClient.DownloadFile(gdriveUrl, fileListArchive);
new SevenZip(silent: true).Extract(fileListArchive, signaturesDir);
}
if (!File.Exists(signaturesFile))
{
if (File.Exists(signaturesFileBak))
{
Console.WriteLine("Failed to download signatures");
Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile);
File.Move(signaturesFileBak, signaturesFile);
}
}
else
{
File.Delete(signaturesFileBak);
}
}
public string GetYandexDiskPath(QualityGroupConfig quality, FsPath subdirectory) => string.Format(YandexDirPath, quality.YandexName, subdirectory);
private async Task downloadSignatures(QualityGroupConfig qualityGroup, CancellationToken token)
{
if (qualityGroup.FileListMegaId == null && qualityGroup.YandexName == null)
{
return;
}
(FsPath signaturesDir, FsPath signaturesFile) = getSignaturesFile(qualityGroup);
FsPath signaturesFileBak = signaturesFile.Concat(".bak");
if (signaturesFile.IsFile())
{
if (signaturesFileBak.IsFile())
{
signaturesFileBak.DeleteFile();
}
signaturesFile.MoveFileTo(signaturesFileBak);
}
signaturesDir.CreateDirectory();
if (qualityGroup.FileListMegaId != null)
{
string megaUrl = _config.MegaPrefix + qualityGroup.FileListMegaId;
await _megatools.Download(megaUrl, signaturesDir,
name : $"Signatures for {qualityGroup.Quality} images", silent : true, token : token);
}
else if (qualityGroup.YandexName != null)
{
FsPath fileListArchive = signaturesDir.Join("filelist.7z");
var client = new YandexDiskClient();
Console.Write("{0} filelist.7z: get YandexDisk download url ... ", qualityGroup.Name);
var url = await client.GetFilelistDownloadUrl(_config, qualityGroup, token);
Console.Write("downloading ... ");
bool success;
try
{
await client.DownloadFile(url, fileListArchive, token);
Console.WriteLine($"done");
success = true;
}
catch (Exception ex)
{
Console.WriteLine($"failed: {ex.Message}");
_log.Warn(ex, $"Failed download {fileListArchive} from {url}");
success = false;
}
if (success)
{
if (fileListArchive.IsFile())
{
new SevenZip(silent: true).Extract(fileListArchive, signaturesDir);
}
}
}
else
{
throw new ArgumentException($"No downloader can get filelist for quality {qualityGroup.Quality}");
}
if (!signaturesFile.IsFile())
{
if (signaturesFileBak.IsFile())
{
Console.WriteLine("Failed to unzip signatures");
Console.WriteLine("Move {0} {1}", signaturesFileBak, signaturesFile);
signaturesFileBak.MoveFileTo(signaturesFile);
}
}
else
{
signaturesFileBak.DeleteFile();
}
}
public bool SignaturesFileExist(QualityGroupConfig qualityGroup) => getSignaturesFile(qualityGroup).File.IsFile();
private static FsPath getSignaturesDir(QualityGroupConfig qualityGroup) => _updateImgDir.Join(qualityGroup.Name);
private static (FsPath Directory, FsPath File) getSignaturesFile(QualityGroupConfig qualityGroup)
{
FsPath dir = getSignaturesDir(qualityGroup);
return(Directory : dir, File : dir.Join(Signer.SignaturesFile));
}
public Task <string> GetFilelistDownloadUrl(ImageSourcesConfig source, QualityGroupConfig quality, CancellationToken token) => GetDownloadLink(source.YandexKey, string.Format(source.YandexListPath, quality.YandexName), token);