public IList <ImageDownloadProgress> GetProgress()
{
var result = new List <ImageDownloadProgress>();
foreach (var qualityGroup in _config.QualityGroups)
{
string signaturesDir = Path.Combine(_updateImgDir, qualityGroup.Quality);
string signaturesFile = Path.Combine(signaturesDir, Signer.SignaturesFile);
var imagesOnline = Signer.ReadFromFile(signaturesFile);
foreach (var dir in qualityGroup.Dirs)
{
var progress = new ImageDownloadProgress(_config, qualityGroup, dir, imagesOnline);
if (progress.FilesOnline != null && progress.FilesOnline.Count == 0)
{
continue;
}
result.Add(progress);
}
}
return(result);
}
public IReadOnlyList <ImageDownloadProgress> GetProgress()
{
var result = new List <ImageDownloadProgress>();
foreach (var qualityGroup in _config.QualityGroups)
{
var imagesOnline = GetOnlineImages(qualityGroup);
if (qualityGroup.YandexName != null && imagesOnline != null)
{
qualityGroup.Dirs = imagesOnline
.Select(_ => _.Path.Parent())
.Distinct()
.OrderBy(_ => _.Value, Str.Comparer)
.Select(_ => new ImageDirConfig {
Subdir = _
}).ToArray();
}
foreach (var dir in qualityGroup.Dirs)
{
var progress = new ImageDownloadProgress(_config, qualityGroup, dir, imagesOnline);
if (progress.FilesOnline != null && progress.FilesOnline.Count == 0)
{
continue;
}
result.Add(progress);
}
}
return(result);
}
public static void WriteExistingSignatures(ImageDownloadProgress progress, IEnumerable <FileSignature> signatures = null)
{
FsPath targetSubdirectory = progress.TargetSubdirectory;
FsPath existingSignaturesFile = targetSubdirectory.Join(Signer.SignaturesFile);
signatures ??= Signer.CreateSignatures(targetSubdirectory);
Signer.WriteToFile(existingSignaturesFile, signatures);
}
public static void WriteExistingSignatures(ImageDownloadProgress progress, IEnumerable <FileSignature> signatures = null)
{
string targetSubdirectory = progress.TargetSubdirectory;
var existingSignaturesFile = Path.Combine(targetSubdirectory, Signer.SignaturesFile);
signatures = signatures ?? Signer.CreateSignatures(targetSubdirectory);
Signer.WriteToFile(existingSignaturesFile, signatures);
}
private string getReleaseDate(ImageDownloadProgress progress)
{
if (!_repo.IsLoadingComplete.Signaled)
{
return(null);
}
return(_repo.SetsByCode.TryGet(progress.Dir.Subdir.Value)?.ReleaseDate);
}
private string getReleaseDate(ImageDownloadProgress progress)
{
if (!_repository.IsLoadingComplete)
{
return(null);
}
return(_repository.SetsByCode.TryGet(progress.Dir.Subdirectory)?.ReleaseDate);
}
private void downloadFromMega(ImageDownloadProgress task)
{
lock (_syncOutput)
Console.WriteLine($"Downloading {task.Dir.Subdirectory} from {task.MegaUrl}");
_megatools.Download(
task.MegaUrl,
task.TargetSubdirectory,
name: task.Dir.Subdirectory,
silent: true);
}
private void downloadFromGdrive(ImageDownloadProgress task, GdriveWebClient webClient)
{
string fileName = task.Dir.Subdirectory + ".7z";
string archiveFileName = task.TargetDirectory.AddPath(fileName);
lock (_syncOutput)
Console.WriteLine($"Downloading {task.Dir.Subdirectory} from {task.GdriveUrl}");
if (File.Exists(archiveFileName))
{
try
{
File.Delete(archiveFileName);
}
catch (Exception ex)
{
Console.WriteLine($"Failed to remove {archiveFileName}: {ex.Message}");
return;
}
}
try
{
webClient.DownloadFromGdrive(task.GdriveUrl, task.TargetDirectory);
}
catch (Exception ex)
{
Console.WriteLine($"Error downloading {archiveFileName} from {task.GdriveUrl}: {ex.Message}");
return;
}
if (!File.Exists(archiveFileName))
{
Console.WriteLine($"Failed to download {archiveFileName} from {task.GdriveUrl}");
return;
}
var sevenZip = new SevenZip(silent: true);
sevenZip.Extract(archiveFileName, task.TargetDirectory, Enumerable.Empty <string>());
Interlocked.Add(ref _countInDownloadedDirs, task.FilesOnline.Count - task.FilesDownloaded.Count);
ProgressChanged?.Invoke();
try
{
File.Delete(archiveFileName);
}
catch (Exception ex)
{
Console.WriteLine($"Failed to remove {archiveFileName}: {ex.Message}");
}
}
public async Task <bool> Download(ImageDownloadProgress task, CancellationToken token)
{
var client = new YandexDiskClientWrapper(_client, task.ImageSource.YandexKey, _syncOutput);
string remotePath = task.ImageSource.GetYandexDiskPath(task.QualityGroup, task.Dir.Subdir);
bool success = await client.DownloadAndExtract(remotePath, task.TargetDirectory, task.Dir.Subdir.Concat(".7z"), token);
if (success)
{
ProgressChanged?.Invoke(task);
}
return(success);
}
public async Task <bool> Download(ImageDownloadProgress task, CancellationToken token)
{
lock (_syncOutput)
Console.WriteLine($"Downloading {task.Dir.Subdir} from {task.MegaUrl}");
await _megatools.Download(
task.MegaUrl,
task.TargetSubdirectory,
name : task.Dir.Subdir.Value,
silent : true,
token : token);
return(true);
}
private void downloadFromGdrive(ImageDownloadProgress task, GdriveWebClient webClient)
{
string fileName = task.Dir.Subdirectory + ".7z";
string targetDirectory = task.TargetDirectory;
string gdriveUrl = task.GdriveUrl;
lock (_syncOutput)
Console.WriteLine($"Downloading {task.Dir.Subdirectory} from {gdriveUrl}");
if (!webClient.DownloadAndExtract(gdriveUrl, targetDirectory, fileName))
{
return;
}
Interlocked.Add(ref _countInDownloadedDirs, task.FilesOnline.Count - task.FilesDownloaded.Count);
ProgressChanged?.Invoke();
}
private void download(ImageDownloadProgress task, Action <ImageDownloadProgress> downloader)
{
if (isAlreadyDownloaded(task))
{
Console.WriteLine("[Skip] {0}", task.Dir.Subdirectory);
Interlocked.Add(ref _countInDownloadedDirs, task.FilesOnline.Count);
ProgressChanged?.Invoke();
}
else
{
Interlocked.Add(ref _countInDownloadedDirs, task.FilesDownloaded.Count);
ProgressChanged?.Invoke();
downloader(task);
ImageDownloadProgressReader.WriteExistingSignatures(task);
}
}
public bool PushFailedTaskBack(IDownloader downloader, ImageDownloadProgress task)
{
bool success = false;
lock (_sync)
{
_failedByDownloader[downloader].Add(task);
foreach (var(dr, queue) in _queueByDownloader)
{
if (!_failedByDownloader[dr].Contains(task))
{
success = true;
queue.Add(task);
}
}
}
return(success);
}
public bool CanDownload(ImageDownloadProgress task) => task.QualityGroup.YandexName != null;
public bool CanDownload(ImageDownloadProgress task) => task.MegaUrl != null;
private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
{
FsPath targetSubdirectory = progress.TargetSubdirectory;
targetSubdirectory.CreateDirectory();
if (progress.FilesOnline == null)
{
return(false);
}
bool alreadyDownloaded = true;
var existingFiles = new HashSet <FsPath>(
targetSubdirectory.EnumerateFiles("*", SearchOption.AllDirectories));
var existingSignatures = new Dictionary <FsPath, FileSignature>();
foreach (var fileOnline in progress.FilesOnline.Values)
{
FsPath filePath = targetSubdirectory.Join(fileOnline.Path);
if (!existingFiles.Contains(filePath))
{
alreadyDownloaded = false;
continue;
}
FileSignature tempQualifier = Signer.CreateSignature(filePath, useAbsolutePath: true);
var existingSignature =
progress.FilesCorrupted.TryGet(fileOnline.Path) ??
progress.FilesDownloaded.TryGet(fileOnline.Path) ??
new FileSignature
{
Path = tempQualifier.Path.RelativeTo(targetSubdirectory).Intern(true),
Md5Hash = tempQualifier.Md5Hash
};
if (existingSignature.Md5Hash != fileOnline.Md5Hash)
{
alreadyDownloaded = false;
Console.WriteLine("Deleting modified or corrupted file {0}", filePath);
lock (ImageLoader.SyncIo)
{
try
{
filePath.DeleteFile();
}
catch (IOException ex)
{
Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
}
}
}
else
{
existingSignatures.Add(existingSignature.Path, existingSignature);
}
}
foreach (FsPath file in existingFiles)
{
var relativePath = file.RelativeTo(targetSubdirectory);
if (!progress.FilesOnline.ContainsKey(relativePath) && relativePath != Signer.SignaturesFile)
{
Console.WriteLine("Deleting {0}", file);
file.DeleteFile();
}
}
if (alreadyDownloaded)
{
ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
}
return(alreadyDownloaded);
}
private static bool isAlreadyDownloaded(ImageDownloadProgress progress)
{
string targetSubdirectory = progress.TargetSubdirectory;
Directory.CreateDirectory(targetSubdirectory);
if (progress.FilesOnline == null)
{
return(false);
}
bool alreadyDownloaded = true;
var existingFiles = new HashSet <string>(
Directory.GetFiles(targetSubdirectory, "*.*", SearchOption.AllDirectories),
Str.Comparer);
var existingSignatures = new Dictionary <string, FileSignature>(Str.Comparer);
foreach (var fileOnline in progress.FilesOnline.Values)
{
string filePath = Path.Combine(targetSubdirectory, fileOnline.Path);
if (!existingFiles.Contains(filePath))
{
alreadyDownloaded = false;
continue;
}
var existingSignature =
progress.FilesCorrupted.TryGet(fileOnline.Path) ??
progress.FilesDownloaded.TryGet(fileOnline.Path) ??
Signer.CreateSignature(filePath, useAbsolutePath: true).AsRelativeTo(targetSubdirectory);
if (existingSignature.Md5Hash != fileOnline.Md5Hash)
{
alreadyDownloaded = false;
Console.WriteLine("Deleting modified or corrupted file {0}", filePath);
lock (ImageLoader.SyncIo)
{
try
{
File.Delete(filePath);
}
catch (IOException ex)
{
Console.WriteLine($"Failed to remove {filePath}. {ex.Message}");
}
}
}
else
{
existingSignatures.Add(existingSignature.Path, existingSignature);
}
}
foreach (string file in existingFiles)
{
var relativePath = file.Substring(targetSubdirectory.Length + 1);
if (!progress.FilesOnline.ContainsKey(relativePath) && !Str.Equals(relativePath, Signer.SignaturesFile))
{
Console.WriteLine("Deleting {0}", file);
File.Delete(file);
}
}
if (alreadyDownloaded)
{
ImageDownloadProgressReader.WriteExistingSignatures(progress, existingSignatures.Values);
}
return(alreadyDownloaded);
}