private IEnumerable <int> GetPoolIdsForPost(Post post)
{
var url = $"https://e621.net/post/show/{post.Id}";
var data = WebClientHelper.GetE621WebClient().DownloadString(url);
var regex = new Regex(@"\/pool\/show\/(\d+)");
if (regex.IsMatch(data))
{
var ids = new List <int>();
foreach (Match m in regex.Matches(data))
{
ids.Add(Convert.ToInt32(m.Groups[1].Value));
}
return(ids);
}
return(null);
}
private async Task DownloadPostsByPoolIdAsync(int poolId, string directory, Action onNewPostDiscovered, Action onNewPostDownload, Action onFinishedPostDownload, int tryCount = 0)
{
if (tryCount > 5)
{
return;
}
Debug.WriteLine($"Downloading Pool {poolId}");
await this.PoolDownloadSemaphore.WaitAsync().ConfigureAwait(true);
try
{
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
var tasks = new List <Task>();
var index = 0;
foreach (var p in this.GetPostsByPoolId(poolId))
{
onNewPostDiscovered.Invoke();
await this.PostDownloadSemaphore.WaitAsync().ConfigureAwait(true);
Debug.WriteLine($"Downloading Post {p}");
onNewPostDownload.Invoke();
var url = p.FileUrl;
var file = Path.Combine(directory, index + "." + p.Data.Element("file_ext").Value);
if (File.Exists(file))
{
this.PostDownloadSemaphore.Release();
onFinishedPostDownload.Invoke();
continue;
}
var task = Task.Run(async() =>
{
var postdownloadtry = 0;
var downloaded = false;
do
{
postdownloadtry++;
try
{
await WebClientHelper.GetE621WebClient().DownloadFileTaskAsync(new Uri(url), file);
downloaded = true;
}
catch
{
Debug.WriteLine($"Downloading Post {p} FAILED. Try: {postdownloadtry}");
}
} while (!downloaded && postdownloadtry < 5);
this.PostDownloadSemaphore.Release();
onFinishedPostDownload.Invoke();
});
index++;
tasks.Add(task);
}
await Task.WhenAll(tasks);
}
catch (Exception ex)
{
tryCount++;
await Task.Run(() => this.DownloadPostsByPoolIdAsync(poolId, directory, onNewPostDiscovered, onNewPostDownload, onFinishedPostDownload, tryCount));
}
finally
{
this.PoolDownloadSemaphore.Release();
}
}
public async Task DownloadPostsByTagsAsync(string tags, string targetDirAll, Action <float?, string> onUpdate)
{
var postsFound = 0;
var postsDownloaded = 0;
var currentDownloadingPosts = 0;
void sendUpdate()
{
var str = "Downloading posts...\n" +
$"Posts: {postsDownloaded} / {postsFound} (Downloading {currentDownloadingPosts})\n";
var percent = 0f;
if (postsFound != 0)
{
percent = postsDownloaded * 100 / postsFound;
}
onUpdate.Invoke(percent, str);
}
await Task.Run(async() =>
{
var tasks = new List <Task>();
foreach (var post in this.GetPostsByTags(tags))
{
postsFound++;
sendUpdate();
var task = Task.Run(async() =>
{
await this.PostDownloadSemaphore.WaitAsync().ConfigureAwait(true);
currentDownloadingPosts++;
sendUpdate();
var url = post.FileUrl;
var filename = post.Data.Element("id").Value + "." + post.Data.Element("file_ext").Value;
var file = Path.Combine(targetDirAll, filename);
if (!File.Exists(file))
{
var postdownloadtry = 0;
var downloaded = false;
do
{
postdownloadtry++;
try
{
Debug.WriteLine($"Downloading Post {post.Id} START. Try: {postdownloadtry}");
await WebClientHelper.GetE621WebClient().DownloadFileTaskAsync(new Uri(url), file);
downloaded = true;
}
catch
{
Debug.WriteLine($"Downloading Post {post.Id} FAILED. Try: {postdownloadtry}");
}
} while (!downloaded && postdownloadtry < 5);
}
this.PostDownloadSemaphore.Release();
postsDownloaded++;
currentDownloadingPosts--;
sendUpdate();
});
tasks.Add(task);
}
await Task.WhenAll(tasks);
});
}
