public static async Task ScrapReddit(string subredditName, ISocketMessageChannel channel)
{
DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, true);
var reddit = new RedditClient(Program.ApplicationSetting.RedditSetting.AppId, Program.ApplicationSetting.RedditSetting.RefreshToken, Program.ApplicationSetting.RedditSetting.AppSecret);
using (var context = new ETHBotDBContext())
{
SubredditManager subManager = new SubredditManager(subredditName, reddit, context);
bool ended = subManager.SubredditInfo.ReachedOldest;
try
{
bool beforeWasNotFull = false;
string last = "";
DateTime lastTime = subManager.SubredditInfo.ReachedOldest ? DateTime.MinValue : DateTime.MaxValue;
while (true)
{
List <Post> posts = null;
if (!subManager.SubredditInfo.ReachedOldest)
{
posts = subManager.GetAfterPosts();
}
else
{
posts = subManager.GetBeforePosts();
}
if (posts == null)
{
return; // see what to do
}
if (!subManager.SubredditInfo.ReachedOldest && posts.Count == 0)
{
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the end. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
subManager.ConfirmOldestPost(last, lastTime, true);
break;
}
if (posts.Count == 0)
{
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the newest post. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
subManager.ConfirmNewestPost(last, lastTime);
break;
}
string lastPrev = last;
foreach (var post in posts)
{
await Task.Delay(25);
if (post.Created < lastTime && !subManager.SubredditInfo.ReachedOldest)
{
last = post.Fullname;
lastTime = post.Created;
}
if (post.Created > lastTime && subManager.SubredditInfo.ReachedOldest)
{
last = post.Fullname;
lastTime = post.Created;
}
PostManager manager = new PostManager(post, subManager.SubredditInfo, context);
if (manager.IsImage())
{
var imageInfos = manager.DownloadImage(Path.Combine(Program.ApplicationSetting.BasePath, "Reddit")); // TODO send path in constructor
context.RedditImages.AddRange(imageInfos);
context.SaveChanges();
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine($"IGNORED {post.Title} at {last}");
Console.ForegroundColor = ConsoleColor.White;
}
}
if (lastPrev == last)
{
// TODO set end reached if its not yet set?
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} stopped because last/first did not change Count ({posts.Count})", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
break;
}
if (!subManager.SubredditInfo.ReachedOldest)
{
subManager.ConfirmOldestPost(last, lastTime);
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) after {subManager.SubredditInfo.OldestPost}/{subManager.SubredditInfo.OldestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
else
{
subManager.ConfirmNewestPost(last, lastTime);
//subManager.GetBeforePosts();
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) before {subManager.SubredditInfo.NewestPost}/{subManager.SubredditInfo.NewestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
}
}
catch (Exception ex)
{
await channel.SendMessageAsync($"{subManager.SubredditName} scraper died RIP {ex.Message}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
//if(!ended)
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper ended :D", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, false);
}
