public static async Task ScrapReddit(string subredditName, ISocketMessageChannel channel) { DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, true); var reddit = new RedditClient(Program.ApplicationSetting.RedditSetting.AppId, Program.ApplicationSetting.RedditSetting.RefreshToken, Program.ApplicationSetting.RedditSetting.AppSecret); using (var context = new ETHBotDBContext()) { SubredditManager subManager = new SubredditManager(subredditName, reddit, context); bool ended = subManager.SubredditInfo.ReachedOldest; try { bool beforeWasNotFull = false; string last = ""; DateTime lastTime = subManager.SubredditInfo.ReachedOldest ? DateTime.MinValue : DateTime.MaxValue; while (true) { List <Post> posts = null; if (!subManager.SubredditInfo.ReachedOldest) { posts = subManager.GetAfterPosts(); } else { posts = subManager.GetBeforePosts(); } if (posts == null) { return; // see what to do } if (!subManager.SubredditInfo.ReachedOldest && posts.Count == 0) { //Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the end. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW} subManager.ConfirmOldestPost(last, lastTime, true); break; } if (posts.Count == 0) { //Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the newest post. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW} subManager.ConfirmNewestPost(last, lastTime); break; } string lastPrev = last; foreach (var post in posts) { await Task.Delay(25); if (post.Created < lastTime && !subManager.SubredditInfo.ReachedOldest) { last = post.Fullname; lastTime = post.Created; } if (post.Created > lastTime && subManager.SubredditInfo.ReachedOldest) { last = post.Fullname; lastTime = post.Created; } PostManager manager = new PostManager(post, subManager.SubredditInfo, context); if (manager.IsImage()) { var imageInfos = manager.DownloadImage(Path.Combine(Program.ApplicationSetting.BasePath, "Reddit")); // TODO send path in constructor context.RedditImages.AddRange(imageInfos); context.SaveChanges(); } else { Console.ForegroundColor = ConsoleColor.Red; //Console.WriteLine($"IGNORED {post.Title} at {last}"); Console.ForegroundColor = ConsoleColor.White; } } if (lastPrev == last) { // TODO set end reached if its not yet set? //Context.Channel.SendMessageAsync($"{subManager.SubredditName} stopped because last/first did not change Count ({posts.Count})", false); // NSFW: {subManager.SubredditInfo.IsNSFW} break; } if (!subManager.SubredditInfo.ReachedOldest) { subManager.ConfirmOldestPost(last, lastTime); //Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) after {subManager.SubredditInfo.OldestPost}/{subManager.SubredditInfo.OldestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW} } else { subManager.ConfirmNewestPost(last, lastTime); //subManager.GetBeforePosts(); //Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) before {subManager.SubredditInfo.NewestPost}/{subManager.SubredditInfo.NewestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW} } } } catch (Exception ex) { await channel.SendMessageAsync($"{subManager.SubredditName} scraper died RIP {ex.Message}", false); // NSFW: {subManager.SubredditInfo.IsNSFW} } //if(!ended) //Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper ended :D", false); // NSFW: {subManager.SubredditInfo.IsNSFW} } DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, false); }