public static async Task ScrapReddit(string subredditName, ISocketMessageChannel channel)
{
DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, true);
var reddit = new RedditClient(Program.ApplicationSetting.RedditSetting.AppId, Program.ApplicationSetting.RedditSetting.RefreshToken, Program.ApplicationSetting.RedditSetting.AppSecret);
using (var context = new ETHBotDBContext())
{
SubredditManager subManager = new SubredditManager(subredditName, reddit, context);
bool ended = subManager.SubredditInfo.ReachedOldest;
try
{
bool beforeWasNotFull = false;
string last = "";
DateTime lastTime = subManager.SubredditInfo.ReachedOldest ? DateTime.MinValue : DateTime.MaxValue;
while (true)
{
List <Post> posts = null;
if (!subManager.SubredditInfo.ReachedOldest)
{
posts = subManager.GetAfterPosts();
}
else
{
posts = subManager.GetBeforePosts();
}
if (posts == null)
{
return; // see what to do
}
if (!subManager.SubredditInfo.ReachedOldest && posts.Count == 0)
{
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the end. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
subManager.ConfirmOldestPost(last, lastTime, true);
break;
}
if (posts.Count == 0)
{
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper reached the newest post. Setting up end flags", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
subManager.ConfirmNewestPost(last, lastTime);
break;
}
string lastPrev = last;
foreach (var post in posts)
{
await Task.Delay(25);
if (post.Created < lastTime && !subManager.SubredditInfo.ReachedOldest)
{
last = post.Fullname;
lastTime = post.Created;
}
if (post.Created > lastTime && subManager.SubredditInfo.ReachedOldest)
{
last = post.Fullname;
lastTime = post.Created;
}
PostManager manager = new PostManager(post, subManager.SubredditInfo, context);
if (manager.IsImage())
{
var imageInfos = manager.DownloadImage(Path.Combine(Program.ApplicationSetting.BasePath, "Reddit")); // TODO send path in constructor
context.RedditImages.AddRange(imageInfos);
context.SaveChanges();
}
else
{
Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine($"IGNORED {post.Title} at {last}");
Console.ForegroundColor = ConsoleColor.White;
}
}
if (lastPrev == last)
{
// TODO set end reached if its not yet set?
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} stopped because last/first did not change Count ({posts.Count})", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
break;
}
if (!subManager.SubredditInfo.ReachedOldest)
{
subManager.ConfirmOldestPost(last, lastTime);
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) after {subManager.SubredditInfo.OldestPost}/{subManager.SubredditInfo.OldestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
else
{
subManager.ConfirmNewestPost(last, lastTime);
//subManager.GetBeforePosts();
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper is happy and well :) Count ({posts.Count}) before {subManager.SubredditInfo.NewestPost}/{subManager.SubredditInfo.NewestPostDate}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
}
}
catch (Exception ex)
{
await channel.SendMessageAsync($"{subManager.SubredditName} scraper died RIP {ex.Message}", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
//if(!ended)
//Context.Channel.SendMessageAsync($"{subManager.SubredditName} scraper ended :D", false); // NSFW: {subManager.SubredditInfo.IsNSFW}
}
DatabaseManager.Instance().SetSubredditScaperStatus(subredditName, false);
}
public override Task DoWork(CancellationToken cancellationToken)
{
try
{
if (Program.Client == null)
{
return(Task.CompletedTask);
}
_logger.LogInformation($"{DateTime.Now:hh:mm:ss} {Name} is working.");
var guild = Program.Client.GetGuild(GuildId);
var textChannel = guild.GetTextChannel(ChannelId);
var dbManager = DatabaseManager.Instance();
var settings = dbManager.GetBotSettings(); // could prob use the object from program cs
// TODO DEV Only
//settings.LastSpaceXRedditPost = "t3_m2x82i";
if (settings?.LastSpaceXRedditPost == null)
{
_logger.LogInformation($"{DateTime.Now:hh:mm:ss} No SpaceX subreddit info defined. Abort.");
return(Task.CompletedTask);
}
string subredditName = "spacex"; // spacex
var reddit = new RedditClient(Program.RedditAppId, Program.RedditRefreshToken, Program.RedditAppSecret);
SubredditManager subManager = new SubredditManager(subredditName, reddit, null, settings.LastSpaceXRedditPost, null);
// TODO maybe use MonitorNew ?
var posts = subManager.GetBeforePosts();
if (posts == null)
{
return(Task.CompletedTask);
}
if (posts.Count > 0)
{
foreach (var redditPost in posts)
{
EmbedBuilder builder = new EmbedBuilder();
var title = redditPost.Title;
builder.WithTitle(title.Substring(0, Math.Min(256, title.Length)));
builder.WithUrl("https://www.reddit.com/" + redditPost.Permalink);
var content = redditPost.Listing.IsSelf ? redditPost.Listing.SelfText : "";
if (content.Length > 2000)
{
content = content.Substring(0, 2000);
}
// TODO if subreddit name null get the subreddit
builder.WithDescription(content);
builder.WithColor(0, 0, 255);
//builder.WithThumbnailUrl("https://cdn.discordapp.com/avatars/774276700557148170/62279315dd469126ca4e5ab89a5e802a.png");
builder.WithCurrentTimestamp();
string url = redditPost.Listing.URL;
if (url.Contains("v.redd.it"))
{
// TODO Handle video case this doesnt work in embed yet
url += "/DASH_720.mp4";
}
builder.WithImageUrl(url);
builder.AddField("Infos", $"Posted by: {redditPost.Author} in /r/{subredditName} at {redditPost.Created}");
textChannel.SendMessageAsync("", false, builder.Build());
}
settings.LastSpaceXRedditPost = posts.OrderByDescending(i => i.Created).First().Fullname; // newest is on top
dbManager.UpdateBotSettings(settings);
}
}
catch (Exception ex)
{
}
return(Task.CompletedTask);
}
