public static async Task PostPrediction(long userId, DiscordChannel channel) { var context = new DiscordContext(); SankakuImage imgToPost = null; Random rnd = new Random(); var channelIDLong = Convert.ToInt64(channel.Id); var blacklistedTags = await context.SankakuTagBlacklist.Where(blackTag => blackTag.ChannelId == channelIDLong).Select(tag => tag.TagId).ToListAsync(); var startDate = DateTime.Now - TimeSpan.FromDays(90); var trainedModel = Train(userId); int skip = 0; while (imgToPost == null) { //imgToPost = context.SankakuImage.Where(si => si.SankakuPost.Count == 0).Skip(skip).First(); imgToPost = context.SankakuImage.Find(88415l); if (imgToPost == null) { await ScrapeNewest(); } List <MLSankakuPost> posts = new List <MLSankakuPost>(); MLSankakuPost mlPost = new MLSankakuPost() { //imgRating = imgToPost.Rating, //score = Convert.ToSingle(imgToPost.Score), //Tags = context.SankakuImageTag.Where(imageTag => imageTag.ImageId == imgToPost.Id).Select(tag => Convert.ToSingle(tag.TagId)).ToArray(), Tags = 1, }; posts.Add(mlPost); var prediction = trainedModel.Predict(posts, true); if (prediction.First().Score < 1) { skip++; imgToPost = null; } } DiscordEmbedBuilder builder = new DiscordEmbedBuilder() { Color = DiscordColor.Orange, Title = context.Puns.Skip(new Random().Next(context.Puns.Count() - 1)).First().Text }; var link = HttpUtility.HtmlDecode(await GetOriginalImageLink(imgToPost.Id)); var request = Helpers.CreateRequestWithHeaders(link); var response = request.GetResponse(); SKImage img = SKImage.FromEncodedData(response.GetResponseStream()); if (img.ColorType == SKColorType.Bgra8888) { //Bgra8888 needs an indexed Colortable, so convert it to Rgba8888 first var bitmap = new SKBitmap(new SKImageInfo(img.Width, img.Height, SKColorType.Rgba8888)); var canvas = new SKCanvas(bitmap); canvas.DrawImage(img, 0, 0, null); canvas.Flush(); img = SKImage.FromBitmap(bitmap); } var imageData = img.Encode(SKEncodedImageFormat.Jpeg, 75); //Encode in Jpeg instead of Webp because of IOS using (var outStream = imageData.AsStream()) { var msg = await channel.SendFileAsync(outStream, builder.Title + ".jpeg", embed : builder.Build()); context.SankakuPost.Add(new Models.SankakuPost() { Date = DateTime.Now, Image = imgToPost, MessageId = Convert.ToInt64(msg.Id) }); await context.SaveChangesAsync(); } }
private static async Task ScrapeNewest() { DiscordContext context = new DiscordContext(); HttpWebRequest request = Helpers.CreateRequestWithHeaders(_baseUrl + "/post/index.content?tags=feet+order%3Arandom+-contentious_content+-furry+-rating%3As&page=3"); if (request.CookieContainer == null) { request.CookieContainer = new CookieContainer(); } request.CookieContainer.Add(new Cookie("login", Config.SankakuLogin, "/", _domain)); request.CookieContainer.Add(new Cookie("pass_hash", Config.SankakuPassword, "/", _domain)); var data = (await request.GetResponseAsync()).GetResponseStream(); var doc = new HtmlDocument(); doc.Load(data); var postNodes = doc.DocumentNode.Descendants("span"); ConcurrentBag <SankakuImage> sankakuImages = new ConcurrentBag <SankakuImage>(); Parallel.ForEach(postNodes, async postNode => { var localContext = new DiscordContext(); var cPost = new SankakuPost(postNode); cPost.GetScoreCount(); if ((await localContext.SankakuImage.FindAsync(cPost.Id)) != null) { return; } var sankakuImage = new SankakuImage(); sankakuImage.Id = cPost.Id; sankakuImage.Score = cPost.Score; sankakuImage.Rating = (int)cPost.imgRating; sankakuImage.RatingCount = cPost.ScoreCount != null ? cPost.ScoreCount.Value : 0; foreach (var tag in cPost.Tags) { var dbTag = localContext.SankakuTag.FirstOrDefault(sTag => sTag.Name == tag); var imgTag = new SankakuImageTag(); if (dbTag == null) { dbTag = new SankakuTag() { Name = tag }; imgTag.Tag = dbTag; } else { imgTag.TagId = dbTag.Id; } sankakuImage.SankakuImageTag.Add(imgTag); } sankakuImages.Add(sankakuImage); }); await context.SankakuImage.AddRangeAsync(sankakuImages); await context.SaveChangesAsync(); }
public static async Task PostRandom(DiscordChannel channel, int skip = 0) { var context = new DiscordContext(); SankakuImage imgToPost = null; Random rnd = new Random(skip); var channelIDLong = Convert.ToInt64(channel.Id); var blacklistedTags = await context.SankakuTagBlacklist.Where(blackTag => blackTag.ChannelId == channelIDLong).Select(tag => tag.TagId).ToListAsync(); var startDate = DateTime.Now - TimeSpan.FromDays(90); var viableImages = context.SankakuImage.Where(si => si.SankakuPost.Count == 0 && si.Score > 4 && si.RatingCount > 6); while (imgToPost == null) { var firstNotPostedPicture = viableImages.Where(si => !si.SankakuImageTag.Any(tag => blacklistedTags.Contains(tag.TagId))) .Skip(skip).First(); if (firstNotPostedPicture == null) { var oldestPostedPicture = viableImages.Where(si => si.SankakuPost.OrderByDescending(post => post.Date).First().Date < startDate) .Skip(skip); } else { imgToPost = firstNotPostedPicture; } if (imgToPost == null) { await ScrapeNewest(); } } DiscordEmbedBuilder builder = new DiscordEmbedBuilder() { Color = DiscordColor.SpringGreen, Title = context.Puns.Skip(rnd.Next(context.Puns.Count() - 1)).First().Text }; var link = HttpUtility.HtmlDecode(await GetOriginalImageLink(imgToPost.Id)); var request = Helpers.CreateRequestWithHeaders(link); var response = request.GetResponse(); SKImage img = SKImage.FromEncodedData(response.GetResponseStream()); if (img.ColorType == SKColorType.Bgra8888) { //Bgra8888 needs an indexed Colortable, so convert it to Rgba8888 first var bitmap = new SKBitmap(new SKImageInfo(img.Width, img.Height, SKColorType.Rgba8888)); var canvas = new SKCanvas(bitmap); canvas.DrawImage(img, 0, 0, null); canvas.Flush(); img = SKImage.FromBitmap(bitmap); } var imageData = img.Encode(SKEncodedImageFormat.Webp, 70); using (var outStream = imageData.AsStream()) { var msg = await channel.SendFileAsync(outStream, builder.Title + ".webp", embed : builder.Build()); context.SankakuPost.Add(new Models.SankakuPost() { Date = DateTime.Now, Image = imgToPost, MessageId = Convert.ToInt64(msg.Id) }); await context.SaveChangesAsync(); } }
private async Task <bool> Scrape(Order order = Order.Random, int limit = 100) { Console.WriteLine("SankakuBot: scraping"); var context = new DiscordContext(); string orderString = "random"; switch (order) { case Order.Random: orderString = "random"; break; case Order.Newest: orderString = "newest"; break; } var response = await _httpClient.GetAsync(_baseUrl + _indexUrl + $"?limit={limit}&tags=order:{orderString}+feet").ConfigureAwait(false); if (response.StatusCode != HttpStatusCode.OK) { return(false); } var json = await response.Content.ReadAsStringAsync(); var images = Entities.Sankaku.Image.FromJson(json); var allTags = context.SankakuTag.ToList(); foreach (Entities.Sankaku.Image image in images) { if (context.SankakuImage.Find(image.Id) != null) { continue; } SankakuImage dbImage = new SankakuImage() { Id = image.Id, Rating = (int)image.Rating, RatingCount = Convert.ToInt32(image.VoteCount), Score = Convert.ToInt32(image.TotalScore) }; foreach (var tag in image.Tags) { var dbTag = allTags.FirstOrDefault(savedTag => savedTag.Id == Convert.ToInt32(tag.Id)); if (dbTag == null) { dbTag = new SankakuTag() { Name = tag.Name, Id = Convert.ToInt32(tag.Id) }; context.SankakuTag.Add(dbTag); allTags.Add(dbTag); } SankakuImageTag imageTag = new SankakuImageTag() { TagId = dbTag.Id }; dbImage.SankakuImageTag.Add(imageTag); } await context.SankakuImage.AddAsync(dbImage).ConfigureAwait(false); } await context.SaveChangesAsync().ConfigureAwait(false); return(true); }