示例#1
0
        private static async Task ScrapeNewest()
        {
            DiscordContext context = new DiscordContext();

            HttpWebRequest request =
                Helpers.CreateRequestWithHeaders(_baseUrl + "/post/index.content?tags=feet+order%3Arandom+-contentious_content+-furry+-rating%3As&page=3");

            if (request.CookieContainer == null)
            {
                request.CookieContainer = new CookieContainer();
            }

            request.CookieContainer.Add(new Cookie("login", Config.SankakuLogin, "/", _domain));
            request.CookieContainer.Add(new Cookie("pass_hash", Config.SankakuPassword, "/", _domain));

            var data = (await request.GetResponseAsync()).GetResponseStream();
            var doc  = new HtmlDocument();

            doc.Load(data);

            var postNodes = doc.DocumentNode.Descendants("span");

            ConcurrentBag <SankakuImage> sankakuImages = new ConcurrentBag <SankakuImage>();

            Parallel.ForEach(postNodes, async postNode =>
            {
                var localContext = new DiscordContext();

                var cPost = new SankakuPost(postNode);

                cPost.GetScoreCount();

                if ((await localContext.SankakuImage.FindAsync(cPost.Id)) != null)
                {
                    return;
                }

                var sankakuImage = new SankakuImage();

                sankakuImage.Id          = cPost.Id;
                sankakuImage.Score       = cPost.Score;
                sankakuImage.Rating      = (int)cPost.imgRating;
                sankakuImage.RatingCount = cPost.ScoreCount != null ? cPost.ScoreCount.Value : 0;

                foreach (var tag in cPost.Tags)
                {
                    var dbTag  = localContext.SankakuTag.FirstOrDefault(sTag => sTag.Name == tag);
                    var imgTag = new SankakuImageTag();

                    if (dbTag == null)
                    {
                        dbTag = new SankakuTag()
                        {
                            Name = tag
                        };
                        imgTag.Tag = dbTag;
                    }
                    else
                    {
                        imgTag.TagId = dbTag.Id;
                    }

                    sankakuImage.SankakuImageTag.Add(imgTag);
                }

                sankakuImages.Add(sankakuImage);
            });

            await context.SankakuImage.AddRangeAsync(sankakuImages);

            await context.SaveChangesAsync();
        }
示例#2
0
        private async Task <bool> Scrape(Order order = Order.Random, int limit = 100)
        {
            Console.WriteLine("SankakuBot: scraping");
            var context = new DiscordContext();

            string orderString = "random";

            switch (order)
            {
            case Order.Random:
                orderString = "random";
                break;

            case Order.Newest:
                orderString = "newest";
                break;
            }

            var response = await _httpClient.GetAsync(_baseUrl + _indexUrl + $"?limit={limit}&tags=order:{orderString}+feet").ConfigureAwait(false);

            if (response.StatusCode != HttpStatusCode.OK)
            {
                return(false);
            }

            var json = await response.Content.ReadAsStringAsync();

            var images = Entities.Sankaku.Image.FromJson(json);

            var allTags = context.SankakuTag.ToList();

            foreach (Entities.Sankaku.Image image in images)
            {
                if (context.SankakuImage.Find(image.Id) != null)
                {
                    continue;
                }

                SankakuImage dbImage = new SankakuImage()
                {
                    Id          = image.Id,
                    Rating      = (int)image.Rating,
                    RatingCount = Convert.ToInt32(image.VoteCount),
                    Score       = Convert.ToInt32(image.TotalScore)
                };

                foreach (var tag in image.Tags)
                {
                    var dbTag = allTags.FirstOrDefault(savedTag => savedTag.Id == Convert.ToInt32(tag.Id));

                    if (dbTag == null)
                    {
                        dbTag = new SankakuTag()
                        {
                            Name = tag.Name,
                            Id   = Convert.ToInt32(tag.Id)
                        };

                        context.SankakuTag.Add(dbTag);
                        allTags.Add(dbTag);
                    }

                    SankakuImageTag imageTag = new SankakuImageTag()
                    {
                        TagId = dbTag.Id
                    };

                    dbImage.SankakuImageTag.Add(imageTag);
                }

                await context.SankakuImage.AddAsync(dbImage).ConfigureAwait(false);
            }

            await context.SaveChangesAsync().ConfigureAwait(false);

            return(true);
        }