private static async Task ScrapeNewest()
{
DiscordContext context = new DiscordContext();
HttpWebRequest request =
Helpers.CreateRequestWithHeaders(_baseUrl + "/post/index.content?tags=feet+order%3Arandom+-contentious_content+-furry+-rating%3As&page=3");
if (request.CookieContainer == null)
{
request.CookieContainer = new CookieContainer();
}
request.CookieContainer.Add(new Cookie("login", Config.SankakuLogin, "/", _domain));
request.CookieContainer.Add(new Cookie("pass_hash", Config.SankakuPassword, "/", _domain));
var data = (await request.GetResponseAsync()).GetResponseStream();
var doc = new HtmlDocument();
doc.Load(data);
var postNodes = doc.DocumentNode.Descendants("span");
ConcurrentBag <SankakuImage> sankakuImages = new ConcurrentBag <SankakuImage>();
Parallel.ForEach(postNodes, async postNode =>
{
var localContext = new DiscordContext();
var cPost = new SankakuPost(postNode);
cPost.GetScoreCount();
if ((await localContext.SankakuImage.FindAsync(cPost.Id)) != null)
{
return;
}
var sankakuImage = new SankakuImage();
sankakuImage.Id = cPost.Id;
sankakuImage.Score = cPost.Score;
sankakuImage.Rating = (int)cPost.imgRating;
sankakuImage.RatingCount = cPost.ScoreCount != null ? cPost.ScoreCount.Value : 0;
foreach (var tag in cPost.Tags)
{
var dbTag = localContext.SankakuTag.FirstOrDefault(sTag => sTag.Name == tag);
var imgTag = new SankakuImageTag();
if (dbTag == null)
{
dbTag = new SankakuTag()
{
Name = tag
};
imgTag.Tag = dbTag;
}
else
{
imgTag.TagId = dbTag.Id;
}
sankakuImage.SankakuImageTag.Add(imgTag);
}
sankakuImages.Add(sankakuImage);
});
await context.SankakuImage.AddRangeAsync(sankakuImages);
await context.SaveChangesAsync();
}
private async Task <bool> Scrape(Order order = Order.Random, int limit = 100)
{
Console.WriteLine("SankakuBot: scraping");
var context = new DiscordContext();
string orderString = "random";
switch (order)
{
case Order.Random:
orderString = "random";
break;
case Order.Newest:
orderString = "newest";
break;
}
var response = await _httpClient.GetAsync(_baseUrl + _indexUrl + $"?limit={limit}&tags=order:{orderString}+feet").ConfigureAwait(false);
if (response.StatusCode != HttpStatusCode.OK)
{
return(false);
}
var json = await response.Content.ReadAsStringAsync();
var images = Entities.Sankaku.Image.FromJson(json);
var allTags = context.SankakuTag.ToList();
foreach (Entities.Sankaku.Image image in images)
{
if (context.SankakuImage.Find(image.Id) != null)
{
continue;
}
SankakuImage dbImage = new SankakuImage()
{
Id = image.Id,
Rating = (int)image.Rating,
RatingCount = Convert.ToInt32(image.VoteCount),
Score = Convert.ToInt32(image.TotalScore)
};
foreach (var tag in image.Tags)
{
var dbTag = allTags.FirstOrDefault(savedTag => savedTag.Id == Convert.ToInt32(tag.Id));
if (dbTag == null)
{
dbTag = new SankakuTag()
{
Name = tag.Name,
Id = Convert.ToInt32(tag.Id)
};
context.SankakuTag.Add(dbTag);
allTags.Add(dbTag);
}
SankakuImageTag imageTag = new SankakuImageTag()
{
TagId = dbTag.Id
};
dbImage.SankakuImageTag.Add(imageTag);
}
await context.SankakuImage.AddAsync(dbImage).ConfigureAwait(false);
}
await context.SaveChangesAsync().ConfigureAwait(false);
return(true);
}