public IActionResult ExportAsCSV([FromBody] ElasticSearchRequest request)
{
IEnumerable <ScrapedPost> history = PostScraper.All(request.Query, request.Sort).Data;
byte[] serialized = CsvSerialization.Serialize(history, CsvSerialization.MapPost);
return(File(serialized, "text/csv", "export.csv"));
}
public IActionResult ExportAsJson([FromBody] ElasticSearchRequest request)
{
IEnumerable <ScrapedPost> history = PostScraper.All(request.Query, request.Sort).Data;
byte[] serialized = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(history));
return(File(serialized, "application/json-download", "export.json"));
}
public void GoThroughEachPostAndGetTheCommentsOhMyGodThisWillDestroyMyLaptop()
{
const int LastScrapeAmount = 0;
int i = 0;
AllResponse <ScrapedPost> posts = PostScraper.All(new SortField[] { new SortField {
Field = "created_time", Order = SortOrder.Descending
} });
foreach (ScrapedPost post in posts.Data)
{
i++;
if (post.CreatedTime < new DateTime(2017, 04, 01))
{
continue;
}
if (i > LastScrapeAmount)
{
List <ScrapedComment> comments = CommentScraper.Scrape(post).ToList();
Console.WriteLine($"{i}/{posts.TotalCount}: {post.Id}; {comments.Count}");
}
else
{
Console.WriteLine($"{i}/{posts.TotalCount}: {post.Id}; Already scraped.");
}
}
}
