public IEnumerable <ScrapedComment> Scrape(ScrapedPost post)
{
Debug.Assert(post != null);
var comments = new List <ScrapedComment>();
DateTime now = DateTime.Now;
CommentsRequest graphRequest = new CommentsRequest(post.Id)
{
PaginationLimit = 100
};
PagedResponse <ScrapedComment> commentsResponse = GraphClient.GetComments <ScrapedComment>(graphRequest);
// Could be null if the post doesn't exist anymore.
if (commentsResponse != null)
{
foreach (ScrapedComment comment in commentsResponse.AllData())
{
if (comment.FirstScraped == DateTime.MinValue)
{
comment.FirstScraped = now;
}
comment.LastScraped = now;
comment.Post = post;
comments.Add(Save(comment, Refresh.False));
}
}
return(comments);
}
public IEnumerable <ScrapedPost> Scrape(PageMetadata[] pages, DateTime since, DateTime until)
{
Debug.Assert(pages != null);
Console.WriteLine($"Started scraping {pages.Length} pages for their posts");
DateTime start = DateTime.Now;
int numberOfPosts = 0;
for (int i = 0; i < pages.Length; i++)
{
PageMetadata page = pages[i];
Console.WriteLine($"{i + 1}/{pages.Length}: {page.Name}");
// Query the Facebook Graph API to get all posts in the given range, published only by
// the page.
var graphRequest = new PostsRequest(page.FacebookId, PostsRequestEdge.Posts)
{
Since = since,
Until = until,
PaginationLimit = 100
};
PagedResponse <ScrapedPost> postsResponse = GraphClient.GetPosts <ScrapedPost>(graphRequest);
foreach (ScrapedPost post in postsResponse.AllData())
{
UpdateMetadata(post, page.Name);
Save(post, Refresh.False);
numberOfPosts++;
yield return(post);
}
Console.WriteLine(numberOfPosts);
}
Console.WriteLine($"Done scraping {pages.Length} pages for their posts");
}
protected async Task <IEnumerable <Post> > GetAllPosts(DateTime?since)
{
Console.WriteLine("Loading posts...");
Page page = await Client.GetPage <Page>(new PageRequest("oxfess"));
PagedResponse <Post> posts = await Client.GetPosts(new PostsRequest(page.Id, PostsRequestEdge.Posts)
{
Since = since,
Fields = new RequestField[]
{
RequestField.Id,
RequestField.Message,
RequestField.Created,
RequestField.Permalink,
RequestField.Reactions.Summary(true),
RequestField.Comments.Summary(true)
},
PaginationLimit = 100
});
Post[] allPosts = posts.AllData().ToArray();
Console.WriteLine($"Loaded {allPosts.Length} posts");
return(allPosts);
}
