public IEnumerable <ScrapedComment> Scrape(ScrapedPost post) { Debug.Assert(post != null); var comments = new List <ScrapedComment>(); DateTime now = DateTime.Now; CommentsRequest graphRequest = new CommentsRequest(post.Id) { PaginationLimit = 100 }; PagedResponse <ScrapedComment> commentsResponse = GraphClient.GetComments <ScrapedComment>(graphRequest); // Could be null if the post doesn't exist anymore. if (commentsResponse != null) { foreach (ScrapedComment comment in commentsResponse.AllData()) { if (comment.FirstScraped == DateTime.MinValue) { comment.FirstScraped = now; } comment.LastScraped = now; comment.Post = post; comments.Add(Save(comment, Refresh.False)); } } return(comments); }
public IEnumerable <ScrapedPost> Scrape(PageMetadata[] pages, DateTime since, DateTime until) { Debug.Assert(pages != null); Console.WriteLine($"Started scraping {pages.Length} pages for their posts"); DateTime start = DateTime.Now; int numberOfPosts = 0; for (int i = 0; i < pages.Length; i++) { PageMetadata page = pages[i]; Console.WriteLine($"{i + 1}/{pages.Length}: {page.Name}"); // Query the Facebook Graph API to get all posts in the given range, published only by // the page. var graphRequest = new PostsRequest(page.FacebookId, PostsRequestEdge.Posts) { Since = since, Until = until, PaginationLimit = 100 }; PagedResponse <ScrapedPost> postsResponse = GraphClient.GetPosts <ScrapedPost>(graphRequest); foreach (ScrapedPost post in postsResponse.AllData()) { UpdateMetadata(post, page.Name); Save(post, Refresh.False); numberOfPosts++; yield return(post); } Console.WriteLine(numberOfPosts); } Console.WriteLine($"Done scraping {pages.Length} pages for their posts"); }
protected async Task <IEnumerable <Post> > GetAllPosts(DateTime?since) { Console.WriteLine("Loading posts..."); Page page = await Client.GetPage <Page>(new PageRequest("oxfess")); PagedResponse <Post> posts = await Client.GetPosts(new PostsRequest(page.Id, PostsRequestEdge.Posts) { Since = since, Fields = new RequestField[] { RequestField.Id, RequestField.Message, RequestField.Created, RequestField.Permalink, RequestField.Reactions.Summary(true), RequestField.Comments.Summary(true) }, PaginationLimit = 100 }); Post[] allPosts = posts.AllData().ToArray(); Console.WriteLine($"Loaded {allPosts.Length} posts"); return(allPosts); }