/// <summary> /// Fetch latest posts /// </summary> /// <returns>List or null in case of errors</returns> internal async Task <List <Post> > fetchPosts() { // fetch posts (amount of limit set) HashSet <RedditPost> rpList; try { RedditPostPage rpp = await RedditScraper.ParsePosts(FUNNY_URL + "&limit=" + POST_FETCH_LIMIT); rpList = rpp.Posts; } catch (WebException we) { Debug.WriteLine(we.Message); return(null); } // convert reddit posts into funnyposts ConcurrentStack <Post> fpStack = new ConcurrentStack <Post>(); int dclBackup = ServicePointManager.DefaultConnectionLimit; ServicePointManager.DefaultConnectionLimit = MAX_TASKS; Parallel.ForEach(rpList, new ParallelOptions { MaxDegreeOfParallelism = MAX_TASKS }, post => { Post fp = createFunnyPost(post).Result; if (fp != null) { fpStack.Push(fp); } }); ServicePointManager.DefaultConnectionLimit = dclBackup; return(new List <Post>(fpStack)); }
/// <summary> /// Create funny post from reddit post /// </summary> /// <param name="rp"></param> /// <returns>Post or null in case of error</returns> internal static async Task <Post> createFunnyPost(RedditPost rp) { // funny post Post fp = new Post { Title = rp.Title, RedditId = rp.Id }; // timestamp fp.Created = UNIX_START.AddSeconds(rp.Created_Utc); // get direct link to picture fp.ImageUrl = await createPictureLink(rp.Url); if (fp.ImageUrl == null) { // no picture => no post return(null); } // fetch comments try { List <RedditComment> comments = await RedditScraper.ParseComments(fp.RedditId); // take 3 first comments to be added into db foreach (RedditComment comment in comments.Take(3)) { fp.Comments.Add(new Comment { Text = comment.Body }); } } catch (WebException we) { // comments are not crucial, so we just forget them in case of error Debug.WriteLine(we.Message); } return(fp); }
public void ParseComments() { List <RedditComment> lrc = RedditScraper.ParseComments("31ho69").Result; //List<RedditComment> lrc = RedditScraper.ParseComments("31ixrv").Result; //List<RedditComment> lrc = RedditScraper.ParseComments("31iw5k").Result; }
public void ParsePostsInvalidUri() { RedditPostPage rpp = RedditScraper.ParsePosts("http://127.0.0.1.json").Result; }
public void ParsePosts() { RedditPostPage rpp = RedditScraper.ParsePosts("http://www.reddit.com/r/csharp.json").Result; Assert.IsFalse(rpp.Posts.Count == 0); }
public CronScrapAndRotateMemeImages(IMemeImageRepository memeImageRepository, RedditScraper scraper) { _memeImageRepository = memeImageRepository; _scraper = scraper; }