/// <summary>
/// Fetch latest posts
/// </summary>
/// <returns>List or null in case of errors</returns>
internal async Task <List <Post> > fetchPosts()
{
// fetch posts (amount of limit set)
HashSet <RedditPost> rpList;
try
{
RedditPostPage rpp = await RedditScraper.ParsePosts(FUNNY_URL + "&limit=" + POST_FETCH_LIMIT);
rpList = rpp.Posts;
}
catch (WebException we)
{
Debug.WriteLine(we.Message);
return(null);
}
// convert reddit posts into funnyposts
ConcurrentStack <Post> fpStack = new ConcurrentStack <Post>();
int dclBackup = ServicePointManager.DefaultConnectionLimit;
ServicePointManager.DefaultConnectionLimit = MAX_TASKS;
Parallel.ForEach(rpList, new ParallelOptions {
MaxDegreeOfParallelism = MAX_TASKS
}, post =>
{
Post fp = createFunnyPost(post).Result;
if (fp != null)
{
fpStack.Push(fp);
}
});
ServicePointManager.DefaultConnectionLimit = dclBackup;
return(new List <Post>(fpStack));
}
/// <summary>
/// Create funny post from reddit post
/// </summary>
/// <param name="rp"></param>
/// <returns>Post or null in case of error</returns>
internal static async Task <Post> createFunnyPost(RedditPost rp)
{
// funny post
Post fp = new Post {
Title = rp.Title, RedditId = rp.Id
};
// timestamp
fp.Created = UNIX_START.AddSeconds(rp.Created_Utc);
// get direct link to picture
fp.ImageUrl = await createPictureLink(rp.Url);
if (fp.ImageUrl == null)
{
// no picture => no post
return(null);
}
// fetch comments
try
{
List <RedditComment> comments = await RedditScraper.ParseComments(fp.RedditId);
// take 3 first comments to be added into db
foreach (RedditComment comment in comments.Take(3))
{
fp.Comments.Add(new Comment {
Text = comment.Body
});
}
}
catch (WebException we)
{
// comments are not crucial, so we just forget them in case of error
Debug.WriteLine(we.Message);
}
return(fp);
}
public void ParseComments()
{
List <RedditComment> lrc = RedditScraper.ParseComments("31ho69").Result;
//List<RedditComment> lrc = RedditScraper.ParseComments("31ixrv").Result;
//List<RedditComment> lrc = RedditScraper.ParseComments("31iw5k").Result;
}
public void ParsePostsInvalidUri()
{
RedditPostPage rpp = RedditScraper.ParsePosts("http://127.0.0.1.json").Result;
}
public void ParsePosts()
{
RedditPostPage rpp = RedditScraper.ParsePosts("http://www.reddit.com/r/csharp.json").Result;
Assert.IsFalse(rpp.Posts.Count == 0);
}
public CronScrapAndRotateMemeImages(IMemeImageRepository memeImageRepository, RedditScraper scraper)
{
_memeImageRepository = memeImageRepository;
_scraper = scraper;
}
