private static void GetScrapees(Facebook fb)
{
log.Info("Verifying scrapees");
using (var db = new FacebookDebatEntities())
{
var scrapees = db.Scrapees.Where(x => x.entity_id == null && x.enabled).ToList();
log.Info("Found " + scrapees.Count + " new entities.");
foreach (var scrapee in scrapees)
{
try
{
var fb_entity = fb.GetUser(scrapee.fb_id);
var entity = new Entity
{
fb_id = fb_entity.fb_id,
name = fb_entity.name
};
db.Entities.Add(entity);
scrapee.Entity = entity;
scrapee.name = fb_entity.name;
db.Entry(scrapee).State = System.Data.Entity.EntityState.Modified;
}
catch (Exception) { }
}
log.Info("Done");
db.SaveChanges();
}
}
private static void GetPosts(Facebook fb, int lookbackDays, DateTime dateFrom)
{
using (var db = new FacebookDebatEntities())
{
db.Configuration.AutoDetectChangesEnabled = false;
var scrapees = db.Scrapees.Where(x => x.enabled).Select(x => new { name = x.name, entity = x.Entity }).ToList();
log.Info("Getting posts for " + scrapees.Count + " scrapees.");
// Get posts for each page
Parallel.ForEach(scrapees, new ParallelOptions {
MaxDegreeOfParallelism = 15
}, (scrapee) =>
{
if (scrapee.entity == null)
{
return;
}
try
{
var fb_posts = fb.GetPosts(scrapee.entity.fb_id, lookbackDays, dateFrom);
foreach (var fb_post in fb_posts)
{
Post post;
lock (db)
post = db.Posts.SingleOrDefault(x => x.fb_id == fb_post.id);
if (post == null)
{
post = new Post()
{
fb_id = fb_post.id,
message = fb_post.message,
Entity = scrapee.entity,
date = fb_post.date,
shares = fb_post.shares
};
lock (db)
db.Posts.Add(post);
}
else
{
lock (db)
{
if (post.message != fb_post.message || post.shares != fb_post.shares)
{
post.message = fb_post.message;
post.shares = fb_post.shares;
db.Entry(post).State = System.Data.Entity.EntityState.Modified;
}
}
}
}
lock (db)
db.SaveChanges();
}
catch (Exception e)
{
log.Error("Problem with " + scrapee.name);
log.Error(e.ToString());
}
});
log.Info("Done.");
}
}