private static JobPost CreatePostLocal(string id, string date, string title, string company)
{
var post = new JobPost(id, date, title, company);
post.Details = BoilerPipe.ArticleExtractor.GetText(new Uri(post.Url, UriKind.Absolute));
return post;
}
private static JobPost CreatePost(string id, string date, string title, string company)
{
var post = new JobPost(id, date, title, company);
var articleUrl = string.Format(BoilerpipeUrl, post.Url);
var client = new WebClient();
using (var articleStream = client.OpenRead(articleUrl))
{
using (var reader = new StreamReader(articleStream))
{
JavaScriptSerializer serializer = new JavaScriptSerializer();
var article = serializer.DeserializeObject(reader.ReadToEnd()) as Dictionary<string, object>;
var response = (Dictionary<string, object>)article["response"];
title = response["title"].ToString();
var content = response["content"].ToString();
post.Details = content;
}
}
return post;
}
private void AddPost(JobPost post)
{
System.Windows.Application.Current.Dispatcher.BeginInvoke(new Action(() =>
{
this.posts.Add(post);
this.StatusText = String.Format("Total job posts: {0}, Failed {1}", posts.Count, posts.Count(p => p.IsFailed));
}));
}
private static string[] GetTags(JobPost post)
{
var text = post.Title + " " + post.Details;
return WordCounter.GetTags(text).ToArray();
}
private static string[] GetCategories(JobPost post)
{
var text = post.Title + " " + post.Details;
return mentions.Where(m => m.Regex.IsMatch(text)).Select(m => m.Technology).ToArray();
}
public static void AddPost(this MongoDatabase db, JobPost post)
{
var posts = db.GetCollection<JobPost>(PostsName);
posts.Save(post);
}