public void Scrape()
{
KopipastaContext db = new KopipastaContext();
var posts = db.PostsUncreated.Where(x => x.Status == PostUnscrapedStatus.unscraped).Take(10).ToList();
//var posts = new List<PostUnscraped> { new PostUnscraped { Id = 0, SiteId = 1, Status = PostUnscrapedStatus.unscraped }, new PostUnscraped { Id = 1, SiteId = 2, Status = PostUnscrapedStatus.unscraped } };
List<Post> CreatedPosts = new List<Post>();
IWebDriver driver = new ChromeDriver(@"C:\Selenium\");
for (int i = 0; i < posts.Count(); i++)
{
var post = posts[i];
driver.Navigate().GoToUrl(baseUrl + post.SiteId);
try
{
log.Info("parsing post №: " + post.SiteId);
WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(3));
string postTitle = wait.Until((d) => { return d.FindElement(By.CssSelector(titleSelector)).Text; });
string postBody = driver.FindElement(By.ClassName(bodySelector)).Text;
string postRating = ratingBase + post.SiteId;
int rating = int.Parse(driver.FindElement(By.Id(postRating)).Text);
string DateString = driver.FindElement(By.ClassName(postDate)).Text;
var ScrapedPost = new Post() { Body = postBody, Header = postTitle, Rating = rating, SiteId = post.SiteId, Created = DateString };
CreatedPosts.Add(ScrapedPost);
try
{
db.Posts.Add(ScrapedPost);
db.SaveChanges();
db.PostsUncreated.Remove(post);
}
catch (Exception ex)
{
log.Error("error saving post to db, post №: " + post.SiteId);
log.Error(ex.StackTrace);
}
}
catch (System.FormatException ex)
{
log.Error("error on parsing post rating, post №: " + post.SiteId);
log.Error(ex.StackTrace);
}
catch (Exception ex)
{
log.Error("error on scraping post №: " + post.SiteId);
log.Error(ex.StackTrace);
}
}
driver.Quit();
}
protected void makePostBtn_Click(object sender, EventArgs e)
{
var context = new UsersEntities();
using (context)
{
var text = this.usersPostText.Text;
var user = context.AspNetUsers.FirstOrDefault(u => u.UserName == User.Identity.Name);
Post post = new Post()
{
Text = text
};
user.Posts.Add(post);
context.SaveChanges();
this.usersPosts.DataBind();
}
}
protected override void Seed(BlogContext context)
{
////
// This Seed method will be run everytime we run build command. Be careful as database
// will be dropped and data regenerated (use this during development or testing, NOT in production)
////
// We add a contributor and an administrator for being able to enter the application
User adminUser = context.Users.Add(
new User
{
Email = "*****@*****.**",
Password = "******", // Password: admin
Role = UserRoles.Administrator,
Profile = "This is the administrator for the blog",
ImagePath = "user_1.png"
});
User contributorUser = context.Users.Add(
new User
{
Email = "*****@*****.**",
Password = "******", // Password: contributor
Role = UserRoles.Contributor,
Profile = "This is the contributor for the blog",
ImagePath = "user_2.png"
}
);
// We add some static pages in the application
StaticPage page1 = context.StaticPages.Add(
new StaticPage
{
Title = "First page",
Content = GetLipsum(4)
}
);
StaticPage page2 = context.StaticPages.Add(
new StaticPage
{
Title = "Second page",
Content = GetLipsum(4)
}
);
// We add some categories to start using application
Category programmingCategory = context.Categories.Add(
new Category {
Name = "Programming languages"
}
);
Category managementCategory = context.Categories.Add(
new Category {
Name = "Management techniques"
}
);
Category databaseCategory = context.Categories.Add(
new Category {
Name = "Databases"
}
);
// We add some tags to start using application
Tag aspnetTag = context.Tags.Add(
new Tag {
Name = "ASP.NET"
}
);
Tag nodejsTag = context.Tags.Add(
new Tag {
Name = "NodeJS"
}
);
Tag apiTag = context.Tags.Add(
new Tag {
Name = "API"
}
);
Tag mvc5Tag = context.Tags.Add(
new Tag {
Name = "MVC 5"
}
);
Tag javascriptTag = context.Tags.Add(
new Tag {
Name = "Javascript"
}
);
// We add some posts to start using application
Post post1 = context.Posts.Add(
new Post
{
Title = "First post",
Content = GetLipsum(),
User = adminUser,
Category = managementCategory,
Tags = new List <Tag>()
{
aspnetTag, mvc5Tag, javascriptTag
},
Visible = true,
AddedOn = DateTime.Today,
ModifiedOn = DateTime.Today,
Attachments = new List <Attachment>()
{
new Attachment()
{
Name = "First attachment",
Path = "Seeded_file_1.txt",
Size = 13
},
new Attachment()
{
Name = "Second attachment",
Path = "Seeded_file_2.txt",
Size = 13
}
}
}
);
PostComment comment1_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post1
});
PostComment comment1_2 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post1
});
PostComment comment1_3 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post1
});
PostComment comment1_1_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post1,
ParentPostComment = comment1_1
});
PostComment comment1_1_2 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post1,
ParentPostComment = comment1_1
});
PostComment comment1_3_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post1,
ParentPostComment = comment1_3
});
Post post2 = context.Posts.Add(
new Post
{
Title = "Second post",
Content = GetLipsum(),
User = adminUser,
Category = programmingCategory,
Tags = new List <Tag>()
{
aspnetTag, mvc5Tag
},
Visible = false,
AddedOn = DateTime.Today,
ModifiedOn = DateTime.Today
}
);
PostComment comment2_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post2
});
PostComment comment2_2 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post2
});
PostComment comment2_3 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post2
});
PostComment comment2_1_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post2,
ParentPostComment = comment2_1
});
PostComment comment2_1_2 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post2,
ParentPostComment = comment2_1
});
PostComment comment2_3_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post2,
ParentPostComment = comment2_3
});
Post post3 = context.Posts.Add(
new Post
{
Title = "Third post",
Content = GetLipsum(),
User = contributorUser,
Category = databaseCategory,
Tags = new List <Tag>()
{
nodejsTag, javascriptTag, apiTag
},
Visible = true,
AddedOn = DateTime.Today,
ModifiedOn = DateTime.Today
}
);
PostComment comment3_1 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
AuthorEmail = "*****@*****.**",
AddedOn = DateTime.Today,
Post = post3
});
PostComment comment3_2 = context.PostComments.Add(new PostComment
{
Content = GetLipsum(1, false),
User = contributorUser,
AddedOn = DateTime.Today,
Post = post3
});
context.SaveChanges();
base.Seed(context);
}
