示例#1
0
        public void Scrape()
        {
            KopipastaContext db = new KopipastaContext();
            var posts = db.PostsUncreated.Where(x => x.Status == PostUnscrapedStatus.unscraped).Take(10).ToList();

            //var posts = new List<PostUnscraped> { new PostUnscraped { Id = 0, SiteId = 1, Status = PostUnscrapedStatus.unscraped }, new PostUnscraped { Id = 1, SiteId = 2, Status = PostUnscrapedStatus.unscraped } };

            List<Post> CreatedPosts = new List<Post>();

            IWebDriver driver = new ChromeDriver(@"C:\Selenium\");
            for (int i = 0; i < posts.Count(); i++)
            {
                var post = posts[i];
                driver.Navigate().GoToUrl(baseUrl + post.SiteId);

                try
                {
                    log.Info("parsing post №: " + post.SiteId);
                    WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(3));
                    string postTitle = wait.Until((d) => { return d.FindElement(By.CssSelector(titleSelector)).Text; });
                    string postBody = driver.FindElement(By.ClassName(bodySelector)).Text;

                    string postRating = ratingBase + post.SiteId;
                    int rating = int.Parse(driver.FindElement(By.Id(postRating)).Text);

                    string DateString = driver.FindElement(By.ClassName(postDate)).Text;

                    var ScrapedPost = new Post() { Body = postBody, Header = postTitle, Rating = rating, SiteId = post.SiteId, Created = DateString };
                    CreatedPosts.Add(ScrapedPost);

                    try
                    {
                        db.Posts.Add(ScrapedPost);
                        db.SaveChanges();
                        db.PostsUncreated.Remove(post);
                    }
                    catch (Exception ex)
                    {
                        log.Error("error saving post to db, post №: " + post.SiteId);
                        log.Error(ex.StackTrace);
                    }

                }
                catch (System.FormatException ex)
                {
                    log.Error("error on parsing post rating, post №: " + post.SiteId);
                    log.Error(ex.StackTrace);
                }
                catch (Exception ex)
                {
                    log.Error("error on scraping post №: " + post.SiteId);
                    log.Error(ex.StackTrace);
                }
            }
            driver.Quit();
        }
        protected void makePostBtn_Click(object sender, EventArgs e)
        {
            var context = new UsersEntities();

            using (context)
            {
                var text = this.usersPostText.Text;
                var user = context.AspNetUsers.FirstOrDefault(u => u.UserName == User.Identity.Name);
                Post post = new Post()
                {
                    Text = text
                };
                user.Posts.Add(post);

                context.SaveChanges();

                this.usersPosts.DataBind();

            }
        }
示例#3
0
        protected override void Seed(BlogContext context)
        {
            ////
            // This Seed method will be run everytime we run build command. Be careful as database
            // will be dropped and data regenerated (use this during development or testing, NOT in production)
            ////

            // We add a contributor and an administrator for being able to enter the application

            User adminUser = context.Users.Add(
                new User
            {
                Email     = "*****@*****.**",
                Password  = "******",         // Password: admin
                Role      = UserRoles.Administrator,
                Profile   = "This is the administrator for the blog",
                ImagePath = "user_1.png"
            });

            User contributorUser = context.Users.Add(
                new User
            {
                Email     = "*****@*****.**",
                Password  = "******",         // Password: contributor
                Role      = UserRoles.Contributor,
                Profile   = "This is the contributor for the blog",
                ImagePath = "user_2.png"
            }
                );

            // We add some static pages in the application
            StaticPage page1 = context.StaticPages.Add(
                new StaticPage
            {
                Title   = "First page",
                Content = GetLipsum(4)
            }
                );

            StaticPage page2 = context.StaticPages.Add(
                new StaticPage
            {
                Title   = "Second page",
                Content = GetLipsum(4)
            }
                );

            // We add some categories to start using application
            Category programmingCategory = context.Categories.Add(
                new Category {
                Name = "Programming languages"
            }
                );

            Category managementCategory = context.Categories.Add(
                new Category {
                Name = "Management techniques"
            }
                );

            Category databaseCategory = context.Categories.Add(
                new Category {
                Name = "Databases"
            }
                );

            // We add some tags to start using application
            Tag aspnetTag = context.Tags.Add(
                new Tag {
                Name = "ASP.NET"
            }
                );

            Tag nodejsTag = context.Tags.Add(
                new Tag {
                Name = "NodeJS"
            }
                );

            Tag apiTag = context.Tags.Add(
                new Tag {
                Name = "API"
            }
                );

            Tag mvc5Tag = context.Tags.Add(
                new Tag {
                Name = "MVC 5"
            }
                );

            Tag javascriptTag = context.Tags.Add(
                new Tag {
                Name = "Javascript"
            }
                );

            // We add some posts to start using application
            Post post1 = context.Posts.Add(
                new Post
            {
                Title    = "First post",
                Content  = GetLipsum(),
                User     = adminUser,
                Category = managementCategory,
                Tags     = new List <Tag>()
                {
                    aspnetTag, mvc5Tag, javascriptTag
                },
                Visible     = true,
                AddedOn     = DateTime.Today,
                ModifiedOn  = DateTime.Today,
                Attachments = new List <Attachment>()
                {
                    new Attachment()
                    {
                        Name = "First attachment",
                        Path = "Seeded_file_1.txt",
                        Size = 13
                    },
                    new Attachment()
                    {
                        Name = "Second attachment",
                        Path = "Seeded_file_2.txt",
                        Size = 13
                    }
                }
            }
                );

            PostComment comment1_1 = context.PostComments.Add(new PostComment
            {
                Content     = GetLipsum(1, false),
                AuthorEmail = "*****@*****.**",
                AddedOn     = DateTime.Today,
                Post        = post1
            });

            PostComment comment1_2 = context.PostComments.Add(new PostComment
            {
                Content = GetLipsum(1, false),
                User    = contributorUser,
                AddedOn = DateTime.Today,
                Post    = post1
            });

            PostComment comment1_3 = context.PostComments.Add(new PostComment
            {
                Content = GetLipsum(1, false),
                User    = contributorUser,
                AddedOn = DateTime.Today,
                Post    = post1
            });

            PostComment comment1_1_1 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                User              = contributorUser,
                AddedOn           = DateTime.Today,
                Post              = post1,
                ParentPostComment = comment1_1
            });

            PostComment comment1_1_2 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                AuthorEmail       = "*****@*****.**",
                AddedOn           = DateTime.Today,
                Post              = post1,
                ParentPostComment = comment1_1
            });

            PostComment comment1_3_1 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                AuthorEmail       = "*****@*****.**",
                AddedOn           = DateTime.Today,
                Post              = post1,
                ParentPostComment = comment1_3
            });

            Post post2 = context.Posts.Add(
                new Post
            {
                Title    = "Second post",
                Content  = GetLipsum(),
                User     = adminUser,
                Category = programmingCategory,
                Tags     = new List <Tag>()
                {
                    aspnetTag, mvc5Tag
                },
                Visible    = false,
                AddedOn    = DateTime.Today,
                ModifiedOn = DateTime.Today
            }
                );

            PostComment comment2_1 = context.PostComments.Add(new PostComment
            {
                Content     = GetLipsum(1, false),
                AuthorEmail = "*****@*****.**",
                AddedOn     = DateTime.Today,
                Post        = post2
            });

            PostComment comment2_2 = context.PostComments.Add(new PostComment
            {
                Content = GetLipsum(1, false),
                User    = contributorUser,
                AddedOn = DateTime.Today,
                Post    = post2
            });

            PostComment comment2_3 = context.PostComments.Add(new PostComment
            {
                Content = GetLipsum(1, false),
                User    = contributorUser,
                AddedOn = DateTime.Today,
                Post    = post2
            });

            PostComment comment2_1_1 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                User              = contributorUser,
                AddedOn           = DateTime.Today,
                Post              = post2,
                ParentPostComment = comment2_1
            });

            PostComment comment2_1_2 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                AuthorEmail       = "*****@*****.**",
                AddedOn           = DateTime.Today,
                Post              = post2,
                ParentPostComment = comment2_1
            });

            PostComment comment2_3_1 = context.PostComments.Add(new PostComment
            {
                Content           = GetLipsum(1, false),
                AuthorEmail       = "*****@*****.**",
                AddedOn           = DateTime.Today,
                Post              = post2,
                ParentPostComment = comment2_3
            });

            Post post3 = context.Posts.Add(
                new Post
            {
                Title    = "Third post",
                Content  = GetLipsum(),
                User     = contributorUser,
                Category = databaseCategory,
                Tags     = new List <Tag>()
                {
                    nodejsTag, javascriptTag, apiTag
                },
                Visible    = true,
                AddedOn    = DateTime.Today,
                ModifiedOn = DateTime.Today
            }
                );

            PostComment comment3_1 = context.PostComments.Add(new PostComment
            {
                Content     = GetLipsum(1, false),
                AuthorEmail = "*****@*****.**",
                AddedOn     = DateTime.Today,
                Post        = post3
            });

            PostComment comment3_2 = context.PostComments.Add(new PostComment
            {
                Content = GetLipsum(1, false),
                User    = contributorUser,
                AddedOn = DateTime.Today,
                Post    = post3
            });

            context.SaveChanges();

            base.Seed(context);
        }