Exemplo n.º 1
0
        static void AnalyzeFrontPagePosts(string inRepoFileName)
        {
            ModelRepository repo        = new ModelRepository();
            List <string>   listOfPosts = new List <string>();

            Logger.Info("Opening data store: " + inRepoFileName);
            repo.OpenDataStore(inRepoFileName);

            Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
            for (int j = 0; j <= 500; j += 100)
            {
                for (int i = 85; i < 100; i++)
                {
                    Logger.InfoFormat("  DOING FRONT PAGE - {0}", j + i);
                    var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);

                    listOfPosts.AddRange(listPosts);
                }
            }

            Logger.Info("\nLIST OF POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            repo.UpdateDataStore();

            PrintStatistics(repo);
        }
Exemplo n.º 2
0
        static public void FetchPosts()
        {
            // setup - repozitoriji za spremanje podataka
            ModelRepository repo = new ModelRepository();

            repo.OpenDataStore("pollitika.db");

            // prođemo kroz naslovnicu i pokupimo sve postove (i sve komentare i dodamo sve korisnike)
            for (int j = 200; j <= 600; j++)
            {
                for (int i = 0; i < 1; i++)
                {
                    Console.WriteLine("DOING PAGE - {0}", j + i);

                    var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);

                    foreach (string postUrl in listPosts)
                    {
                        //Console.WriteLine("Post url {0}", postUrl);
                        Post newPost = PostAnalyzer.AnalyzePost("http://pollitika.com" + postUrl, repo, true, true);

                        if (newPost != null)
                        {
                            repo.AddPost(newPost);
                        }
                    }

                    repo.UpdateDataStore();
                }
            }

            // zatim prolazimo kroz sve korisnike

            // i za svakog od njih proći kroz sve njegove dnevnike

            // i za svaki dnevnik provjeriti da li je već dodan, i ak onije dodati ga
        }
Exemplo n.º 3
0
        static void CreateTestDatabase(string inFileName)
        {
            ModelRepository repo        = new ModelRepository();
            List <string>   listOfPosts = new List <string>();

            Logger.Info("Creating data store: " + inFileName);
            repo.CreateNewDataStore(inFileName);

            Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
            for (int j = 0; j <= 650; j += 700)
            {
                for (int i = 0; i < 1; i++)
                {
                    Logger.InfoFormat("  DOING FRONT PAGE - {0}", j + i);
                    var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);

                    listOfPosts.AddRange(listPosts);
                }
            }

            Logger.Info("\nLIST OF POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            // now we will fetch posts from some users
            Logger.Info("\nANALYZING USERS POSTS:");
            listOfPosts.Clear();
            List <string> usersTofetch = new List <string>
            {
                //"mrak",
                //"zoran-ostric",
                "ppetra",
                "zvone-radikalni",
                "otpisani"
            };

            foreach (string user in usersTofetch)
            {
                var list = UserPostsAnalyzer.GetListOfUserPosts(user);

                listOfPosts.AddRange(list);
            }

            Logger.Info("\nLIST OF USERS POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            repo.UpdateDataStore();

            PrintStatistics(repo);
        }