Ejemplo n.º 1
0
        static void AnalyzeFrontPagePosts(string inRepoFileName)
        {
            ModelRepository repo        = new ModelRepository();
            List <string>   listOfPosts = new List <string>();

            Logger.Info("Opening data store: " + inRepoFileName);
            repo.OpenDataStore(inRepoFileName);

            Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
            for (int j = 0; j <= 500; j += 100)
            {
                for (int i = 85; i < 100; i++)
                {
                    Logger.InfoFormat("  DOING FRONT PAGE - {0}", j + i);
                    var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);

                    listOfPosts.AddRange(listPosts);
                }
            }

            Logger.Info("\nLIST OF POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            repo.UpdateDataStore();

            PrintStatistics(repo);
        }
Ejemplo n.º 2
0
        static void AnalyzePostsFromList(ModelRepository repo, List <string> inListPosts, bool isFrontPage)
        {
            Logger.Info("\nLIST OF POSTS TO ANALYZE:");
            for (int i = 0; i < inListPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + inListPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(inListPosts, repo, isFrontPage, true);

            repo.UpdateDataStore();

            PrintStatistics(repo);
        }
Ejemplo n.º 3
0
        static void CreateTestDatabase(string inFileName)
        {
            ModelRepository repo        = new ModelRepository();
            List <string>   listOfPosts = new List <string>();

            Logger.Info("Creating data store: " + inFileName);
            repo.CreateNewDataStore(inFileName);

            Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
            for (int j = 0; j <= 650; j += 700)
            {
                for (int i = 0; i < 1; i++)
                {
                    Logger.InfoFormat("  DOING FRONT PAGE - {0}", j + i);
                    var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);

                    listOfPosts.AddRange(listPosts);
                }
            }

            Logger.Info("\nLIST OF POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            // now we will fetch posts from some users
            Logger.Info("\nANALYZING USERS POSTS:");
            listOfPosts.Clear();
            List <string> usersTofetch = new List <string>
            {
                //"mrak",
                //"zoran-ostric",
                "ppetra",
                "zvone-radikalni",
                "otpisani"
            };

            foreach (string user in usersTofetch)
            {
                var list = UserPostsAnalyzer.GetListOfUserPosts(user);

                listOfPosts.AddRange(list);
            }

            Logger.Info("\nLIST OF USERS POSTS TO ANALYZE:");
            for (int i = 0; i < listOfPosts.Count; i++)
            {
                Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
            }

            Logger.Info("\nSTARTING ANALYSIS:");
            MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);

            repo.UpdateDataStore();

            PrintStatistics(repo);
        }