static void AnalyzeFrontPagePosts(string inRepoFileName)
{
ModelRepository repo = new ModelRepository();
List <string> listOfPosts = new List <string>();
Logger.Info("Opening data store: " + inRepoFileName);
repo.OpenDataStore(inRepoFileName);
Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
for (int j = 0; j <= 500; j += 100)
{
for (int i = 85; i < 100; i++)
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
listOfPosts.AddRange(listPosts);
}
}
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
repo.UpdateDataStore();
PrintStatistics(repo);
}
static void AnalyzePostsFromList(ModelRepository repo, List <string> inListPosts, bool isFrontPage)
{
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < inListPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + inListPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(inListPosts, repo, isFrontPage, true);
repo.UpdateDataStore();
PrintStatistics(repo);
}
static void CreateTestDatabase(string inFileName)
{
ModelRepository repo = new ModelRepository();
List <string> listOfPosts = new List <string>();
Logger.Info("Creating data store: " + inFileName);
repo.CreateNewDataStore(inFileName);
Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
for (int j = 0; j <= 650; j += 700)
{
for (int i = 0; i < 1; i++)
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
listOfPosts.AddRange(listPosts);
}
}
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
// now we will fetch posts from some users
Logger.Info("\nANALYZING USERS POSTS:");
listOfPosts.Clear();
List <string> usersTofetch = new List <string>
{
//"mrak",
//"zoran-ostric",
"ppetra",
"zvone-radikalni",
"otpisani"
};
foreach (string user in usersTofetch)
{
var list = UserPostsAnalyzer.GetListOfUserPosts(user);
listOfPosts.AddRange(list);
}
Logger.Info("\nLIST OF USERS POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
repo.UpdateDataStore();
PrintStatistics(repo);
}