static void AnalyzeFrontPagePosts(string inRepoFileName)
{
ModelRepository repo = new ModelRepository();
List <string> listOfPosts = new List <string>();
Logger.Info("Opening data store: " + inRepoFileName);
repo.OpenDataStore(inRepoFileName);
Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
for (int j = 0; j <= 500; j += 100)
{
for (int i = 85; i < 100; i++)
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
listOfPosts.AddRange(listPosts);
}
}
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
repo.UpdateDataStore();
PrintStatistics(repo);
}
static public void FetchPosts()
{
// setup - repozitoriji za spremanje podataka
ModelRepository repo = new ModelRepository();
repo.OpenDataStore("pollitika.db");
// prođemo kroz naslovnicu i pokupimo sve postove (i sve komentare i dodamo sve korisnike)
for (int j = 200; j <= 600; j++)
{
for (int i = 0; i < 1; i++)
{
Console.WriteLine("DOING PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
foreach (string postUrl in listPosts)
{
//Console.WriteLine("Post url {0}", postUrl);
Post newPost = PostAnalyzer.AnalyzePost("http://pollitika.com" + postUrl, repo, true, true);
if (newPost != null)
{
repo.AddPost(newPost);
}
}
repo.UpdateDataStore();
}
}
// zatim prolazimo kroz sve korisnike
// i za svakog od njih proći kroz sve njegove dnevnike
// i za svaki dnevnik provjeriti da li je već dodan, i ak onije dodati ga
}
static void CreateTestDatabase(string inFileName)
{
ModelRepository repo = new ModelRepository();
List <string> listOfPosts = new List <string>();
Logger.Info("Creating data store: " + inFileName);
repo.CreateNewDataStore(inFileName);
Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
for (int j = 0; j <= 650; j += 700)
{
for (int i = 0; i < 1; i++)
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
listOfPosts.AddRange(listPosts);
}
}
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
// now we will fetch posts from some users
Logger.Info("\nANALYZING USERS POSTS:");
listOfPosts.Clear();
List <string> usersTofetch = new List <string>
{
//"mrak",
//"zoran-ostric",
"ppetra",
"zvone-radikalni",
"otpisani"
};
foreach (string user in usersTofetch)
{
var list = UserPostsAnalyzer.GetListOfUserPosts(user);
listOfPosts.AddRange(list);
}
Logger.Info("\nLIST OF USERS POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
Logger.Info("\nSTARTING ANALYSIS:");
MultithreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
repo.UpdateDataStore();
PrintStatistics(repo);
}