public static void StandardDownloadFromFrontpage()
{
ModelRepository repo = new ModelRepository();
List <string> listOfPosts = new List <string>();
string repoName = "pollitikaNew.db";
Logger.Info("Opening data store: " + repoName);
repo.OpenDataStore(repoName);
Logger.Info("\nFETCHING POSTS FROM FRONTPAGE:");
for (int j = 0; j <= 500; j += 100)
{
for (int i = 80; i < 100; i++) // došli smo do 60
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", j + i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(j + i);
listOfPosts.AddRange(listPosts);
}
}
Logger.Info("\nLIST OF POSTS TO ANALYZE:");
for (int i = 0; i < listOfPosts.Count; i++)
{
Logger.Info((i + 1).ToString() + ". " + listOfPosts[i]);
}
ContinuousMultiThreadedScrapper.AnalyzeListOfPosts_Multithreaded(listOfPosts, repo, true, true);
PrintStatistics(repo);
repo.UpdateDataStore();
}
public static void GetListOfFrontPagePostsToFile(string inFileName)
{
using (System.IO.StreamWriter file = new System.IO.StreamWriter(inFileName))
{
List <string> listOfPosts = new List <string>();
for (int i = 0; i < 681; i++)
{
Logger.InfoFormat(" DOING FRONT PAGE - {0}", i);
var listPosts = FrontPageAnalyzer.GetPostLinksFromFrontPage(i);
for (int j = 0; j < listPosts.Count; j++)
{
Logger.Info(" ." + listPosts[j]);
file.WriteLine(listPosts[j]);
}
listOfPosts.AddRange(listPosts);
}
}
}
