static void Main(string[] args)
{
string[] keywords = new string[]
{
"hello",
"csharp"
};
Policy p = new GooglePolicy();
Crawler c = new Crawler(p);
c.PrepareQueries(keywords);
c.Start();
}
static void Main(string[] args)
{
//const string baseUrl = "https://softuni.bg/users/profile/show/";
//const string baseUrl = "https://softuni.bg/forum?page=0&pageSize=5000";
//get content from forum: https://softuni.bg/forum/4400/0/100
const string fileName = "content";
const string user = "******";
const string forumBaseUrl = "https://softuni.bg/forum/";
var list = new List<int>();
for (int i = 4; i < 4500; i++)
{
var spider = new Crawler();
var htmlContent = spider.GetHtml(forumBaseUrl + i + "/0/100");
Console.Write("Forum [" + i +"]");
if (htmlContent.ToString().Contains(user))
{
list.Add(i);
Console.Write("[1]");
}
else
{
Console.Write("[0]");
}
Console.WriteLine();
}
Console.WriteLine("Completed");
Console.WriteLine(string.Join(", ", list));
Console.ReadLine();
//try
//{
// var spider = new Crawler();
// var htmlContent = spider.GetHtml(forumBaseUrl);
// spider.SaveHtmlContent(htmlContent, fileName);
//}
//catch (Exception e)
//{
// throw new Exception("There's an error");
//}
//finally
//{
// Console.WriteLine("Process comepleted!");
// Console.ReadLine();
//}
}
public void CrawlWebSite(string domain, IFilter whatAreWeSearchingForToDay)
{
Crawler c = new Crawler(domain, userAgent == null ? "User-agent: *".ToLower() : userAgent, whatAreWeSearchingForToDay, this);
c.Start();
}
public void OnResult(object source, Crawler.Event.SearchResultEventArgs e)
{
}
/// <summary>
/// Creates a crawler, the nth.
/// </summary>
/// <param name="n">The number of the crawler to start.</param>
/// <param name="trainTest">If true, the crawler will actually only train+test, not crawl.</param>
private void CreateCrawler(int n, bool trainTest)
{
crawlers[n] = new Crawler(n, this);
if (!trainTest)
{
// Train the crawlers here, if we're not testing
crawlers[n].Train();
SetRichText("Created and trained crawler " + n + "\n");
}
}