public static void SimpleTaskSample()
{
Console.WriteLine("Start Simple Task Sample");
var engineConfig = new CrawlerEngineConfig();
var engine = new CrawlerEngine(engineConfig);
engine.Start();
Console.WriteLine();
Console.WriteLine("Start Task");
Uri fileURI = new Uri(URLbox2.Text);
engine.AddTask(new SimpleTaskRequest {
Url = fileURI
});
while (!engine.IsIdle)
{
Thread.Sleep(100);
TaskResultBase[] results = engine.GetFinishedTaskResults();
if (results.Length > 0)
{
Console.WriteLine();
Console.WriteLine("Processing Results");
foreach (SimpleTaskResult result in results)
{
foreach (var link in result.Links)
{
Console.WriteLine(link);
}
}
}
}
}
private void StartCrawler()
{
CrawlerEngineConfig config = new CrawlerEngineConfig();
config.MaxTasksPerMinute = 10;
config.MaxFinishedTasks = 1000;
config.MaxWorkingTasks = 3;
using (CrawlerEngine engine = new CrawlerEngine(config))
{
}
}
