public SimpleCrawlerTest(string url, ILinkStorage linkStorage)
{
_linkStorage = linkStorage ?? new ConsoleLinkStorage();
Settings.SeedsAddress.Add(url);
//Settings.ThreadCount = 20;
Settings.Depth = 5;
Settings.EscapeLinks.Add(".jpg");
Settings.EscapeLinks.Add(".gif");
Settings.EscapeLinks.Add(".png");
Settings.EscapeLinks.Add(".pdf");
Settings.EscapeLinks.Add(".doc");
Settings.EscapeLinks.Add(".xls");
Settings.AutoSpeedLimit = true;
_master = new CrawlMaster(Settings);
_master.AddUrlEvent += MasterAddUrlEvent;
_master.DataReceivedEvent += MasterDataReceivedEvent;
}
private static void TestAbot(string url, ILinkStorage linkStorage = null)
{
var abotTest = new AbotTest(url, linkStorage);
abotTest.Execute();
}
private static void TestLoew(string url, ILinkStorage linkStorage = null)
{
var loew = new LoewTest(url, linkStorage);
loew.Execute();
}
public LinkService(ILinkStorage storage)
{
_storage = storage;
}
private static void TestSimpleCrawler(string url, ILinkStorage linkStorage = null)
{
var abotTest = new SimpleCrawlerTest(url, linkStorage);
abotTest.Execute();
}
public WebsiteDownloader(HtmlLinkManager linkManager, IWebResourceSave resourceRecorder, ILinkStorage linkStorage = null)
{
_resourceLinkStorage = linkStorage ?? new MemoryLinkStorage();
_resourceRecorder = resourceRecorder;
_linkManager = linkManager;
}
public LoewTest(string url, ILinkStorage linkStorage)
{
_linkStorage = linkStorage ?? new ConsoleLinkStorage();
_url = url;
_loader = new SitemapLoader();
}
public AbotTest(string url, ILinkStorage linkStorage)
{
_linkStorage = linkStorage ?? new ConsoleLinkStorage();
_url = url;
}
