public void New_Crawler()
{
#region Init constructor
Parameters param = new Parameters(new Uri("http://stackoverflow.com/"),
"http://stackoverflow.com/questions(/[a-z0-9/-]+)?",
"http://stackoverflow.com/questions/([a-z0-9/-]+)?",
2,
"test");
Crawler crawler = new Crawler(param);
Assert.AreEqual("test", crawler.param.Name);
#endregion
#region LoadXML
string filePath = Path.GetDirectoryName(Path.GetDirectoryName(System.IO.Directory.GetCurrentDirectory())) + "\\Xml_Files\\" + param.Name + ".xml";
XElement doc = XElement.Parse(File.ReadAllText(filePath));
Crawler crawler2 = new Crawler(doc, param);
int count = crawler2.CountToVisiteUrls();
Assert.That(crawler2.CountToVisiteUrls() == 1);
#endregion
#region Crawler Run
Assert.That(crawler2.CountToVisiteUrls() == 1);
crawler2.RunCrawler();
Assert.That(crawler2.CountToVisiteUrls() == 0);
#endregion
}
public void New_Paramaters()
{
#region Init constructor
Parameters param = new Parameters(new Uri("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/"),
"http://tlevesque.developpez.com/(cours|tutoriels)/([a-z0-9/-]+)?",
"http://[a-z0-9-]+.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/",
2,
"Tuto");
Assert.AreEqual("Tuto", param.Name);
Assert.AreEqual("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/", param.Url.AbsoluteUri);
#endregion
}
public Crawler(XElement e, Parameters param)
{
this.param = param;
_urls = DictionaryFromXml(e);
Extractor = new Extractor();
}
public Crawler(Parameters param)
{
this.param = param;
Extractor = new Extractor();
}