public void New_Crawler() { #region Init constructor Parameters param = new Parameters(new Uri("http://stackoverflow.com/"), "http://stackoverflow.com/questions(/[a-z0-9/-]+)?", "http://stackoverflow.com/questions/([a-z0-9/-]+)?", 2, "test"); Crawler crawler = new Crawler(param); Assert.AreEqual("test", crawler.param.Name); #endregion #region LoadXML string filePath = Path.GetDirectoryName(Path.GetDirectoryName(System.IO.Directory.GetCurrentDirectory())) + "\\Xml_Files\\" + param.Name + ".xml"; XElement doc = XElement.Parse(File.ReadAllText(filePath)); Crawler crawler2 = new Crawler(doc, param); int count = crawler2.CountToVisiteUrls(); Assert.That(crawler2.CountToVisiteUrls() == 1); #endregion #region Crawler Run Assert.That(crawler2.CountToVisiteUrls() == 1); crawler2.RunCrawler(); Assert.That(crawler2.CountToVisiteUrls() == 0); #endregion }
public void New_Paramaters() { #region Init constructor Parameters param = new Parameters(new Uri("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/"), "http://tlevesque.developpez.com/(cours|tutoriels)/([a-z0-9/-]+)?", "http://[a-z0-9-]+.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/", 2, "Tuto"); Assert.AreEqual("Tuto", param.Name); Assert.AreEqual("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/", param.Url.AbsoluteUri); #endregion }
public Crawler(XElement e, Parameters param) { this.param = param; _urls = DictionaryFromXml(e); Extractor = new Extractor(); }
public Crawler(Parameters param) { this.param = param; Extractor = new Extractor(); }