public void New_Crawler()
        {
            #region Init constructor
            Parameters param = new Parameters(new Uri("http://stackoverflow.com/"),
                  "http://stackoverflow.com/questions(/[a-z0-9/-]+)?",
                  "http://stackoverflow.com/questions/([a-z0-9/-]+)?",
                  2,
                  "test");
            Crawler crawler = new Crawler(param);
            Assert.AreEqual("test", crawler.param.Name);
            #endregion

            #region LoadXML
            string filePath = Path.GetDirectoryName(Path.GetDirectoryName(System.IO.Directory.GetCurrentDirectory())) + "\\Xml_Files\\" + param.Name + ".xml";
            XElement doc = XElement.Parse(File.ReadAllText(filePath));
            Crawler crawler2 = new Crawler(doc, param);
            int count = crawler2.CountToVisiteUrls();
            Assert.That(crawler2.CountToVisiteUrls() == 1);
            #endregion

            #region Crawler Run
            Assert.That(crawler2.CountToVisiteUrls() == 1);
            crawler2.RunCrawler();
            Assert.That(crawler2.CountToVisiteUrls() == 0);

            #endregion
        }
 public void New_Paramaters()
 {
     #region Init constructor
     Parameters param = new Parameters(new Uri("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/"),
         "http://tlevesque.developpez.com/(cours|tutoriels)/([a-z0-9/-]+)?",
         "http://[a-z0-9-]+.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/",
         2,
         "Tuto");
     Assert.AreEqual("Tuto", param.Name);
     Assert.AreEqual("http://tlevesque.developpez.com/tutoriels/dotnet/extraction-donnees-web-html-agility-pack/", param.Url.AbsoluteUri);
     #endregion
 }
Ejemplo n.º 3
0
 public Crawler(XElement e, Parameters param)
 {
     this.param = param;
     _urls = DictionaryFromXml(e);
     Extractor = new Extractor();
 }
Ejemplo n.º 4
0
 public Crawler(Parameters param)
 {
     this.param = param;
     Extractor = new Extractor();
 }