Пример #1
0
 /* Class methods */
 public CrawlerPlugin(Website website, DatabaseAccessor db, int crawlID, Log l)
 {
     this.db = db;
     this.website = website;
     this.crawlID = crawlID;
     this.log = l;
 }
Пример #2
0
        public void TestChainOneLink()
        {
            MockRepository mocks = new MockRepository();
            var web = mocks.StrictMock<IWebInteractor>();
            var site = new Website("www.whocares.com", "whatever");
            var mockFSI = MockRepository.GenerateStub<IFileSystemInteractor>();

            CrawlResult testResult = new CrawlResult();
            testResult.ReturnCode = 200;
            testResult.ReturnStatus = "OK";
            testResult.Html = "href=\"/csse.html\"";

            var resultTwo = new CrawlResult();
            resultTwo.ReturnCode = 200;
            resultTwo.ReturnStatus = "OK";
            resultTwo.Html = "";

            Expect.On(web).Call(web.GetPage("www.test.com")).Return(testResult);
            Expect.On(web).Call(web.GetPage("www.test.com/csse.html")).Return(resultTwo);

            mocks.ReplayAll();

            Bot b = new Bot(new Website("www.test.com","simplepath"),null,null,web,mockFSI);
            List<CrawlResult> results = b.CrawlSite(2);

            mocks.VerifyAll();

            Assert.AreEqual(2,results.Count);
        }
Пример #3
0
 public HttpHeaderPlugin(Website website, DatabaseAccessor db, int crawlID, Log l)
     : base(website, db, crawlID, l)
 {
     this.website = website;
     this.db = db;
     this.crawlID = crawlID;
     this.logger = l;
 }
Пример #4
0
        public SSLConfirmationPlugin(Website website, IDatabaseAccessor db, int crawlID, Log l)
            : base(website, db, crawlID, l)
        {
            result = new List<String>();
            logger = l;
            web = website;
            this.crawlID = crawlID;

            /* Ensure we're not doing the effective date stuff wrong */
        }
Пример #5
0
        public Bot(Website website, Log l,DatabaseAccessor dba, IWebInteractor wi, IFileSystemInteractor fsi)
        {
            _baseurl = website.url;
            _website = website;
            _log = l;
            _dba = dba;
            _webinteractor = wi;
            _fsinteractor = fsi;

            ResultsList = new List<CrawlResult>();
        }
Пример #6
0
        public void TestBotCallsCrawlSiteOnWebInteractor()
        {
            MockRepository mocks = new MockRepository();
            var web = MockRepository.GenerateStub<IWebInteractor>();
            var site = new Website("www.whocares.com", "whatever");
            var mockFSI = MockRepository.GenerateStub<FileSystemInteractor>();

            var useableBot = new Bot(site, null, null, web, mockFSI);
            useableBot.CrawlSite(1);
            web.AssertWasCalled(s => s.GetPage("www.whocares.com"));
        }
Пример #7
0
        public SSLConfirmationPlugin(Website website, DatabaseAccessor db, int crawlID, Log l)
            : base(website, db, crawlID, l)
        {
            result = new List<String>();
            logger = l;
            web = website;
            this.crawlID = crawlID;

            /* Ensure we're not doing the effective date stuff wrong */
            if (DateTime.Now.CompareTo(DateTime.Parse(NextRevision)) > 0)
            {
                lastTSLRevision = DateTime.Parse(preNextRevision);
                logger.writeDebug("TLS revision set to " + lastTSLRevision.ToShortDateString());
            }
            else
            {
                lastTSLRevision = DateTime.Parse(NextRevision);
                logger.writeDebug("TLS revision set to " + lastTSLRevision.ToShortDateString());
            }
        }
Пример #8
0
        public CrawlerController(String path, int depth, string email)
        {
            Assembly assem = Assembly.GetEntryAssembly();
            AssemblyName aName = assem.GetName();
            Version ver = aName.Version;
            Console.Out.WriteLine("Application {0}, Version {1}", aName.Name, ver.ToString());

            //String outputPath = path.Substring(path.IndexOf('.') + 1,path.LastIndexOf('.') - path.IndexOf('.') -1) + "-" + DateTime.Now.Month.ToString().PadLeft(2, '0') + "-" + DateTime.Now.Day.ToString().PadLeft(2, '0') +  "-" + DateTime.Now.Year.ToString() + "_" + DateTime.Now.Hour.ToString() + "-" + DateTime.Now.Minute.ToString() ;
            string outputPath = string.Format("{0}_{1}", path, DateTime.Now.ToString("hh-mm_MM-dd-yyyy"));

            String arguments = path + " -g -r" + depth.ToString() + " -O " + outputPath;
            //DEBUG - Console.WriteLine("Running httrack with arguments: " + arguments);
            //Process p = Process.Start(Directory.GetCurrentDirectory() + "/httrack/httrack.exe", arguments);
            //p.WaitForExit();
            //Directory.CreateDirectory(output);

            String foldername = outputPath;
            outputPath =  Directory.GetCurrentDirectory() + "\\"  +  outputPath;
            Directory.CreateDirectory(outputPath);

            //initialize the website
            Website site = new Website(path,outputPath);

            //initialize the log
            Log log = new Log(outputPath + "\\log.txt");
            log.writeInfo("Log created correctly");
            log.writeInfo("Website: " + path + " == CrawlLevel: " + depth.ToString());
            log.writeInfo("Running version: " + aName.Version.ToString());

            //initalize the database accessor class
            log.writeDebug("Creating database object");
            DatabaseAccessor dbAccess = null;
            try
            {
                dbAccess = new DatabaseAccessor(log, ConfigReader.ReadDatabaseAccessorString());
            }
            catch(Exception e)
            {
                Console.Out.WriteLine("Error creating database connection: " + e.Message);
                Console.Out.WriteLine("Reverting to default CrawlID");
                log.writeError("Error creating database connection: " + e.Message);
                log.writeError("Reverting to default CrawlID");
            }
            if (dbAccess != null) dbAccess.addWebsite(path, null, null, null);

            int crawlID;
            if (dbAccess != null)
            {
                crawlID = dbAccess.newCrawl(path, email);
            }
            else
            {
                crawlID = 0;
            }

            var fsi = new FileSystemInteractor(log);
            Bot b = new Bot(site, log, null, new WebInteractor(log), fsi);
            b.CrawlSite(depth);

            //Parse website
            WebsiteParser siteparser = new WebsiteParser(site, dbAccess, crawlID, log,fsi);
            List<String> result = siteparser.analyzeSite();

            //Try to analyse an SSL certificate, if there is one
            log.writeDebug("Creating SSL object");
            CrawlerPlugin ssl = new SSLConfirmationPlugin(site, dbAccess, crawlID, log);
            result.AddRange(ssl.analyzeSite());

            //Get headers
            CrawlerPlugin headers = new HttpHeaderPlugin(site, dbAccess, crawlID, log);
            result.AddRange(headers.analyzeSite());

            //HTML Parser
            log.writeDebug("Creating HTML parsing object");
            HTMLParsingModule HTMLParser = new HTMLParsingModule(site, dbAccess, crawlID, log);
            result.AddRange(HTMLParser.analyzeSite());
            log.writeDebug("Done parsing HTML");

            //Write to database
            dbAccess.AddVulnerabilities(crawlID, result);

            //notify
            log.writeDebug("Preparing to send message");
            try
            {
                NotifyClient.sendMessage(ConfigReader.ReadEmailAddress());
            }
            catch(Exception e)
            {
                Console.Out.WriteLine("Error in Notify client: " + e.Message);
                log.writeError("Error in Notify client: " + e.Message);
            }
            log.writeDebug("Done sending notification");

            log.writeDebug("Destroying log....program exiting");
            log.destroy();
        }
Пример #9
0
        public void TestStatus404()
        {
            MockRepository mocks = new MockRepository();
            var web = mocks.StrictMock<IWebInteractor>();
            var site = new Website("www.whocares.com", "whatever");
            var mockFSI = MockRepository.GenerateStub<FileSystemInteractor>();

            var retVal = new CrawlResult();
            retVal.ReturnCode = 404;
            retVal.Html = @"";

            Expect.On(web).Call(web.GetPage("www.whocares.com")).Return(retVal);
            mocks.ReplayAll();

            var useableBot = new Bot(site, null, null, web, mockFSI);
            var check = useableBot.CrawlSite(1);

            Assert.IsTrue(check[0].ReturnCode == 404);

            mocks.VerifyAll();
        }
Пример #10
0
        public void TestCreateDirectory()
        {
            MockRepository mocks = new MockRepository();

            Website web = new Website("www.test.com", "");
            Log l = MockRepository.GenerateStub<Log>("samplepath");
            WebInteractor wi = MockRepository.GenerateStub<WebInteractor>();
            var fsi = mocks.StrictMock<IFileSystemInteractor>();

            Bot b = new Bot(web, l, null, wi, fsi);

            using (mocks.Record())
            {
                fsi.MakeDirectory("samplepath");
            }
        }
Пример #11
0
 public WebsiteParser(Website website, DatabaseAccessor db, int crawlID, Log l, FileSystemInteractor fs)
     : base(website, db, crawlID, l)
 {
     fsInteractor = fs;
 }
Пример #12
0
 public HTMLParsingModule(Website w, IDatabaseAccessor db, int id, Log log)
     : base(w, db, id, log)
 {
     searchStringList = new List<String>();
     descriptionList = new List<String>();
 }