예제 #1
0
        public void TestRobotsCache()
        {
            CollectorManager cm = new CollectorManager();

            RobotsCache cache = new RobotsCache(cm, "http://www.yahoo.com");
            Assert.IsTrue(cache.RobotsExclusion.Count > 0, "Robots not loaded");
        }
예제 #2
0
파일: LinkInfo.cs 프로젝트: quartz12345/c
        protected virtual void LoadAndCheckRobotsRule()
        {
            RobotsCache cache = new RobotsCache(this.CM, this.DomainSchemeAndServer);
            RobotsExclusion = cache.RobotsExclusion;
            cache = null;

            foreach (string robotExclusion in this.RobotsExclusion)
            {
                Uri robotUri = new Uri(new Uri(this.DomainSchemeAndServer), robotExclusion);
                if (this.Link.StartsWith(robotUri.ToString(), StringComparison.InvariantCultureIgnoreCase))
                {
                    LinkExcludedInRobots = true;
                    break;
                }
            }
        }