public CrawlingSharedResource(CrawlSetting crawlerSetting, Host host, Queue <Webpage> sharedQueue, object sharedLock, HashSet <string> sharedPageHash, HashSet <string> sharedFormHash)
 {
     CrawlerSetting = crawlerSetting;
     Host           = host;
     SharedQueue    = sharedQueue;
     SharedLock     = sharedLock;
     SharedFormHash = sharedFormHash;
     SharedPageHash = sharedPageHash;
 }
示例#2
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="crawlerSetting">Settings for this job</param>
        /// <param name="host">the host to do crawling on it</param>
        /// <param name="threadsNumber">number of threads for crawling the specified host</param>
        public Crawler(CrawlSetting crawlerSetting, Host[] hosts, int threadsNumber = 1)
        {
            _crawlSetting  = crawlerSetting;
            _threadsNumber = threadsNumber;

            for (int i = 0; i < hosts.Length; i++)
            {
                if (!Uri.IsWellFormedUriString(hosts[i].HostName, UriKind.Absolute))
                {
                    throw new InvalidDataException(string.Format("invalid host name : {0}", hosts[i].HostName));
                }
            }

            _hosts = hosts;
        }
示例#3
0
        static void Main(string[] args)
        {
            CrawlSetting setting = new CrawlSetting();

            setting.MaxDepth = 1000;

            Host[] hosts = new Host[1];
            hosts[0]          = new Host();
            hosts[0].HostName = "http://salamandroid.ir";

            Crawler crawler = new Crawler(setting, hosts, 10);

            crawler.Crawl();

            Console.ReadLine();
        }
示例#4
0
        public static void Save(CrawlSetting crawlSetting)
        {
            xKnightEntities context = new xKnightEntities();

            if (crawlSetting.Id == 0)
            {
                context.CrawlSettings.AddObject(crawlSetting);
                context.SaveChanges();
            }
            else
            {
                context.CrawlSettings.Attach(crawlSetting);
                context.ObjectStateManager.ChangeObjectState(crawlSetting, System.Data.EntityState.Modified);
            }
            context.Dispose();
        }
示例#5
0
 public MainWindow()
 {
     InitializeComponent();
     _crawlSetting = new CrawlSetting();
     _host         = new Host();
 }
示例#6
0
 public Crawler(CrawlSetting crawlSetting, Host host, int threadsNumber = 1)
     : this(crawlSetting, new Host[] { host }, threadsNumber)
 {
 }