public CrawlingSharedResource(CrawlSetting crawlerSetting, Host host, Queue<Webpage> sharedQueue, object sharedLock, HashSet<string> sharedPageHash, HashSet<string> sharedFormHash)
 {
     CrawlerSetting = crawlerSetting;
     Host = host;
     SharedQueue = sharedQueue;
     SharedLock = sharedLock;
     SharedFormHash = sharedFormHash;
     SharedPageHash = sharedPageHash;
 }
Example #2
0
        static void Main(string[] args)
        {
            CrawlSetting setting = new CrawlSetting();
            setting.MaxDepth = 1000;

            Host[] hosts=new Host[1];
            hosts[0] = new Host();
            hosts[0].HostName = "http://salamandroid.ir";

            Crawler crawler = new Crawler(setting,hosts,10);
            crawler.Crawl();

            Console.ReadLine();
        }
Example #3
0
 public static void Save(CrawlSetting crawlSetting)
 {
     xKnightEntities context = new xKnightEntities();
     if (crawlSetting.Id == 0)
     {
         context.CrawlSettings.AddObject(crawlSetting);
         context.SaveChanges();
     }
     else
     {
         context.CrawlSettings.Attach(crawlSetting);
         context.ObjectStateManager.ChangeObjectState(crawlSetting, System.Data.EntityState.Modified);
     }
     context.Dispose();
 }
Example #4
0
        /// <summary>
        /// 
        /// </summary>
        /// <param name="crawlerSetting">Settings for this job</param>
        /// <param name="host">the host to do crawling on it</param>
        /// <param name="threadsNumber">number of threads for crawling the specified host</param>
        public Crawler(CrawlSetting crawlerSetting, Host[] hosts, int threadsNumber = 1)
        {
            _crawlSetting = crawlerSetting;
            _threadsNumber = threadsNumber;

            for (int i = 0; i < hosts.Length; i++)
            {
                if (!Uri.IsWellFormedUriString(hosts[i].HostName, UriKind.Absolute))
                {
                    throw new InvalidDataException(string.Format("invalid host name : {0}",hosts[i].HostName));
                }
            }

            _hosts = hosts;
        }
 /// <summary>
 /// Create a new CrawlSetting object.
 /// </summary>
 /// <param name="id">Initial value of the Id property.</param>
 public static CrawlSetting CreateCrawlSetting(global::System.Int32 id)
 {
     CrawlSetting crawlSetting = new CrawlSetting();
     crawlSetting.Id = id;
     return crawlSetting;
 }
 /// <summary>
 /// Deprecated Method for adding a new object to the CrawlSettings EntitySet. Consider using the .Add method of the associated ObjectSet&lt;T&gt; property instead.
 /// </summary>
 public void AddToCrawlSettings(CrawlSetting crawlSetting)
 {
     base.AddObject("CrawlSettings", crawlSetting);
 }
Example #7
0
 public MainWindow()
 {
     InitializeComponent();
     _crawlSetting = new CrawlSetting();
     _host = new Host();
 }
Example #8
0
 public Crawler(CrawlSetting crawlSetting, Host host, int threadsNumber=1)
     : this(crawlSetting,new Host[]{host},threadsNumber)
 {
 }