public CrawlingSharedResource(CrawlSetting crawlerSetting, Host host, Queue <Webpage> sharedQueue, object sharedLock, HashSet <string> sharedPageHash, HashSet <string> sharedFormHash)
{
CrawlerSetting = crawlerSetting;
Host = host;
SharedQueue = sharedQueue;
SharedLock = sharedLock;
SharedFormHash = sharedFormHash;
SharedPageHash = sharedPageHash;
}
/// <summary>
///
/// </summary>
/// <param name="crawlerSetting">Settings for this job</param>
/// <param name="host">the host to do crawling on it</param>
/// <param name="threadsNumber">number of threads for crawling the specified host</param>
public Crawler(CrawlSetting crawlerSetting, Host[] hosts, int threadsNumber = 1)
{
_crawlSetting = crawlerSetting;
_threadsNumber = threadsNumber;
for (int i = 0; i < hosts.Length; i++)
{
if (!Uri.IsWellFormedUriString(hosts[i].HostName, UriKind.Absolute))
{
throw new InvalidDataException(string.Format("invalid host name : {0}", hosts[i].HostName));
}
}
_hosts = hosts;
}
static void Main(string[] args)
{
CrawlSetting setting = new CrawlSetting();
setting.MaxDepth = 1000;
Host[] hosts = new Host[1];
hosts[0] = new Host();
hosts[0].HostName = "http://salamandroid.ir";
Crawler crawler = new Crawler(setting, hosts, 10);
crawler.Crawl();
Console.ReadLine();
}
public static void Save(CrawlSetting crawlSetting)
{
xKnightEntities context = new xKnightEntities();
if (crawlSetting.Id == 0)
{
context.CrawlSettings.AddObject(crawlSetting);
context.SaveChanges();
}
else
{
context.CrawlSettings.Attach(crawlSetting);
context.ObjectStateManager.ChangeObjectState(crawlSetting, System.Data.EntityState.Modified);
}
context.Dispose();
}
public MainWindow()
{
InitializeComponent();
_crawlSetting = new CrawlSetting();
_host = new Host();
}
public Crawler(CrawlSetting crawlSetting, Host host, int threadsNumber = 1)
: this(crawlSetting, new Host[] { host }, threadsNumber)
{
}