public CrawlingSharedResource(CrawlSetting crawlerSetting, Host host, Queue<Webpage> sharedQueue, object sharedLock, HashSet<string> sharedPageHash, HashSet<string> sharedFormHash)
{
CrawlerSetting = crawlerSetting;
Host = host;
SharedQueue = sharedQueue;
SharedLock = sharedLock;
SharedFormHash = sharedFormHash;
SharedPageHash = sharedPageHash;
}
static void Main(string[] args)
{
CrawlSetting setting = new CrawlSetting();
setting.MaxDepth = 1000;
Host[] hosts=new Host[1];
hosts[0] = new Host();
hosts[0].HostName = "http://salamandroid.ir";
Crawler crawler = new Crawler(setting,hosts,10);
crawler.Crawl();
Console.ReadLine();
}
public static void Save(CrawlSetting crawlSetting)
{
xKnightEntities context = new xKnightEntities();
if (crawlSetting.Id == 0)
{
context.CrawlSettings.AddObject(crawlSetting);
context.SaveChanges();
}
else
{
context.CrawlSettings.Attach(crawlSetting);
context.ObjectStateManager.ChangeObjectState(crawlSetting, System.Data.EntityState.Modified);
}
context.Dispose();
}
/// <summary>
///
/// </summary>
/// <param name="crawlerSetting">Settings for this job</param>
/// <param name="host">the host to do crawling on it</param>
/// <param name="threadsNumber">number of threads for crawling the specified host</param>
public Crawler(CrawlSetting crawlerSetting, Host[] hosts, int threadsNumber = 1)
{
_crawlSetting = crawlerSetting;
_threadsNumber = threadsNumber;
for (int i = 0; i < hosts.Length; i++)
{
if (!Uri.IsWellFormedUriString(hosts[i].HostName, UriKind.Absolute))
{
throw new InvalidDataException(string.Format("invalid host name : {0}",hosts[i].HostName));
}
}
_hosts = hosts;
}
/// <summary>
/// Create a new CrawlSetting object.
/// </summary>
/// <param name="id">Initial value of the Id property.</param>
public static CrawlSetting CreateCrawlSetting(global::System.Int32 id)
{
CrawlSetting crawlSetting = new CrawlSetting();
crawlSetting.Id = id;
return crawlSetting;
}
/// <summary>
/// Deprecated Method for adding a new object to the CrawlSettings EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead.
/// </summary>
public void AddToCrawlSettings(CrawlSetting crawlSetting)
{
base.AddObject("CrawlSettings", crawlSetting);
}
public MainWindow()
{
InitializeComponent();
_crawlSetting = new CrawlSetting();
_host = new Host();
}
public Crawler(CrawlSetting crawlSetting, Host host, int threadsNumber=1)
: this(crawlSetting,new Host[]{host},threadsNumber)
{
}