public CrawlingSharedResource(CrawlSetting crawlerSetting, Host host, Queue<Webpage> sharedQueue, object sharedLock, HashSet<string> sharedPageHash, HashSet<string> sharedFormHash)
{
CrawlerSetting = crawlerSetting;
Host = host;
SharedQueue = sharedQueue;
SharedLock = sharedLock;
SharedFormHash = sharedFormHash;
SharedPageHash = sharedPageHash;
}
static void Main(string[] args)
{
CrawlSetting setting = new CrawlSetting();
setting.MaxDepth = 1000;
Host[] hosts=new Host[1];
hosts[0] = new Host();
hosts[0].HostName = "http://salamandroid.ir";
Crawler crawler = new Crawler(setting,hosts,10);
crawler.Crawl();
Console.ReadLine();
}
public static void Save(Host host)
{
xKnightEntities context = new xKnightEntities();
if (host.Id == 0)
{
context.Hosts.AddObject(host);
context.SaveChanges();
}
else
{
context.Hosts.Attach(host);
context.ObjectStateManager.ChangeObjectState(host, System.Data.EntityState.Modified);
}
context.Dispose();
}
/// <summary>
///
/// </summary>
/// <param name="crawlerSetting">Settings for this job</param>
/// <param name="host">the host to do crawling on it</param>
/// <param name="threadsNumber">number of threads for crawling the specified host</param>
public Crawler(CrawlSetting crawlerSetting, Host[] hosts, int threadsNumber = 1)
{
_crawlSetting = crawlerSetting;
_threadsNumber = threadsNumber;
for (int i = 0; i < hosts.Length; i++)
{
if (!Uri.IsWellFormedUriString(hosts[i].HostName, UriKind.Absolute))
{
throw new InvalidDataException(string.Format("invalid host name : {0}",hosts[i].HostName));
}
}
_hosts = hosts;
}
public EncodedXssAttacker(Attack attack, int numberOfThreads, Host[] hosts)
: base(attack,numberOfThreads)
{
_hosts = hosts;
}
public static Form[] GetHostsForms(Host[] _hosts)
{
throw new NotImplementedException();
}
/// <summary>
/// Deprecated Method for adding a new object to the Hosts EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead.
/// </summary>
public void AddToHosts(Host host)
{
base.AddObject("Hosts", host);
}
/// <summary>
/// Create a new Host object.
/// </summary>
/// <param name="id">Initial value of the Id property.</param>
/// <param name="crawlId">Initial value of the CrawlId property.</param>
/// <param name="hostName">Initial value of the HostName property.</param>
/// <param name="indexedPages">Initial value of the IndexedPages property.</param>
/// <param name="bytesDownloaded">Initial value of the BytesDownloaded property.</param>
public static Host CreateHost(global::System.Int32 id, global::System.Int32 crawlId, global::System.String hostName, global::System.Int32 indexedPages, global::System.Int64 bytesDownloaded)
{
Host host = new Host();
host.Id = id;
host.CrawlId = crawlId;
host.HostName = hostName;
host.IndexedPages = indexedPages;
host.BytesDownloaded = bytesDownloaded;
return host;
}
public MainWindow()
{
InitializeComponent();
_crawlSetting = new CrawlSetting();
_host = new Host();
}
public Crawler(CrawlSetting crawlSetting, Host host, int threadsNumber=1)
: this(crawlSetting,new Host[]{host},threadsNumber)
{
}