public Spider(string Frontier, SpiderJobType JobType, IPersistenceInserter persistenceInserter, int MaxAllowedPages = 1000, bool OnDomainPagesOnly = true, int MaxAllowedTasks = 3)
{
this.Frontier = Frontier;
this.MaxAllowedPages = MaxAllowedPages;
this.MaxAllowedTasks = MaxAllowedTasks;
this.UserAgent = DefaultUserAgent;
this.JobType = JobType;
this.OnDomainPagesOnly = OnDomainPagesOnly;
this.Unvisited = new ConcurrentDictionary <string, UrlObject>();
this.Visited = new ConcurrentDictionary <string, Page>();
this.SpiderTasks = new List <Task <Page> >();
this.PersistenceCancellationSource = new CancellationTokenSource();
this.PersistenceInserter = persistenceInserter;
var ct = PersistenceCancellationSource.Token;
}
public Spider(string Frontier, SpiderJobType JobType, IPersistenceInserter persistenceInserter, int MaxAllowedPages = 1000, bool OnDomainPagesOnly = true, int MaxAllowedTasks = 3)
{
this.Frontier = Frontier;
this.MaxAllowedPages = MaxAllowedPages;
this.MaxAllowedTasks = MaxAllowedTasks;
this.UserAgent = DefaultUserAgent;
this.JobType = JobType;
this.OnDomainPagesOnly = OnDomainPagesOnly;
this.Unvisited = new ConcurrentDictionary<string, UrlObject>();
this.Visited = new ConcurrentDictionary<string, Page>();
this.SpiderTasks = new List<Task<Page>>();
this.PersistenceCancellationSource = new CancellationTokenSource();
this.PersistenceInserter = persistenceInserter;
var ct = PersistenceCancellationSource.Token;
}
