public WebsiteParser(WebsiteParserRequest WebsiteParserRequest)
{
taskId = WebsiteParserRequest.TaskId;
baseUrl = WebsiteParserRequest.DomainName;
maxDeep = WebsiteParserRequest.MaxDeep;
domainExtentions = WebsiteParserRequest.DomainExtentions;
domainName = WebsiteParserRequest.DomainName;
domainLevel = WebsiteParserRequest.DomainLevel;
DicAllInternalUrls = new Dictionary <string, int>();
}
public static async Task Start()
{
var websiteParserRequest = new WebsiteParserRequest()
{
DomainName = "www.sport5.co.il",
MaxDeep = 2,
DomainExtentions = new List <string>()
{
"co.il",
"org.il"
}
};
WebSitesConcurrentQueue.WebSites = new ConcurrentQueue <string>();
WebSitesConcurrentQueue.AllWebSites = new ConcurrentQueue <string>();
using (var websiteParser = new WebsiteParser(websiteParserRequest))
{
await websiteParser.Parse();
await FileData.Save <object>("links.txt", websiteParser.DicAllInternalUrls.Select(x => new { url = x.Key, deep = x.Value }));
}
}
