public static async Task Start()
{
var websiteParserRequest = new WebsiteParserRequest()
{
DomainName = "www.sport5.co.il",
MaxDeep = 2,
DomainExtentions = new List <string>()
{
"co.il",
"org.il"
}
};
WebSitesConcurrentQueue.WebSites = new ConcurrentQueue <string>();
WebSitesConcurrentQueue.AllWebSites = new ConcurrentQueue <string>();
using (var websiteParser = new WebsiteParser(websiteParserRequest))
{
await websiteParser.Parse();
await FileData.Save <object>("links.txt", websiteParser.DicAllInternalUrls.Select(x => new { url = x.Key, deep = x.Value }));
}
}
public void SetUp()
{
_websiteParser = new WebsiteParser("http://www.cloudhouse.com/");
}
internal void WebsiteParse(ExtractFileInfo fileInfo)
{
WebsiteParser sectionParser = new WebsiteParser(_log, fileInfo, IsPreservation, DatabasePreservationNoPrefix, Version);
WriteTables(sectionParser);
}
