private async Task Search(CrawlerStartModel crawlerStartModel, int currentDepth, List <string> controlledSites, string crawleName)
{
if (!controlledSites.Any(x => x == crawlerStartModel.URL))
{
WebRequest wr = WebRequest.Create(crawlerStartModel.URL);
WebResponse ws = wr.GetResponse();
StreamReader sr = new StreamReader(ws.GetResponseStream(), Encoding.UTF8);
string response = sr.ReadToEnd();
sr.Close();
ws.Close();
CrawlerSearch crawlerSearchResult = new CrawlerSearch()
{
Date = DateTime.Now,
URL = crawlerStartModel.URL,
Expression = crawlerStartModel.Expression,
Crawlername = crawleName
};
if (response.Length > 0)
{
crawlerSearchResult.HitCount = response.Split(crawlerStartModel.Expression).Count() - 1;
}
else
{
crawlerSearchResult.HitCount = 0;
}
await _crawlerSearchRepository.InsertCrawlerSearch(crawlerSearchResult);
controlledSites.Add(crawlerStartModel.URL);
if (crawlerStartModel.Depth - 1 > currentDepth)
{
List <string> subsites = FindSubSites(response);
currentDepth++;
foreach (var item in subsites)
{
await Search(new CrawlerStartModel()
{
URL = item,
Expression = crawlerStartModel.Expression,
Depth = crawlerStartModel.Depth,
}, currentDepth, controlledSites, crawleName);
}
}
}
}
public async Task InsertCrawlerSearch(CrawlerSearch crawlerSearch)
{
await _context.CrawlerSearch.AddAsync(crawlerSearch);
await _context.SaveChangesAsync();
}
