public async Task CrawlerStart(CrawlerStartModel crawlerStartModel)
{
List <string> controlledSites = new List <string>();
StringBuilder crawlerName = new StringBuilder();
crawlerName.AppendFormat("\"{0}\" in \"{1}\"", crawlerStartModel.Expression, crawlerStartModel.URL);
await Search(crawlerStartModel, 0, controlledSites, crawlerName.ToString());
}
private async Task Search(CrawlerStartModel crawlerStartModel, int currentDepth, List <string> controlledSites, string crawleName)
{
if (!controlledSites.Any(x => x == crawlerStartModel.URL))
{
WebRequest wr = WebRequest.Create(crawlerStartModel.URL);
WebResponse ws = wr.GetResponse();
StreamReader sr = new StreamReader(ws.GetResponseStream(), Encoding.UTF8);
string response = sr.ReadToEnd();
sr.Close();
ws.Close();
CrawlerSearch crawlerSearchResult = new CrawlerSearch()
{
Date = DateTime.Now,
URL = crawlerStartModel.URL,
Expression = crawlerStartModel.Expression,
Crawlername = crawleName
};
if (response.Length > 0)
{
crawlerSearchResult.HitCount = response.Split(crawlerStartModel.Expression).Count() - 1;
}
else
{
crawlerSearchResult.HitCount = 0;
}
await _crawlerSearchRepository.InsertCrawlerSearch(crawlerSearchResult);
controlledSites.Add(crawlerStartModel.URL);
if (crawlerStartModel.Depth - 1 > currentDepth)
{
List <string> subsites = FindSubSites(response);
currentDepth++;
foreach (var item in subsites)
{
await Search(new CrawlerStartModel()
{
URL = item,
Expression = crawlerStartModel.Expression,
Depth = crawlerStartModel.Depth,
}, currentDepth, controlledSites, crawleName);
}
}
}
}
public async Task <IActionResult> Create([Bind("URL,Expression,Depth")] CrawlerStartModel crawlerStartModel)
{
if (crawlerStartModel.Depth == null)
{
crawlerStartModel.Depth = 1;
}
if (ModelState.IsValid)
{
await _crawlerSearchService.CrawlerStart(crawlerStartModel);
return(RedirectToAction(nameof(Finish)));
}
return(View(crawlerStartModel));
}
