public async Task CrawlerStart(CrawlerStartModel crawlerStartModel) { List <string> controlledSites = new List <string>(); StringBuilder crawlerName = new StringBuilder(); crawlerName.AppendFormat("\"{0}\" in \"{1}\"", crawlerStartModel.Expression, crawlerStartModel.URL); await Search(crawlerStartModel, 0, controlledSites, crawlerName.ToString()); }
private async Task Search(CrawlerStartModel crawlerStartModel, int currentDepth, List <string> controlledSites, string crawleName) { if (!controlledSites.Any(x => x == crawlerStartModel.URL)) { WebRequest wr = WebRequest.Create(crawlerStartModel.URL); WebResponse ws = wr.GetResponse(); StreamReader sr = new StreamReader(ws.GetResponseStream(), Encoding.UTF8); string response = sr.ReadToEnd(); sr.Close(); ws.Close(); CrawlerSearch crawlerSearchResult = new CrawlerSearch() { Date = DateTime.Now, URL = crawlerStartModel.URL, Expression = crawlerStartModel.Expression, Crawlername = crawleName }; if (response.Length > 0) { crawlerSearchResult.HitCount = response.Split(crawlerStartModel.Expression).Count() - 1; } else { crawlerSearchResult.HitCount = 0; } await _crawlerSearchRepository.InsertCrawlerSearch(crawlerSearchResult); controlledSites.Add(crawlerStartModel.URL); if (crawlerStartModel.Depth - 1 > currentDepth) { List <string> subsites = FindSubSites(response); currentDepth++; foreach (var item in subsites) { await Search(new CrawlerStartModel() { URL = item, Expression = crawlerStartModel.Expression, Depth = crawlerStartModel.Depth, }, currentDepth, controlledSites, crawleName); } } } }
public async Task <IActionResult> Create([Bind("URL,Expression,Depth")] CrawlerStartModel crawlerStartModel) { if (crawlerStartModel.Depth == null) { crawlerStartModel.Depth = 1; } if (ModelState.IsValid) { await _crawlerSearchService.CrawlerStart(crawlerStartModel); return(RedirectToAction(nameof(Finish))); } return(View(crawlerStartModel)); }