public virtual void Crawl() { var request = WebRequest.Create(_requestUriString); using (var response = request.GetResponse().GetResponseStream()) { if (response == null) { return; } var document = new HtmlDocument(); document.Load(response, true); var nodes = document.DocumentNode.SelectNodes(_nodesXPath); var allKeywords = _repository.GetAllKeywords(); foreach (var node in nodes) { try { var offer = _parser.ParseOffer(node); if (_repository.InsertOffer(offer)) { var offerKeywords = OfferHelper.ScanTextForKeywords(offer.Text, allKeywords); _repository.InsertKeywordsForOffer(offer.Sha1, offerKeywords); } } catch (Exception ex) { var log = NLog.LogManager.GetCurrentClassLogger(); log.LogException(NLog.LogLevel.Fatal, "Exception occured", ex); } } } }