public virtual void Crawl()
{
var request = WebRequest.Create(_requestUriString);
using (var response = request.GetResponse().GetResponseStream())
{
if (response == null)
{
return;
}
var document = new HtmlDocument();
document.Load(response, true);
var nodes = document.DocumentNode.SelectNodes(_nodesXPath);
var allKeywords = _repository.GetAllKeywords();
foreach (var node in nodes)
{
try
{
var offer = _parser.ParseOffer(node);
if (_repository.InsertOffer(offer))
{
var offerKeywords = OfferHelper.ScanTextForKeywords(offer.Text, allKeywords);
_repository.InsertKeywordsForOffer(offer.Sha1, offerKeywords);
}
}
catch (Exception ex)
{
var log = NLog.LogManager.GetCurrentClassLogger();
log.LogException(NLog.LogLevel.Fatal, "Exception occured", ex);
}
}
}
}
