public void AddUrl(UrlEntries urlEntries)
{
if (urlEntries == null)
{
return;
}
_webCrawlerDbContext.UrlEntries.Add(urlEntries);
_webCrawlerDbContext.SaveChanges();
}
/// <summary>
/// Searches all the urls for the given.
/// </summary>
/// <param name="urlAddress">The URL address.</param>
private void SearchAllTheUrls(Uri urlAddress)
{
if (!_hashUrlSet.Contains(urlAddress) && urlAddress.Host.Contains(_domainName))
{
var htmlData = GetUrlData(urlAddress);
if (htmlData != null)
{
var hostUri = urlAddress;
var htmlDataUrlList = Extract(htmlData);
var expressionCount = Regex.Matches(htmlData, _expression, RegexOptions.IgnoreCase).Count;
if (_hashUrlSet.Add(urlAddress))
{
var urlEntries = new UrlEntries
{
Url = urlAddress,
Date = DateTimeOffset.Now,
EnteredExpression = _expression,
NumberOfHits = expressionCount
};
//Performance issue
//_urlEntries.Add(urlEntries);
AddUrl(urlEntries);
}
var newSubUrls = GetUniqueUrls(htmlDataUrlList, hostUri);
foreach (var subHashUrl in newSubUrls)
{
SearchAllTheUrls(subHashUrl);
}
}
}
}
