public void DeleteCrawledLink(Guid id)
{
if (CrawledLinks.ContainsKey(id))
{
CrawledLinks.Remove(id);
}
}
public void AddCrawledLink(CrawledLink link, bool removeCorrespondingLinkToCrawl)
{
// delete from LinksToCrawl first
Thread.Sleep(100);
if (removeCorrespondingLinkToCrawl)
{
DeleteLinkToCrawl(link.SessionId, link.SourceUrl, link.TargetUrl);
}
link.Id = NextId;
CrawledLinks.Add(link.Id, link);
}
public void ClearCrawledLinks(int sessionId, int crawlerId)
{
List <Guid> toRemove = new List <Guid>();
foreach (var link in CrawledLinks.Values)
{
if (link.SessionId == sessionId && link.CrawlerId == crawlerId)
{
toRemove.Add(link.Id);
}
}
foreach (var id in toRemove)
{
CrawledLinks.Remove(id);
}
}
public int GetCountOfCrawledLinks(int sessionId, int crawlerId)
{
return(CrawledLinks.Count());
}