private void SaveClientResultsToDatabase(CrawlerResultsDTO resultsDto)
{
modelReference.AppendTextToConsole("Crawling results from" + resultsDto.ProcessedSeed.SeedDomainName + " acquired. Saving onto DB...");
var dbContext = new Crawler_DBEntities();
var dbBatch = Mapper.Map<Batch>(resultsDto.BatchInfo);
dbBatch.CrawlerConnectionId = resultsDto.ConnectionInfo.Id;
dbContext.Batches.Add(dbBatch);
if (resultsDto.InternalLinksList != null)
{
foreach (var internalLink in resultsDto.InternalLinksList)
{
var dbLink = ConvertInternalLinkDTOtoDB(internalLink);
dbContext.InternalLinks.Add(dbLink);
}
}
if (resultsDto.ExternalLinksList != null)
{
foreach (var externalLink in resultsDto.ExternalLinksList)
{
var dbLink = ConvertExternalLinkDTOtoDB(externalLink);
dbContext.ExternalLinks.Add(dbLink);
}
}
if (resultsDto.BadLinksList != null)
{
foreach (var badLink in resultsDto.BadLinksList)
{
var dbLink = ConvertBadLinkDTOtoDB(badLink);
dbContext.BadLinks.Add(dbLink);
}
}
dbContext.SaveChanges();
}
public void ReturnCrawlingResults(CrawlerResultsDTO resultsDto)
{
SaveClientResultsToDatabase(resultsDto);
Seed nextSeed;
if (globalSeedStack.TryPop(out nextSeed))
{
var foundcallback = _connectedClientCrawlers.FirstOrDefault(crw => crw.ClientIdentifier == resultsDto.ConnectionInfo.Id);
foundcallback?.SavedCallback.StartCrawling(Mapper.Map<SeedDTO>(nextSeed));
}
else
{
//throw new Exception("Couldn't pop from stack");
}
}