public void GiveInitialTasks()
{
var dbContext = new Crawler_DBEntities();
// ConvertSeedLinkDTOtoDB(dbContext.Seeds.FirstOrDefault());
if (!dbContext.Seeds.Any())
{
return;
}
globalSeedStack.PushRange(dbContext.Seeds.ToArray());
var crawlerCounter = 0;
foreach (var crawlerCallback in _connectedClientCrawlers)
{
Seed nextSeed;
if (globalSeedStack.TryPop(out nextSeed))
{
crawlerCallback.SavedCallback.StartCrawling(Mapper.Map<SeedDTO>(nextSeed));
crawlerCounter++;
}
else
{
//throw new Exception("Couldn't pop from stack");
}
}
modelReference.AppendTextToConsole("Test Crawling started for " + crawlerCounter + " crawlers");
}
public void GiveTestInitialTasks()
{
var dbContext = new Crawler_DBEntities();
globalSeedStack.PushRange(dbContext.Seeds.ToArray());
var crawlerCounter = 0;
foreach (var crawlerCallback in _connectedClientCrawlers)
{
Seed nextSeed;
if (globalSeedStack.TryPop(out nextSeed))
{
crawlerCallback.SavedCallback.StartTestCrawl();
crawlerCounter++;
}
else
{
//throw new Exception("Couldn't pop from stack");
}
}
modelReference.AppendTextToConsole("Test Crawling started for " + crawlerCounter + " crawlers");
}
private void SaveClientResultsToDatabase(CrawlerResultsDTO resultsDto)
{
modelReference.AppendTextToConsole("Crawling results from" + resultsDto.ProcessedSeed.SeedDomainName + " acquired. Saving onto DB...");
var dbContext = new Crawler_DBEntities();
var dbBatch = Mapper.Map<Batch>(resultsDto.BatchInfo);
dbBatch.CrawlerConnectionId = resultsDto.ConnectionInfo.Id;
dbContext.Batches.Add(dbBatch);
if (resultsDto.InternalLinksList != null)
{
foreach (var internalLink in resultsDto.InternalLinksList)
{
var dbLink = ConvertInternalLinkDTOtoDB(internalLink);
dbContext.InternalLinks.Add(dbLink);
}
}
if (resultsDto.ExternalLinksList != null)
{
foreach (var externalLink in resultsDto.ExternalLinksList)
{
var dbLink = ConvertExternalLinkDTOtoDB(externalLink);
dbContext.ExternalLinks.Add(dbLink);
}
}
if (resultsDto.BadLinksList != null)
{
foreach (var badLink in resultsDto.BadLinksList)
{
var dbLink = ConvertBadLinkDTOtoDB(badLink);
dbContext.BadLinks.Add(dbLink);
}
}
dbContext.SaveChanges();
}
public ClientCrawlerInfo[] Join(ClientCrawlerInfo clientCrawlerNewInfo)
{
clientCrawlerNewInfo.SavedCallback = OperationContext.Current.GetCallbackChannel<ICrawlerClientCallback>();
ConnectedClientCrawlers.Add(clientCrawlerNewInfo);
var dbContext = new Crawler_DBEntities();
var dbConnInfo = SeedModel.DataAccessService.ConvertToCrawlerConnection(clientCrawlerNewInfo);
dbConnInfo.ConnectionTime = DateTime.Now;
dbContext.CrawlerConnections.Add(dbConnInfo);
dbContext.SaveChanges();
return ConnectedClientCrawlers.ToArray();
}
