public static void SpawnExternalCrawl(IRepository repo, int sessionId, int crawlerId, Uri externalUri) { //if we are less than max concurrent crawls //and this crawl is not defined //spawn the process int inProgress = repo.GetCountOfCrawlsInProgress(sessionId); if (inProgress < repo.GetSession(sessionId).MaxConcurrentCrawls) { int nextCrawlerId = -1; bool alreadyDefined = false; do { nextCrawlerId = repo.GetNextCrawlerId(sessionId); alreadyDefined = repo.GetCrawl(sessionId, nextCrawlerId) != null; } while (alreadyDefined || nextCrawlerId <= crawlerId); //Spawn { string args = string.Format("{0} {1} {2}", sessionId, nextCrawlerId, externalUri.AbsoluteUri); ProcessStartInfo psi = new ProcessStartInfo("ThrongBot.CrawlRunner.exe", args); Process p = Process.Start(psi); Console.WriteLine("Process {0} spawned, crawlerId: {1}, seedUrl: {1}", p.Id, nextCrawlerId, externalUri.AbsoluteUri); } } }
public static bool CanExternalCrawlBeSpawned(IRepository repo, int sessionId) { // check config canSpawnCrawlsConfigSetting first bool canSpawn = false; int inProgress = repo.GetCountOfCrawlsInProgress(sessionId); if (inProgress < repo.GetSession(sessionId).MaxConcurrentCrawls) { canSpawn = true; } return(canSpawn); }
public static bool CanExternalCrawlBeSpawned(IRepository repo, int sessionId) { // check config canSpawnCrawlsConfigSetting first bool canSpawn = false; int inProgress = repo.GetCountOfCrawlsInProgress(sessionId); if (inProgress < repo.GetSession(sessionId).MaxConcurrentCrawls) { canSpawn = true; } return canSpawn; }