public override PipelinedCrawlDescription Flow(PipelinedCrawlDescription pipelinedCrawlDescription) { HashSet <Tuple <short, short> > proxyShortages = new HashSet <Tuple <short, short> >(); LinkedListNode <CrawlDescription> node = pipelinedCrawlDescription.CrawlDescriptions.First; while (node != null) { Proxy proxy = _proxyManager.GetAvailableProxy( node.Value.SearchEngineID, node.Value.RegionID); if (proxy != null) { // add proxy to crawl description node.Value.ProxyID = proxy.ProxyID; node.Value.IP = proxy.IP; node.Value.Port = proxy.Port; node.Value.ProxyRegion = proxy.RegionID; } else { // add the search engine and region so we can // calculate the next availability proxyShortages.Add(new Tuple <short, short>( node.Value.SearchEngineID, node.Value.RegionID)); // mark the node for deletion pipelinedCrawlDescription.MarkedForRemoval.Add(node); } node = node.Next; } // if there are no proxies available for some of the // crawl descriptions if (proxyShortages.Count > 0) { DateTime nextAvailability = _proxyManager.GetNextAvailability(proxyShortages); // if some other pipe set the next availability // because of a lack of resources, let it take // precedence if it is greater than the time // a proxy is available if (nextAvailability > pipelinedCrawlDescription.NextAvailability) { pipelinedCrawlDescription.NextAvailability = nextAvailability; } } if (_connection != null) { _connection.Flow(pipelinedCrawlDescription); } return(pipelinedCrawlDescription); }