public void CollectorStacker() { CollectorPool stack = new CollectorPool(); Assert.IsTrue(stack.Next() == "google.com"); Assert.IsTrue(stack.Next() == "facebook.com"); stack.Store("jubacs.somee.com"); Assert.IsTrue(stack.Next() == "jubacs.somee.com"); }
static string PoolManager( CollectorPool pool, VisitedUrls urlHistory, VisitedDomains domainHistory) { string link = pool.Next(); Uri uri = link.ToUri(); if (uri.IsNull()) return string.Empty; // No more links in the pool // Check if url has been visited within this session while ( urlHistory.ContainsUrl(uri.ToString()) || domainHistory.ContainsDomain(uri.ToString()) || IsExcludedDomain(pool, uri)) { link = pool.Next(); uri = link.ToUri(); if (uri.IsNull()) return string.Empty; // No more links in the pool } // Recursively check if host is still processed //if (domainHistory.ContainsDomain(uri.ToString())) //{ // // Store the link back to pool // pool.Store(uri.ToString()); // return PoolManager(pool, urlHistory, domainHistory); //} //else //{ // Link is ok, add to history urlHistory.Add(uri.ToString()); domainHistory.Add(uri.ToString()); return uri.ToString(); //} }
static List<Collector> CreateCollectorPool( CollectorPool pool, VisitedUrls history, int parallelCount) { List<Collector> collectors = new List<Collector>(parallelCount); int i = 0; while (i < parallelCount) { string link = pool.Next(); if (!history.ContainsUrl(link.ToUri().ToString())) { collectors.Add(new Collector { SeqNo = i, Link = link }); history.Add(link.ToUri().ToString()); ++i; } } return collectors; }