private IRateLimiter GetRateLimiter(WebProxy proxy, Uri uri, long minCrawlDelayInMillisecs)
{
ConcurrentDictionary<string, IRateLimiter> rateLimiterDict;
_multiProxyRateLimiterLookup.TryGetValue(proxy.Address, out rateLimiterDict);
if (rateLimiterDict == null)
{
rateLimiterDict = new ConcurrentDictionary<string, IRateLimiter>();
_multiProxyRateLimiterLookup.TryAdd(proxy.Address, rateLimiterDict);
}
IRateLimiter rateLimiter;
rateLimiterDict.TryGetValue(uri.Authority, out rateLimiter);
if (rateLimiter != null) return rateLimiter;
long crawlDelayMs;
if (!_domainLimitDictionary.TryGetValue(uri.Authority, out crawlDelayMs))
crawlDelayMs = minCrawlDelayInMillisecs;
rateLimiter = new RateLimiter(1, TimeSpan.FromMilliseconds(crawlDelayMs));
//rateLimiter = new SimpleRateLimiter(proxy, uri.Authority, TimeSpan.FromMilliseconds(crawlDelayMs));
if (rateLimiterDict.TryAdd(uri.Authority, rateLimiter))
_logger.DebugFormat("Added new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
else
_logger.WarnFormat("Unable to add new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
return rateLimiter;
}
public void WaitToProceed_ForCoverage()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(1));
Stopwatch timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed(TimeSpan.FromSeconds(10));
_unitUnderTest.Dispose();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds < 1);
}
public void WaitToProceed_PartialSecWait_Waits()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(1.5));
Stopwatch timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed();//will not wait for first call
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds > 1.5);
}
public void WaitToProceed_ZeroWait_DoesntWait()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(0));
Stopwatch timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds < 1);
}
public void WaitToProceed_NegativeMilli()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(1));
_unitUnderTest.WaitToProceed(-10);
}
