public bool IsDuplicate(Request request)
{
bool isDuplicate = _bloomFilter.Contains(request.Identity);
if (!isDuplicate)
{
_bloomFilter.Add(request.Identity);
_counter.Inc();
}
return(isDuplicate);
}
public bool IsDuplicate(Request request, ISpider spider)
{
bool isDuplicate = _bloomFilter.Contains(request.Url);
if (!isDuplicate)
{
_bloomFilter.Add(request.Url);
_counter.Inc();
}
return(isDuplicate);
}
/// <summary>
/// Check whether the request is duplicate.
/// </summary>
/// <param name="request">Request</param>
/// <returns>Whether the request is duplicate.</returns>
public bool IsDuplicate(Request request)
{
var identity = request.GetIdentity();
bool isDuplicate = _bloomFilter.Contains(identity);
if (!isDuplicate)
{
_bloomFilter.Add(identity);
_counter.Inc();
}
return(isDuplicate);
}
public Request Poll(ITask task)
{
if (_count.Inc() > 1000)
{
return(null);
}
if (_random.Next(100) > 90)
{
return(null);
}
return(new Request("test", null));
}
/// <summary>
/// Check whether the request is duplicate.
/// </summary>
/// <param name="request">Request</param>
/// <returns>Whether the request is duplicate.</returns>
public bool IsDuplicate(Request request)
{
Check.NotNull(request.OwnerId, nameof(request.OwnerId));
var hash = request.Hash;
bool isDuplicate = _bloomFilter.Contains(hash);
if (!isDuplicate)
{
_bloomFilter.Add(hash);
_counter.Inc();
}
return(isDuplicate);
}
public Request Poll(ISpider spider)
{
lock (this)
{
if (_count.Inc() > 1000)
{
return(null);
}
if (_random.Next(100) > 90)
{
return(null);
}
return(new Request("test", 1, null));
}
}
