public void TestMethod1()
{
DuplicateRemover remover = new DuplicateRemover();
int result1 = remover.RemoveDuplicates(new int[] { 1, 1, 1 });
int result2 = remover.RemoveDuplicatesLinq(new int[] { 1, 1, 1 });
int result3 = remover.RemoveDuplicates(new int[] { 1 });
int result4 = remover.RemoveDuplicatesLinq(new int[] { 1 });
int result5 = remover.RemoveDuplicates(new int[] { 1, 2, 3, 4, 4, 5 });
int result6 = remover.RemoveDuplicatesLinq(new int[] { 1, 2, 3, 4, 4, 5 });
int result7 = remover.RemoveDuplicates(new int[] { 1, 2, 3, 4, 5 });
int result8 = remover.RemoveDuplicatesLinq(new int[] { 1, 2, 3, 4, 5 });
int result9 = remover.RemoveDuplicates(null);
int result10 = remover.RemoveDuplicatesLinq(null);
Assert.AreEqual(1, result1);
Assert.AreEqual(1, result2);
Assert.AreEqual(1, result3);
Assert.AreEqual(1, result4);
Assert.AreEqual(5, result5);
Assert.AreEqual(5, result6);
Assert.AreEqual(5, result7);
Assert.AreEqual(5, result8);
Assert.AreEqual(0, result9);
Assert.AreEqual(0, result10);
}
private void DoPush(Request request)
{
if (!DuplicateRemover.IsDuplicate(request) || ShouldReserved(request))
{
PushWhenNoDuplicate(request);
}
}
/// <summary>
/// Reset duplicate check.
/// </summary>
public override void ResetDuplicateCheck()
{
lock (_lock)
{
_queue.Clear();
DuplicateRemover.ResetDuplicateCheck();
}
}
/// <summary>
/// 批量导入
/// </summary>
/// <param name="requests">请求对象</param>
public override void Reload(ICollection <Request> requests)
{
if (requests == null)
{
return;
}
lock (_lock)
{
_queue.Clear();
foreach (var request in requests)
{
if (!DuplicateRemover.IsDuplicate(request))
{
_queue.Add(request);
}
}
}
}
/// <summary>
/// 添加请求对象到队列
/// </summary>
/// <param name="request">请求对象</param>
public void Push(Request request)
{
var action = new Action(() =>
{
if (!DuplicateRemover.IsDuplicate(request))
{
PushWhenNoDuplicate(request);
}
});
if (UseInternet)
{
NetworkCenter.Current.Execute("sch-push", action);
}
else
{
action();
}
}
public int GetTotalRequestsCount(ISpider spider)
{
return(DuplicateRemover.GetTotalRequestsCount(spider));
}
/// <summary>
/// 重置去重器
/// </summary>
public override void ResetDuplicateCheck()
{
DuplicateRemover.ResetDuplicateCheck();
}
public virtual void Clear()
{
DuplicateRemover.ResetDuplicateCheck();
}
public virtual void Dispose()
{
DuplicateRemover.Dispose();
}
public int GetTotalRequestsCount(ITask task)
{
return(DuplicateRemover.GetTotalRequestsCount(task));
}
public void VerifyThreeCharMatchIsHandled()
{
string output = DuplicateRemover.Remove("helhel");
Assert.AreEqual("hel", output);
}
public void EmptyStringReturnsEmptyString()
{
string output = DuplicateRemover.Remove(string.Empty);
Assert.IsEmpty(output);
}
public void NullStringReturnsEmptyString()
{
string output = DuplicateRemover.Remove(null);
Assert.IsEmpty(output);
}
public void VerifyShortStringsAreUnchanged()
{
string output = DuplicateRemover.Remove("he");
Assert.AreEqual("he", output);
}
public void VerifySimpleOutput()
{
string output = DuplicateRemover.Remove("hellohello");
Assert.AreEqual("hello", output);
}
public override long GetTotalRequestsCount()
{
return(DuplicateRemover.GetTotalRequestsCount());
}
public void VerifyNoDuplicatesLeavesStringUnchanged()
{
string output = DuplicateRemover.Remove("hello");
Assert.AreEqual("hello", output);
}
public virtual void Dispose()
{
DuplicateRemover.Dispose();
IsExited = true;
}
public void TestRepetitions(string withoutRepetitions, int maxRepetitions, string withRepetitions)
{
Assert.Equal(withRepetitions, DuplicateRemover.RemoveRepetitions(withoutRepetitions, maxRepetitions));
}
