public void ParallelHashSetDuplicate()
{
var ownerId = Guid.NewGuid().ToString("N");
HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover();
var r1 = new Request("http://www.a.com")
{
OwnerId = ownerId
};
r1.ComputeHash();
bool isDuplicate = scheduler.IsDuplicate(r1);
Assert.False(isDuplicate);
Parallel.For(0, 1000, new ParallelOptions {
MaxDegreeOfParallelism = 30
}, i =>
{
var r = new Request("http://www.a.com")
{
OwnerId = ownerId
};
r.ComputeHash();
isDuplicate = scheduler.IsDuplicate(r);
Assert.True(isDuplicate);
});
}
public async Task ParallelHashSetDuplicate()
{
var hashAlgorithm = new MurmurHashAlgorithmService();
var ownerId = Guid.NewGuid().ToString("N");
var scheduler = new HashSetDuplicateRemover();
var requestHasher = new RequestHasher(hashAlgorithm);
var r1 = new Request("http://www.a.com")
{
Owner = ownerId
};
r1.Hash = requestHasher.ComputeHash(r1);
var isDuplicate = await scheduler.IsDuplicateAsync(r1);
Assert.False(isDuplicate);
Parallel.For(0, 1000, new ParallelOptions {
MaxDegreeOfParallelism = 30
}, async i =>
{
var r = new Request("http://www.a.com")
{
Owner = ownerId
};
r.Hash = requestHasher.ComputeHash(r);
isDuplicate = await scheduler.IsDuplicateAsync(r);
Assert.True(isDuplicate);
});
}
public void TestMemory()
{
int times = 5000000;
IDuplicateRemover duplicateRemover = new BloomFilterDuplicateRemover(times, 0.005);
long used = GetProcessUsedMemory();
long time = DateTime.UtcNow.CurrentTimeMillis();
for (int i = 0; i < times; i++)
{
duplicateRemover.IsDuplicate(new Request(i.ToString(), null), null);
}
Console.WriteLine("Time used by bloomfilter:" + (DateTime.UtcNow.CurrentTimeMillis() - time));
Console.WriteLine("Memory used by bloomfilter:" + (GetProcessUsedMemory() - used));
duplicateRemover = new HashSetDuplicateRemover();
GC.Collect();
used = GetProcessUsedMemory();
time = DateTime.UtcNow.CurrentTimeMillis();
for (int i = 0; i < times; i++)
{
duplicateRemover.IsDuplicate(new Request(i.ToString(), null), null);
}
Console.WriteLine("Time used by hashset:" + (DateTime.UtcNow.CurrentTimeMillis() - time));
Console.WriteLine("Memory used by hashset:" + (GetProcessUsedMemory() - used));
}
public void HashSetDuplicate()
{
HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover();
bool isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null);
Assert.IsFalse(isDuplicate);
isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null);
Assert.IsTrue(isDuplicate);
isDuplicate = scheduler.IsDuplicate(new Request("http://www.b.com", 1, null), null);
Assert.IsFalse(isDuplicate);
isDuplicate = scheduler.IsDuplicate(new Request("http://www.b.com", 1, null), null);
Assert.IsTrue(isDuplicate);
}
public void HashSetDuplicateSynchronized()
{
HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover();
bool isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null);
Assert.IsFalse(isDuplicate);
Parallel.For(0, 1000, new ParallelOptions()
{
MaxDegreeOfParallelism = 30
}, i =>
{
isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null);
Assert.IsTrue(isDuplicate);
});
}
public async Task HashSetDuplicate()
{
var hashAlgorithm = new MurmurHashAlgorithmService();
var scheduler = new HashSetDuplicateRemover();
var requestHasher = new RequestHasher(hashAlgorithm);
var ownerId = Guid.NewGuid().ToString("N");
await scheduler.InitializeAsync(ownerId);
var r1 = new Request("http://www.a.com")
{
Owner = ownerId
};
r1.Headers.Accept = ("asdfasdfasdf");
requestHasher.ComputeHash(r1);
var isDuplicate = await scheduler.IsDuplicateAsync(r1);
Assert.False(isDuplicate);
var r2 = new Request("http://www.a.com")
{
Owner = ownerId
};
requestHasher.ComputeHash(r2);
isDuplicate = await scheduler.IsDuplicateAsync(r2);
Assert.True(isDuplicate);
var r3 = new Request("http://www.b.com")
{
Owner = ownerId
};
requestHasher.ComputeHash(r3);
isDuplicate = await scheduler.IsDuplicateAsync(r3);
Assert.False(isDuplicate);
var r4 = new Request("http://www.b.com")
{
Owner = ownerId
};
requestHasher.ComputeHash(r4);
isDuplicate = await scheduler.IsDuplicateAsync(r4);
Assert.True(isDuplicate);
}
public async Task HashSetDuplicate()
{
var scheduler = new HashSetDuplicateRemover();
var ownerId = Guid.NewGuid().ToString("N");
var r1 = new Request("http://www.a.com")
{
Owner = ownerId,
Accept = "asdfasdfasdf"
};
r1.Hash = r1.ComputeHash();
var isDuplicate = await scheduler.IsDuplicateAsync(r1);
Assert.False(isDuplicate);
var r2 = new Request("http://www.a.com")
{
Owner = ownerId
};
r2.Hash = r2.ComputeHash();
isDuplicate = await scheduler.IsDuplicateAsync(r2);
Assert.True(isDuplicate);
var r3 = new Request("http://www.b.com")
{
Owner = ownerId
};
r3.Hash = r3.ComputeHash();
isDuplicate = await scheduler.IsDuplicateAsync(r3);
Assert.False(isDuplicate);
var r4 = new Request("http://www.b.com")
{
Owner = ownerId
};
r4.Hash = r4.ComputeHash();
isDuplicate = await scheduler.IsDuplicateAsync(r4);
Assert.True(isDuplicate);
}
public void HashSetDuplicate()
{
HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover();
var ownerId = Guid.NewGuid().ToString("N");
var r1 = new Request("http://www.a.com")
{
OwnerId = ownerId
};
r1.ComputeHash();
bool isDuplicate = scheduler.IsDuplicate(r1);
Assert.False(isDuplicate);
var r2 = new Request("http://www.a.com")
{
OwnerId = ownerId
};
r2.ComputeHash();
isDuplicate = scheduler.IsDuplicate(r2);
Assert.True(isDuplicate);
var r3 = new Request("http://www.b.com")
{
OwnerId = ownerId
};
r3.ComputeHash();
isDuplicate = scheduler.IsDuplicate(r3);
Assert.False(isDuplicate);
var r4 = new Request("http://www.b.com")
{
OwnerId = ownerId
};
r4.ComputeHash();
isDuplicate = scheduler.IsDuplicate(r4);
Assert.True(isDuplicate);
}
