public void ParallelHashSetDuplicate() { var ownerId = Guid.NewGuid().ToString("N"); HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover(); var r1 = new Request("http://www.a.com") { OwnerId = ownerId }; r1.ComputeHash(); bool isDuplicate = scheduler.IsDuplicate(r1); Assert.False(isDuplicate); Parallel.For(0, 1000, new ParallelOptions { MaxDegreeOfParallelism = 30 }, i => { var r = new Request("http://www.a.com") { OwnerId = ownerId }; r.ComputeHash(); isDuplicate = scheduler.IsDuplicate(r); Assert.True(isDuplicate); }); }
public async Task ParallelHashSetDuplicate() { var hashAlgorithm = new MurmurHashAlgorithmService(); var ownerId = Guid.NewGuid().ToString("N"); var scheduler = new HashSetDuplicateRemover(); var requestHasher = new RequestHasher(hashAlgorithm); var r1 = new Request("http://www.a.com") { Owner = ownerId }; r1.Hash = requestHasher.ComputeHash(r1); var isDuplicate = await scheduler.IsDuplicateAsync(r1); Assert.False(isDuplicate); Parallel.For(0, 1000, new ParallelOptions { MaxDegreeOfParallelism = 30 }, async i => { var r = new Request("http://www.a.com") { Owner = ownerId }; r.Hash = requestHasher.ComputeHash(r); isDuplicate = await scheduler.IsDuplicateAsync(r); Assert.True(isDuplicate); }); }
public void TestMemory() { int times = 5000000; IDuplicateRemover duplicateRemover = new BloomFilterDuplicateRemover(times, 0.005); long used = GetProcessUsedMemory(); long time = DateTime.UtcNow.CurrentTimeMillis(); for (int i = 0; i < times; i++) { duplicateRemover.IsDuplicate(new Request(i.ToString(), null), null); } Console.WriteLine("Time used by bloomfilter:" + (DateTime.UtcNow.CurrentTimeMillis() - time)); Console.WriteLine("Memory used by bloomfilter:" + (GetProcessUsedMemory() - used)); duplicateRemover = new HashSetDuplicateRemover(); GC.Collect(); used = GetProcessUsedMemory(); time = DateTime.UtcNow.CurrentTimeMillis(); for (int i = 0; i < times; i++) { duplicateRemover.IsDuplicate(new Request(i.ToString(), null), null); } Console.WriteLine("Time used by hashset:" + (DateTime.UtcNow.CurrentTimeMillis() - time)); Console.WriteLine("Memory used by hashset:" + (GetProcessUsedMemory() - used)); }
public void HashSetDuplicate() { HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover(); bool isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null); Assert.IsFalse(isDuplicate); isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null); Assert.IsTrue(isDuplicate); isDuplicate = scheduler.IsDuplicate(new Request("http://www.b.com", 1, null), null); Assert.IsFalse(isDuplicate); isDuplicate = scheduler.IsDuplicate(new Request("http://www.b.com", 1, null), null); Assert.IsTrue(isDuplicate); }
public void HashSetDuplicateSynchronized() { HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover(); bool isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null); Assert.IsFalse(isDuplicate); Parallel.For(0, 1000, new ParallelOptions() { MaxDegreeOfParallelism = 30 }, i => { isDuplicate = scheduler.IsDuplicate(new Request("http://www.a.com", 1, null), null); Assert.IsTrue(isDuplicate); }); }
public async Task HashSetDuplicate() { var hashAlgorithm = new MurmurHashAlgorithmService(); var scheduler = new HashSetDuplicateRemover(); var requestHasher = new RequestHasher(hashAlgorithm); var ownerId = Guid.NewGuid().ToString("N"); await scheduler.InitializeAsync(ownerId); var r1 = new Request("http://www.a.com") { Owner = ownerId }; r1.Headers.Accept = ("asdfasdfasdf"); requestHasher.ComputeHash(r1); var isDuplicate = await scheduler.IsDuplicateAsync(r1); Assert.False(isDuplicate); var r2 = new Request("http://www.a.com") { Owner = ownerId }; requestHasher.ComputeHash(r2); isDuplicate = await scheduler.IsDuplicateAsync(r2); Assert.True(isDuplicate); var r3 = new Request("http://www.b.com") { Owner = ownerId }; requestHasher.ComputeHash(r3); isDuplicate = await scheduler.IsDuplicateAsync(r3); Assert.False(isDuplicate); var r4 = new Request("http://www.b.com") { Owner = ownerId }; requestHasher.ComputeHash(r4); isDuplicate = await scheduler.IsDuplicateAsync(r4); Assert.True(isDuplicate); }
public async Task HashSetDuplicate() { var scheduler = new HashSetDuplicateRemover(); var ownerId = Guid.NewGuid().ToString("N"); var r1 = new Request("http://www.a.com") { Owner = ownerId, Accept = "asdfasdfasdf" }; r1.Hash = r1.ComputeHash(); var isDuplicate = await scheduler.IsDuplicateAsync(r1); Assert.False(isDuplicate); var r2 = new Request("http://www.a.com") { Owner = ownerId }; r2.Hash = r2.ComputeHash(); isDuplicate = await scheduler.IsDuplicateAsync(r2); Assert.True(isDuplicate); var r3 = new Request("http://www.b.com") { Owner = ownerId }; r3.Hash = r3.ComputeHash(); isDuplicate = await scheduler.IsDuplicateAsync(r3); Assert.False(isDuplicate); var r4 = new Request("http://www.b.com") { Owner = ownerId }; r4.Hash = r4.ComputeHash(); isDuplicate = await scheduler.IsDuplicateAsync(r4); Assert.True(isDuplicate); }
public void HashSetDuplicate() { HashSetDuplicateRemover scheduler = new HashSetDuplicateRemover(); var ownerId = Guid.NewGuid().ToString("N"); var r1 = new Request("http://www.a.com") { OwnerId = ownerId }; r1.ComputeHash(); bool isDuplicate = scheduler.IsDuplicate(r1); Assert.False(isDuplicate); var r2 = new Request("http://www.a.com") { OwnerId = ownerId }; r2.ComputeHash(); isDuplicate = scheduler.IsDuplicate(r2); Assert.True(isDuplicate); var r3 = new Request("http://www.b.com") { OwnerId = ownerId }; r3.ComputeHash(); isDuplicate = scheduler.IsDuplicate(r3); Assert.False(isDuplicate); var r4 = new Request("http://www.b.com") { OwnerId = ownerId }; r4.ComputeHash(); isDuplicate = scheduler.IsDuplicate(r4); Assert.True(isDuplicate); }