public void should_flush_multiple_times() { var collection = new FlushableBlockingCollection <int>(); var consumedItems = new List <int>(); var consumerTask = Task.Run(() => { foreach (var batch in collection.GetConsumingEnumerable(10)) { consumedItems.AddRange(batch); Thread.Yield(); } Console.WriteLine("Consumer done"); }); var flushedItems = new List <ConcurrentQueue <int> >(); var flusherTask = Task.Run(() => { for (var i = 0; i < 10; ++i) { flushedItems.Add(collection.Flush()); Thread.Sleep(50); } Console.WriteLine("Flusher done"); }); var adderTask = Task.Run(() => { foreach (var item in Enumerable.Range(0, 5000000).Select(x => 3 * x)) { collection.Add(item); Thread.Yield(); } Console.WriteLine("Adder done"); }); adderTask.Wait(); collection.CompleteAdding(); Task.WaitAll(consumerTask, flusherTask); var flushedItemCount = flushedItems.Sum(x => x.Count); Console.WriteLine("{0} flushed items", flushedItemCount); consumedItems.AddRange(flushedItems.SelectMany(x => x)); consumedItems.Count.ShouldEqual(5000000); }
public void MeasureLatency() { var queue = new FlushableBlockingCollection<Stopwatch>(); var elapsed = TimeSpan.Zero; var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => elapsed += x.Elapsed)); for (var i = 0; i < 5000; ++i) { if (i % 30 == 0) Thread.Sleep(100); queue.Add(Stopwatch.StartNew()); } queue.CompleteAdding(); dequeue.Wait(); Console.WriteLine("{0}", elapsed.TotalMilliseconds / 5000); }
public void MeasureLatency() { var queue = new FlushableBlockingCollection <Stopwatch>(); var elapsed = TimeSpan.Zero; var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => elapsed += x.Elapsed)); for (var i = 0; i < 5000; ++i) { if (i % 30 == 0) { Thread.Sleep(100); } queue.Add(Stopwatch.StartNew()); } queue.CompleteAdding(); dequeue.Wait(); Console.WriteLine("{0}", elapsed.TotalMilliseconds / 5000); }
public void should_flush_collection_with_multiple_writers() { var collection = new FlushableBlockingCollection <int>(); var consumedItems = new List <int>(); var consume = Task.Run(() => { foreach (var batch in collection.GetConsumingEnumerable(5)) { consumedItems.AddRange(batch); // simulate consumption lag Thread.Sleep(0); } Console.WriteLine("{0} consumed items", consumedItems.Count); Console.WriteLine("Consumer done"); }); const int writerItemCount = 30000; var t1 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x)) { collection.Add(item); Thread.SpinWait(1 << 4); } Console.WriteLine("T1 done"); }); var t2 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1)) { collection.Add(item); Thread.SpinWait(1 << 4); } Console.WriteLine("T2 done"); }); var t3 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2)) { collection.Add(item); Thread.SpinWait(1 << 4); Thread.SpinWait(1 << 4); } Console.WriteLine("T3 done"); }); Thread.Sleep(100); Console.WriteLine("Flush #1"); var flushedItems1 = collection.Flush(); Thread.Sleep(100); Console.WriteLine("Flush #2"); var flushedItems2 = collection.Flush(); Task.WaitAll(t1, t2, t3); collection.CompleteAdding(); consume.Wait(); Console.WriteLine("{0} flushed items (#1)", flushedItems1.Count); Console.WriteLine("{0} flushed items (#2)", flushedItems2.Count); const int expectedCount = writerItemCount * 3; var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList(); items.Count.ShouldEqual(expectedCount); foreach (var item in items) { item.ShouldBeGreaterOrEqualThan(0); item.ShouldBeLessThan(expectedCount); } }
public void should_flush_collection_with_single_writer() { var collection = new FlushableBlockingCollection <int>(); var consumedItems = new List <int>(); var consume = Task.Run(() => { foreach (var batch in collection.GetConsumingEnumerable(10)) { consumedItems.AddRange(batch); // simulate very slow consumer Thread.Sleep(2); } Console.WriteLine("{0} consumed items", consumedItems.Count); Console.WriteLine("Consumer done"); }); const int batchSize = 500000; foreach (var item in Enumerable.Range(0 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #1"); var flushedItems1 = collection.Flush(); Console.WriteLine("{0} flushed items", flushedItems1.Count); foreach (var item in Enumerable.Range(1 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #2"); var flushedItems2 = collection.Flush(); Console.WriteLine("{0} flushed items", flushedItems2.Count); foreach (var item in Enumerable.Range(2 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #3"); var flushedItems3 = collection.Flush(); Console.WriteLine("{0} flushed items", flushedItems3.Count); collection.CompleteAdding(); consume.Wait(); const int expectedCount = 1500000; var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).Concat(flushedItems3).ToList(); items.Count.ShouldEqual(expectedCount); foreach (var item in items) { item.ShouldBeGreaterOrEqualThan(0); item.ShouldBeLessThan(expectedCount); } }
public void Enqueue(MessageDispatch dispatch, IMessageHandlerInvoker invoker) { _queue.Add(new Entry(dispatch, invoker)); }
protected override void QueueTask(Task task) { _tasks.Add(task); }
public void should_flush_collection_with_multiple_writers() { var collection = new FlushableBlockingCollection<int>(); var consumedItems = new List<int>(); var consume = Task.Run(() => { var index = 0; foreach (var item in collection.GetConsumingEnumerable()) { consumedItems.Add(item); // simulate consumption lag if (index % 10000 == 0) Thread.Sleep(20); ++index; } Console.WriteLine("Consumer done"); }); const int writerItemCount = 300000; var t1 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x)) { collection.Add(item); if ((item - 0) % 1000 == 0) Thread.Sleep(10); else Thread.Yield(); } Console.WriteLine("T1 done"); }); var t2 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1)) { collection.Add(item); if ((item - 1) % 1000 == 0) Thread.Sleep(10); else Thread.Yield(); } Console.WriteLine("T2 done"); }); var t3 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2)) { collection.Add(item); if ((item - 2) % 1000 == 0) Thread.Sleep(10); else Thread.Yield(); } Console.WriteLine("T3 done"); }); Thread.Sleep(50); Console.WriteLine("Flush #1"); var flushedItems1 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems1.Count); Thread.Sleep(50); Console.WriteLine("Flush #2"); var flushedItems2 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems2.Count); Task.WaitAll(t1, t2, t3); collection.CompleteAdding(); consume.Wait(); var exectedItems = Enumerable.Range(0, writerItemCount * 3).ToHashSet(); var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList(); items.Count.ShouldEqual(exectedItems.Count); foreach (var item in items) { exectedItems.Contains(item).ShouldBeTrue(); } }
public void should_flush_multiple_times(bool waitForCompletion) { var collection = new FlushableBlockingCollection<int>(); var consumedItems = new List<int>(); var consumerTask = Task.Run(() => { foreach (var item in collection.GetConsumingEnumerable()) { consumedItems.Add(item); Thread.Yield(); } Console.WriteLine("Consumer done"); }); var flushedItems = new List<ConcurrentQueue<int>>(); var flusherTask = Task.Run(() => { for (var i = 0; i < 10; ++i) { flushedItems.Add(collection.Flush(waitForCompletion)); Thread.Sleep(50); } Console.WriteLine("Flusher done"); }); var adderTask = Task.Run(() => { foreach (var item in Enumerable.Range(0, 5000000).Select(x => 3 * x)) { collection.Add(item); Thread.Yield(); } Console.WriteLine("Adder done"); }); adderTask.Wait(); collection.CompleteAdding(); Task.WaitAll(consumerTask, flusherTask); var flushedItemCount = flushedItems.Sum(x => x.Count); Console.WriteLine("{0} flushed items", flushedItemCount); consumedItems.AddRange(flushedItems.SelectMany(x => x)); consumedItems.Count.ShouldEqual(5000000); }
public void should_flush_collection_with_single_writer() { var collection = new FlushableBlockingCollection<int>(); var consumedItems = new List<int>(); var consume = Task.Run(() => { foreach (var item in collection.GetConsumingEnumerable()) { consumedItems.Add(item); // simulate very slow consumer Thread.Sleep(10); } Console.WriteLine("Consumer done"); }); const int batchSize = 500000; foreach (var item in Enumerable.Range(0 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #1"); var flushedItems1 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems1.Count); foreach (var item in Enumerable.Range(1 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #2"); var flushedItems2 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems2.Count); foreach (var item in Enumerable.Range(2 * batchSize, batchSize)) { collection.Add(item); } Thread.Sleep(100); Console.WriteLine("Flush #3"); var flushedItems3 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems3.Count); collection.CompleteAdding(); consume.Wait(); var exectedItems = Enumerable.Range(0, 1500000).ToHashSet(); var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).Concat(flushedItems3).ToList(); items.Count.ShouldEqual(exectedItems.Count); foreach (var item in items) { exectedItems.Contains(item).ShouldBeTrue(); } }
public void should_flush_collection() { var collection = new FlushableBlockingCollection <int>(); var consumedItems = new List <int>(); var consume = Task.Run(() => { var index = 0; foreach (var item in collection.GetConsumingEnumerable()) { consumedItems.Add(item); // simulate consumption lag if (index >= 500 && item <= 550) { Thread.Sleep(50); } ++index; } Console.WriteLine("Consumer done"); }); var t1 = Task.Run(() => { foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x)) { collection.Add(item); Thread.Yield(); } Console.WriteLine("T1 done"); }); var t2 = Task.Run(() => { foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x + 1)) { collection.Add(item); Thread.Yield(); } Console.WriteLine("T2 done"); }); var t3 = Task.Run(() => { foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x + 2)) { collection.Add(item); Thread.Yield(); } Console.WriteLine("T3 done"); }); Thread.Sleep(20); Console.WriteLine("Trying to flush"); var flushedItems1 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems1.Count); Thread.Sleep(20); Console.WriteLine("Trying to flush"); var flushedItems2 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems2.Count); Task.WaitAll(t1, t2, t3); collection.CompleteAdding(); Task.WaitAll(consume); var exectedItems = Enumerable.Range(0, 1500000).ToHashSet(); var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList(); items.Count.ShouldEqual(exectedItems.Count); foreach (var item in items) { exectedItems.Contains(item).ShouldBeTrue(); } }
public void should_flush_collection_with_multiple_writers() { var collection = new FlushableBlockingCollection <int>(); var consumedItems = new List <int>(); var consume = Task.Run(() => { var index = 0; foreach (var item in collection.GetConsumingEnumerable()) { consumedItems.Add(item); // simulate consumption lag if (index % 10000 == 0) { Thread.Sleep(20); } ++index; } Console.WriteLine("Consumer done"); }); const int writerItemCount = 300000; var t1 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x)) { collection.Add(item); if ((item - 0) % 1000 == 0) { Thread.Sleep(10); } else { Thread.Yield(); } } Console.WriteLine("T1 done"); }); var t2 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1)) { collection.Add(item); if ((item - 1) % 1000 == 0) { Thread.Sleep(10); } else { Thread.Yield(); } } Console.WriteLine("T2 done"); }); var t3 = Task.Run(() => { foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2)) { collection.Add(item); if ((item - 2) % 1000 == 0) { Thread.Sleep(10); } else { Thread.Yield(); } } Console.WriteLine("T3 done"); }); Thread.Sleep(50); Console.WriteLine("Flush #1"); var flushedItems1 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems1.Count); Thread.Sleep(50); Console.WriteLine("Flush #2"); var flushedItems2 = collection.Flush(true); Console.WriteLine("{0} flushed items", flushedItems2.Count); Task.WaitAll(t1, t2, t3); collection.CompleteAdding(); consume.Wait(); var exectedItems = Enumerable.Range(0, writerItemCount * 3).ToHashSet(); var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList(); items.Count.ShouldEqual(exectedItems.Count); foreach (var item in items) { exectedItems.Contains(item).ShouldBeTrue(); } }