private void ThreadProc()
{
foreach (var t in _tasks.GetConsumingEnumerable())
{
if (!IsRunning)
{
break;
}
TryExecuteTask(t);
}
_logger.Info(_threadName + " processing stopped");
}
public void MeasureCpuUsage()
{
var queue = new FlushableBlockingCollection <Stopwatch>();
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => { }));
Console.WriteLine("Use the Process Explorer, Luke");
Thread.Sleep(20000);
queue.CompleteAdding();
dequeue.Wait();
}
public void should_flush_multiple_times()
{
var collection = new FlushableBlockingCollection <int>();
var consumedItems = new List <int>();
var consumerTask = Task.Run(() =>
{
foreach (var batch in collection.GetConsumingEnumerable(10))
{
consumedItems.AddRange(batch);
Thread.Yield();
}
Console.WriteLine("Consumer done");
});
var flushedItems = new List <ConcurrentQueue <int> >();
var flusherTask = Task.Run(() =>
{
for (var i = 0; i < 10; ++i)
{
flushedItems.Add(collection.Flush());
Thread.Sleep(50);
}
Console.WriteLine("Flusher done");
});
var adderTask = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, 5000000).Select(x => 3 * x))
{
collection.Add(item);
Thread.Yield();
}
Console.WriteLine("Adder done");
});
adderTask.Wait();
collection.CompleteAdding();
Task.WaitAll(consumerTask, flusherTask);
var flushedItemCount = flushedItems.Sum(x => x.Count);
Console.WriteLine("{0} flushed items", flushedItemCount);
consumedItems.AddRange(flushedItems.SelectMany(x => x));
consumedItems.Count.ShouldEqual(5000000);
}
public void MeasureThroughput()
{
var queue = new FlushableBlockingCollection <int>();
var watch = Stopwatch.StartNew();
var enqueue = Task.Run(() => Enumerable.Range(0, 50000000).ForEach(queue.Add));
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => { }));
enqueue.Wait();
queue.CompleteAdding();
dequeue.Wait();
Console.WriteLine("{0} items processed in {1}", 50000000, watch.Elapsed);
}
public void MeasureThroughput()
{
var queue = new FlushableBlockingCollection<int>();
var watch = Stopwatch.StartNew();
var enqueue = Task.Run(() => Enumerable.Range(0, 50000000).ForEach(queue.Add));
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => { }));
enqueue.Wait();
queue.CompleteAdding();
dequeue.Wait();
Console.WriteLine("{0} items processed in {1}", 50000000, watch.Elapsed);
}
private void ThreadProc()
{
_currentDispatchQueueName = Name;
try
{
_logger.InfoFormat("{0} processing started", Name);
var batch = new Batch(_batchSize);
foreach (var entries in _queue.GetConsumingEnumerable(_batchSize))
{
ProcessEntries(entries, batch);
}
_logger.InfoFormat("{0} processing stopped", Name);
}
finally
{
_currentDispatchQueueName = null;
}
}
public void MeasureLatency()
{
var queue = new FlushableBlockingCollection<Stopwatch>();
var elapsed = TimeSpan.Zero;
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => elapsed += x.Elapsed));
for (var i = 0; i < 5000; ++i)
{
if (i % 30 == 0)
Thread.Sleep(100);
queue.Add(Stopwatch.StartNew());
}
queue.CompleteAdding();
dequeue.Wait();
Console.WriteLine("{0}", elapsed.TotalMilliseconds / 5000);
}
public void MeasureLatency()
{
var queue = new FlushableBlockingCollection <Stopwatch>();
var elapsed = TimeSpan.Zero;
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => elapsed += x.Elapsed));
for (var i = 0; i < 5000; ++i)
{
if (i % 30 == 0)
{
Thread.Sleep(100);
}
queue.Add(Stopwatch.StartNew());
}
queue.CompleteAdding();
dequeue.Wait();
Console.WriteLine("{0}", elapsed.TotalMilliseconds / 5000);
}
public void should_get_consuming_enumerable()
{
var consumedItems = new List<int>();
var bc = new FlushableBlockingCollection<int>();
var t1 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x).ForEach(bc.Add));
var t2 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x + 1).ForEach(bc.Add));
var consume = Task.Run(() => bc.GetConsumingEnumerable().ForEach(consumedItems.Add));
Task.WaitAll(t1, t2);
bc.CompleteAdding();
consume.Wait();
var expectedItems = Enumerable.Range(0, 2000000).ToHashSet();
consumedItems.Count.ShouldEqual(expectedItems.Count);
foreach (var item in consumedItems)
{
expectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_get_consuming_enumerable()
{
var consumedItems = new List <int>();
var bc = new FlushableBlockingCollection <int>();
var t1 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x).ForEach(bc.Add));
var t2 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x + 1).ForEach(bc.Add));
var consume = Task.Run(() => bc.GetConsumingEnumerable().ForEach(consumedItems.Add));
Task.WaitAll(t1, t2);
bc.CompleteAdding();
consume.Wait();
var expectedItems = Enumerable.Range(0, 2000000).ToHashSet();
consumedItems.Count.ShouldEqual(expectedItems.Count);
foreach (var item in consumedItems)
{
expectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_get_consuming_enumerable(int batchSize)
{
var consumedItems = new List <int>();
var bc = new FlushableBlockingCollection <int>();
var t1 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x).ForEach(bc.Add));
var t2 = Task.Run(() => Enumerable.Range(0, 1000000).Select(x => 2 * x + 1).ForEach(bc.Add));
var consume = Task.Run(() => bc.GetConsumingEnumerable(batchSize).ForEach(consumedItems.AddRange));
Task.WaitAll(t1, t2);
bc.CompleteAdding();
consume.Wait();
const int itemCount = 2000000;
consumedItems.Count.ShouldEqual(itemCount);
foreach (var item in consumedItems)
{
item.ShouldBeGreaterOrEqualThan(0);
item.ShouldBeLessThan(itemCount);
}
}
public void should_flush_collection_with_single_writer()
{
var collection = new FlushableBlockingCollection<int>();
var consumedItems = new List<int>();
var consume = Task.Run(() =>
{
foreach (var item in collection.GetConsumingEnumerable())
{
consumedItems.Add(item);
// simulate very slow consumer
Thread.Sleep(10);
}
Console.WriteLine("Consumer done");
});
const int batchSize = 500000;
foreach (var item in Enumerable.Range(0 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #1");
var flushedItems1 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems1.Count);
foreach (var item in Enumerable.Range(1 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #2");
var flushedItems2 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems2.Count);
foreach (var item in Enumerable.Range(2 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #3");
var flushedItems3 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems3.Count);
collection.CompleteAdding();
consume.Wait();
var exectedItems = Enumerable.Range(0, 1500000).ToHashSet();
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).Concat(flushedItems3).ToList();
items.Count.ShouldEqual(exectedItems.Count);
foreach (var item in items)
{
exectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_flush_collection()
{
var collection = new FlushableBlockingCollection <int>();
var consumedItems = new List <int>();
var consume = Task.Run(() =>
{
var index = 0;
foreach (var item in collection.GetConsumingEnumerable())
{
consumedItems.Add(item);
// simulate consumption lag
if (index >= 500 && item <= 550)
{
Thread.Sleep(50);
}
++index;
}
Console.WriteLine("Consumer done");
});
var t1 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x))
{
collection.Add(item);
Thread.Yield();
}
Console.WriteLine("T1 done");
});
var t2 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x + 1))
{
collection.Add(item);
Thread.Yield();
}
Console.WriteLine("T2 done");
});
var t3 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, 500000).Select(x => 3 * x + 2))
{
collection.Add(item);
Thread.Yield();
}
Console.WriteLine("T3 done");
});
Thread.Sleep(20);
Console.WriteLine("Trying to flush");
var flushedItems1 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems1.Count);
Thread.Sleep(20);
Console.WriteLine("Trying to flush");
var flushedItems2 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems2.Count);
Task.WaitAll(t1, t2, t3);
collection.CompleteAdding();
Task.WaitAll(consume);
var exectedItems = Enumerable.Range(0, 1500000).ToHashSet();
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList();
items.Count.ShouldEqual(exectedItems.Count);
foreach (var item in items)
{
exectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_flush_collection_with_multiple_writers()
{
var collection = new FlushableBlockingCollection <int>();
var consumedItems = new List <int>();
var consume = Task.Run(() =>
{
var index = 0;
foreach (var item in collection.GetConsumingEnumerable())
{
consumedItems.Add(item);
// simulate consumption lag
if (index % 10000 == 0)
{
Thread.Sleep(20);
}
++index;
}
Console.WriteLine("Consumer done");
});
const int writerItemCount = 300000;
var t1 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x))
{
collection.Add(item);
if ((item - 0) % 1000 == 0)
{
Thread.Sleep(10);
}
else
{
Thread.Yield();
}
}
Console.WriteLine("T1 done");
});
var t2 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1))
{
collection.Add(item);
if ((item - 1) % 1000 == 0)
{
Thread.Sleep(10);
}
else
{
Thread.Yield();
}
}
Console.WriteLine("T2 done");
});
var t3 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2))
{
collection.Add(item);
if ((item - 2) % 1000 == 0)
{
Thread.Sleep(10);
}
else
{
Thread.Yield();
}
}
Console.WriteLine("T3 done");
});
Thread.Sleep(50);
Console.WriteLine("Flush #1");
var flushedItems1 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems1.Count);
Thread.Sleep(50);
Console.WriteLine("Flush #2");
var flushedItems2 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems2.Count);
Task.WaitAll(t1, t2, t3);
collection.CompleteAdding();
consume.Wait();
var exectedItems = Enumerable.Range(0, writerItemCount * 3).ToHashSet();
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList();
items.Count.ShouldEqual(exectedItems.Count);
foreach (var item in items)
{
exectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_flush_collection_with_single_writer()
{
var collection = new FlushableBlockingCollection <int>();
var consumedItems = new List <int>();
var consume = Task.Run(() =>
{
foreach (var batch in collection.GetConsumingEnumerable(10))
{
consumedItems.AddRange(batch);
// simulate very slow consumer
Thread.Sleep(2);
}
Console.WriteLine("{0} consumed items", consumedItems.Count);
Console.WriteLine("Consumer done");
});
const int batchSize = 500000;
foreach (var item in Enumerable.Range(0 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #1");
var flushedItems1 = collection.Flush();
Console.WriteLine("{0} flushed items", flushedItems1.Count);
foreach (var item in Enumerable.Range(1 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #2");
var flushedItems2 = collection.Flush();
Console.WriteLine("{0} flushed items", flushedItems2.Count);
foreach (var item in Enumerable.Range(2 * batchSize, batchSize))
{
collection.Add(item);
}
Thread.Sleep(100);
Console.WriteLine("Flush #3");
var flushedItems3 = collection.Flush();
Console.WriteLine("{0} flushed items", flushedItems3.Count);
collection.CompleteAdding();
consume.Wait();
const int expectedCount = 1500000;
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).Concat(flushedItems3).ToList();
items.Count.ShouldEqual(expectedCount);
foreach (var item in items)
{
item.ShouldBeGreaterOrEqualThan(0);
item.ShouldBeLessThan(expectedCount);
}
}
public void should_flush_multiple_times(bool waitForCompletion)
{
var collection = new FlushableBlockingCollection<int>();
var consumedItems = new List<int>();
var consumerTask = Task.Run(() =>
{
foreach (var item in collection.GetConsumingEnumerable())
{
consumedItems.Add(item);
Thread.Yield();
}
Console.WriteLine("Consumer done");
});
var flushedItems = new List<ConcurrentQueue<int>>();
var flusherTask = Task.Run(() =>
{
for (var i = 0; i < 10; ++i)
{
flushedItems.Add(collection.Flush(waitForCompletion));
Thread.Sleep(50);
}
Console.WriteLine("Flusher done");
});
var adderTask = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, 5000000).Select(x => 3 * x))
{
collection.Add(item);
Thread.Yield();
}
Console.WriteLine("Adder done");
});
adderTask.Wait();
collection.CompleteAdding();
Task.WaitAll(consumerTask, flusherTask);
var flushedItemCount = flushedItems.Sum(x => x.Count);
Console.WriteLine("{0} flushed items", flushedItemCount);
consumedItems.AddRange(flushedItems.SelectMany(x => x));
consumedItems.Count.ShouldEqual(5000000);
}
public void should_flush_collection_with_multiple_writers()
{
var collection = new FlushableBlockingCollection<int>();
var consumedItems = new List<int>();
var consume = Task.Run(() =>
{
var index = 0;
foreach (var item in collection.GetConsumingEnumerable())
{
consumedItems.Add(item);
// simulate consumption lag
if (index % 10000 == 0)
Thread.Sleep(20);
++index;
}
Console.WriteLine("Consumer done");
});
const int writerItemCount = 300000;
var t1 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x))
{
collection.Add(item);
if ((item - 0) % 1000 == 0)
Thread.Sleep(10);
else
Thread.Yield();
}
Console.WriteLine("T1 done");
});
var t2 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1))
{
collection.Add(item);
if ((item - 1) % 1000 == 0)
Thread.Sleep(10);
else
Thread.Yield();
}
Console.WriteLine("T2 done");
});
var t3 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2))
{
collection.Add(item);
if ((item - 2) % 1000 == 0)
Thread.Sleep(10);
else
Thread.Yield();
}
Console.WriteLine("T3 done");
});
Thread.Sleep(50);
Console.WriteLine("Flush #1");
var flushedItems1 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems1.Count);
Thread.Sleep(50);
Console.WriteLine("Flush #2");
var flushedItems2 = collection.Flush(true);
Console.WriteLine("{0} flushed items", flushedItems2.Count);
Task.WaitAll(t1, t2, t3);
collection.CompleteAdding();
consume.Wait();
var exectedItems = Enumerable.Range(0, writerItemCount * 3).ToHashSet();
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList();
items.Count.ShouldEqual(exectedItems.Count);
foreach (var item in items)
{
exectedItems.Contains(item).ShouldBeTrue();
}
}
public void should_flush_collection_with_multiple_writers()
{
var collection = new FlushableBlockingCollection <int>();
var consumedItems = new List <int>();
var consume = Task.Run(() =>
{
foreach (var batch in collection.GetConsumingEnumerable(5))
{
consumedItems.AddRange(batch);
// simulate consumption lag
Thread.Sleep(0);
}
Console.WriteLine("{0} consumed items", consumedItems.Count);
Console.WriteLine("Consumer done");
});
const int writerItemCount = 30000;
var t1 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x))
{
collection.Add(item);
Thread.SpinWait(1 << 4);
}
Console.WriteLine("T1 done");
});
var t2 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 1))
{
collection.Add(item);
Thread.SpinWait(1 << 4);
}
Console.WriteLine("T2 done");
});
var t3 = Task.Run(() =>
{
foreach (var item in Enumerable.Range(0, writerItemCount).Select(x => 3 * x + 2))
{
collection.Add(item);
Thread.SpinWait(1 << 4);
Thread.SpinWait(1 << 4);
}
Console.WriteLine("T3 done");
});
Thread.Sleep(100);
Console.WriteLine("Flush #1");
var flushedItems1 = collection.Flush();
Thread.Sleep(100);
Console.WriteLine("Flush #2");
var flushedItems2 = collection.Flush();
Task.WaitAll(t1, t2, t3);
collection.CompleteAdding();
consume.Wait();
Console.WriteLine("{0} flushed items (#1)", flushedItems1.Count);
Console.WriteLine("{0} flushed items (#2)", flushedItems2.Count);
const int expectedCount = writerItemCount * 3;
var items = consumedItems.Concat(flushedItems1).Concat(flushedItems2).ToList();
items.Count.ShouldEqual(expectedCount);
foreach (var item in items)
{
item.ShouldBeGreaterOrEqualThan(0);
item.ShouldBeLessThan(expectedCount);
}
}
public void MeasureCpuUsage()
{
var queue = new FlushableBlockingCollection<Stopwatch>();
var dequeue = Task.Run(() => queue.GetConsumingEnumerable().ForEach(x => { }));
Console.WriteLine("Use the Process Explorer, Luke");
Thread.Sleep(20000);
queue.CompleteAdding();
dequeue.Wait();
}