public void HappyFlow()
{
// 500m on i9-9900 takes about 5-8secs.. bottle-necked by consumer thread. (Release build)
long expectPayloadCount = 500000000;
TestContext.WriteLine($"Pipe allocation {_stopwatch.Elapsed}");
IPipe <long> sut = new BatchingPipe <long>(100000000);
TestContext.WriteLine($"Pipe allocation end {_stopwatch.Elapsed}");
// Limited buffer prevents this test eating more than ~1GB
Task producingTask = RunProducingTask(sut, expectPayloadCount, 100000000);
TestContext.WriteLine($"Consumer stating {_stopwatch.Elapsed}");
long expectedNextValue = 0;
foreach (long actualValue in sut.ToEnumerable())
{
// Tried using NUnit assert first - Assert.AreEqual(expectedNextValue++, actualValue)
// But it introduced 99%+ of cpu overhead making this test run for minutes instead of seconds.
if (expectedNextValue++ != actualValue)
{
Assert.Fail($"Order was not preserved e:{expectedNextValue-1} a:{actualValue}");
}
}
Assert.IsTrue(producingTask.IsCompletedSuccessfully);
Assert.AreEqual(expectPayloadCount, expectedNextValue);
TestContext.WriteLine($"Consumer completed {_stopwatch.Elapsed}");
}
public static IEnumerable <IEnumerable <T> > ToBatchesAsync <T>(this IEnumerable <T> source, int batchSize)
{
using IEnumerator <T> enumerator = source.GetEnumerator();
int index = 0;
while (enumerator.MoveNext())
{
BatchingPipe <T> batch = new BatchingPipe <T>();
Task task = Task.Run(() => ProduceBatch(enumerator, batch, batchSize));
yield return(batch.ToEnumerable());
Task.WaitAll(task);
if (!task.IsCompletedSuccessfully)
{
throw new Exception("Failure while reading source enumerable", task.Exception);
}
}
}